mirror of
https://github.com/restic/restic.git
synced 2026-02-24 09:46:24 +00:00
Compare commits
2 Commits
v0.12.1
...
debug-wron
| Author | SHA1 | Date | |
|---|---|---|---|
|
271eabd239 | ||
|
|
0333ef3325 |
32
.github/PULL_REQUEST_TEMPLATE.md
vendored
32
.github/PULL_REQUEST_TEMPLATE.md
vendored
@@ -1,7 +1,13 @@
|
||||
|
||||
|
||||
<!--
|
||||
Thank you very much for contributing code or documentation to restic! Please
|
||||
fill out the following questions to make it easier for us to review your
|
||||
changes.
|
||||
|
||||
You do not need to check all the boxes below all at once, feel free to take
|
||||
your time and add more commits. If you're done and ready for review, please
|
||||
check the last box.
|
||||
-->
|
||||
|
||||
What does this PR change? What problem does it solve?
|
||||
@@ -11,8 +17,8 @@ What does this PR change? What problem does it solve?
|
||||
Describe the changes and their purpose here, as detailed as needed.
|
||||
-->
|
||||
|
||||
Was the change previously discussed in an issue or on the forum?
|
||||
----------------------------------------------------------------
|
||||
Was the change discussed in an issue or in the forum before?
|
||||
------------------------------------------------------------
|
||||
|
||||
<!--
|
||||
Link issues and relevant forum posts here.
|
||||
@@ -24,17 +30,11 @@ is closed automatically when this PR is merged.
|
||||
Checklist
|
||||
---------
|
||||
|
||||
<!--
|
||||
You do not need to check all the boxes below all at once. Feel free to take
|
||||
your time and add more commits. If you're done and ready for review, please
|
||||
check the last box. Enable a checkbox by replacing [ ] with [x].
|
||||
-->
|
||||
|
||||
- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
|
||||
- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
|
||||
- [ ] I have added tests for all code changes.
|
||||
- [ ] I have added documentation for relevant changes (in the manual).
|
||||
- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
|
||||
- [ ] I have run `gofmt` on the code in all commits.
|
||||
- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
|
||||
- [ ] I'm done! This pull request is ready for review.
|
||||
- [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
|
||||
- [ ] I have enabled [maintainer edits for this PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork)
|
||||
- [ ] I have added tests for all changes in this PR
|
||||
- [ ] I have added documentation for the changes (in the manual)
|
||||
- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))
|
||||
- [ ] I have run `gofmt` on the code in all commits
|
||||
- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits)
|
||||
- [ ] I'm done, this Pull Request is ready for review
|
||||
|
||||
271
.github/workflows/tests.yml
vendored
271
.github/workflows/tests.yml
vendored
@@ -1,271 +0,0 @@
|
||||
name: test
|
||||
on:
|
||||
# run tests on push to master, but not when other branches are pushed to
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# run tests for all pull requests
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
# list of jobs to run:
|
||||
include:
|
||||
- job_name: Windows
|
||||
go: 1.16.x
|
||||
os: windows-latest
|
||||
|
||||
- job_name: macOS
|
||||
go: 1.16.x
|
||||
os: macOS-latest
|
||||
test_fuse: false
|
||||
|
||||
- job_name: Linux
|
||||
go: 1.16.x
|
||||
os: ubuntu-latest
|
||||
test_cloud_backends: true
|
||||
test_fuse: true
|
||||
check_changelog: true
|
||||
|
||||
- job_name: Linux
|
||||
go: 1.15.x
|
||||
os: ubuntu-latest
|
||||
test_fuse: true
|
||||
|
||||
- job_name: Linux
|
||||
go: 1.14.x
|
||||
os: ubuntu-latest
|
||||
test_fuse: true
|
||||
|
||||
- job_name: Linux
|
||||
go: 1.13.x
|
||||
os: ubuntu-latest
|
||||
test_fuse: true
|
||||
|
||||
name: ${{ matrix.job_name }} Go ${{ matrix.go }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
env:
|
||||
GOPROXY: https://proxy.golang.org
|
||||
|
||||
steps:
|
||||
- name: Set up Go ${{ matrix.go }}
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
|
||||
- name: Get programs (Linux/macOS)
|
||||
run: |
|
||||
echo "build Go tools"
|
||||
go get github.com/restic/rest-server/...
|
||||
|
||||
echo "install minio server"
|
||||
mkdir $HOME/bin
|
||||
if [ "$RUNNER_OS" == "macOS" ]; then
|
||||
wget --no-verbose -O $HOME/bin/minio https://dl.minio.io/server/minio/release/darwin-amd64/minio
|
||||
else
|
||||
wget --no-verbose -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-amd64/minio
|
||||
fi
|
||||
chmod 755 $HOME/bin/minio
|
||||
|
||||
echo "install rclone"
|
||||
if [ "$RUNNER_OS" == "macOS" ]; then
|
||||
wget --no-verbose -O rclone.zip https://downloads.rclone.org/rclone-current-osx-amd64.zip
|
||||
else
|
||||
wget --no-verbose -O rclone.zip https://downloads.rclone.org/rclone-current-linux-amd64.zip
|
||||
fi
|
||||
unzip rclone.zip
|
||||
cp rclone*/rclone $HOME/bin
|
||||
chmod 755 $HOME/bin/rclone
|
||||
rm -rf rclone*
|
||||
|
||||
# add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
|
||||
echo $HOME/bin >> $GITHUB_PATH
|
||||
if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
|
||||
|
||||
- name: Get programs (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
echo "build Go tools"
|
||||
go get github.com/restic/rest-server/...
|
||||
|
||||
echo "install minio server"
|
||||
mkdir $Env:USERPROFILE/bin
|
||||
Invoke-WebRequest https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $Env:USERPROFILE/bin/minio.exe
|
||||
|
||||
echo "install rclone"
|
||||
Invoke-WebRequest https://downloads.rclone.org/rclone-current-windows-amd64.zip -OutFile rclone.zip
|
||||
|
||||
unzip rclone.zip
|
||||
copy rclone*/rclone.exe $Env:USERPROFILE/bin
|
||||
|
||||
# add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
|
||||
echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
|
||||
|
||||
echo "install tar"
|
||||
cd $env:USERPROFILE
|
||||
mkdir tar
|
||||
cd tar
|
||||
|
||||
# install exactly these versions of tar and the libraries, other combinations might not work!
|
||||
|
||||
Invoke-WebRequest https://github.com/restic/test-assets/raw/master/tar-1.13-1-bin.zip -OutFile tar.zip
|
||||
unzip tar.zip
|
||||
Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libintl-0.11.5-2-bin.zip -OutFile libintl.zip
|
||||
unzip libintl.zip
|
||||
Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libiconv-1.8-1-bin.zip -OutFile libiconv.zip
|
||||
unzip libiconv.zip
|
||||
|
||||
# add $USERPROFILE/tar/bin to path
|
||||
echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH
|
||||
if: matrix.os == 'windows-latest'
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Build with build.go
|
||||
run: |
|
||||
go run build.go
|
||||
|
||||
- name: Run local Tests
|
||||
env:
|
||||
RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
|
||||
run: |
|
||||
go test -cover ./...
|
||||
|
||||
- name: Test cloud backends
|
||||
env:
|
||||
RESTIC_TEST_S3_KEY: ${{ secrets.RESTIC_TEST_S3_KEY }}
|
||||
RESTIC_TEST_S3_SECRET: ${{ secrets.RESTIC_TEST_S3_SECRET }}
|
||||
RESTIC_TEST_S3_REPOSITORY: ${{ secrets.RESTIC_TEST_S3_REPOSITORY }}
|
||||
RESTIC_TEST_AZURE_ACCOUNT_NAME: ${{ secrets.RESTIC_TEST_AZURE_ACCOUNT_NAME }}
|
||||
RESTIC_TEST_AZURE_ACCOUNT_KEY: ${{ secrets.RESTIC_TEST_AZURE_ACCOUNT_KEY }}
|
||||
RESTIC_TEST_AZURE_REPOSITORY: ${{ secrets.RESTIC_TEST_AZURE_REPOSITORY }}
|
||||
RESTIC_TEST_B2_ACCOUNT_ID: ${{ secrets.RESTIC_TEST_B2_ACCOUNT_ID }}
|
||||
RESTIC_TEST_B2_ACCOUNT_KEY: ${{ secrets.RESTIC_TEST_B2_ACCOUNT_KEY }}
|
||||
RESTIC_TEST_B2_REPOSITORY: ${{ secrets.RESTIC_TEST_B2_REPOSITORY }}
|
||||
RESTIC_TEST_GS_REPOSITORY: ${{ secrets.RESTIC_TEST_GS_REPOSITORY }}
|
||||
RESTIC_TEST_GS_PROJECT_ID: ${{ secrets.RESTIC_TEST_GS_PROJECT_ID }}
|
||||
GOOGLE_PROJECT_ID: ${{ secrets.RESTIC_TEST_GS_PROJECT_ID }}
|
||||
RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64: ${{ secrets.RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 }}
|
||||
RESTIC_TEST_OS_AUTH_URL: ${{ secrets.RESTIC_TEST_OS_AUTH_URL }}
|
||||
RESTIC_TEST_OS_TENANT_NAME: ${{ secrets.RESTIC_TEST_OS_TENANT_NAME }}
|
||||
RESTIC_TEST_OS_USERNAME: ${{ secrets.RESTIC_TEST_OS_USERNAME }}
|
||||
RESTIC_TEST_OS_PASSWORD: ${{ secrets.RESTIC_TEST_OS_PASSWORD }}
|
||||
RESTIC_TEST_OS_REGION_NAME: ${{ secrets.RESTIC_TEST_OS_REGION_NAME }}
|
||||
RESTIC_TEST_SWIFT: ${{ secrets.RESTIC_TEST_SWIFT }}
|
||||
# fail if any of the following tests cannot be run
|
||||
RESTIC_TEST_DISALLOW_SKIP: "restic/backend/rest.TestBackendREST,\
|
||||
restic/backend/sftp.TestBackendSFTP,\
|
||||
restic/backend/s3.TestBackendMinio,\
|
||||
restic/backend/rclone.TestBackendRclone,\
|
||||
restic/backend/s3.TestBackendS3,\
|
||||
restic/backend/swift.TestBackendSwift,\
|
||||
restic/backend/b2.TestBackendB2,\
|
||||
restic/backend/gs.TestBackendGS,\
|
||||
restic/backend/azure.TestBackendAzure"
|
||||
run: |
|
||||
# prepare credentials for Google Cloud Storage tests in a temp file
|
||||
export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX)
|
||||
echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
|
||||
go test -cover -parallel 4 ./internal/backend/...
|
||||
|
||||
# only run cloud backend tests for pull requests from and pushes to our
|
||||
# own repo, otherwise the secrets are not available
|
||||
if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
|
||||
|
||||
- name: Check changelog files with calens
|
||||
run: |
|
||||
echo "install calens"
|
||||
go get github.com/restic/calens
|
||||
|
||||
echo "check changelog files"
|
||||
calens
|
||||
if: matrix.check_changelog
|
||||
|
||||
cross_compile:
|
||||
strategy:
|
||||
|
||||
# ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
|
||||
matrix:
|
||||
# run cross-compile in two batches parallel so the overall tests run faster
|
||||
targets:
|
||||
- "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
|
||||
openbsd/386 openbsd/amd64"
|
||||
|
||||
- "freebsd/386 freebsd/amd64 freebsd/arm \
|
||||
aix/ppc64 \
|
||||
darwin/amd64 darwin/arm64 \
|
||||
netbsd/386 netbsd/amd64 \
|
||||
windows/386 windows/amd64 \
|
||||
solaris/amd64"
|
||||
|
||||
env:
|
||||
go: 1.16.x
|
||||
GOPROXY: https://proxy.golang.org
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
name: Cross Compile for ${{ matrix.targets }}
|
||||
|
||||
steps:
|
||||
- name: Set up Go ${{ env.go }}
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ env.go }}
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Install gox
|
||||
run: |
|
||||
go get github.com/mitchellh/gox
|
||||
|
||||
- name: Cross-compile with gox for ${{ matrix.targets }}
|
||||
env:
|
||||
GOFLAGS: "-trimpath"
|
||||
GOX_ARCHS: "${{ matrix.targets }}"
|
||||
run: |
|
||||
mkdir build-output
|
||||
gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic
|
||||
gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
go: 1.16.x
|
||||
steps:
|
||||
- name: Set up Go ${{ env.go }}
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ env.go }}
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
|
||||
version: v1.36
|
||||
# Optional: show only new issues if it's a pull request. The default value is `false`.
|
||||
only-new-issues: true
|
||||
args: --verbose --timeout 5m
|
||||
skip-go-installation: true
|
||||
|
||||
# only run golangci-lint for pull requests, otherwise ALL hints get
|
||||
# reported. We need to slowly address all issues until we can enable
|
||||
# linting the master branch :)
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
- name: Check go.mod/go.sum
|
||||
run: |
|
||||
echo "check if go.mod and go.sum are up to date"
|
||||
go mod tidy
|
||||
git diff --exit-code go.mod go.sum
|
||||
@@ -1,57 +0,0 @@
|
||||
# This is the configuration for golangci-lint for the restic project.
|
||||
#
|
||||
# A sample config with all settings is here:
|
||||
# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
|
||||
|
||||
linters:
|
||||
# only enable the linters listed below
|
||||
disable-all: true
|
||||
enable:
|
||||
# make sure all errors returned by functions are handled
|
||||
- errcheck
|
||||
|
||||
# find unused code
|
||||
- deadcode
|
||||
|
||||
# show how code can be simplified
|
||||
- gosimple
|
||||
|
||||
# # make sure code is formatted
|
||||
- gofmt
|
||||
|
||||
# examine code and report suspicious constructs, such as Printf calls whose
|
||||
# arguments do not align with the format string
|
||||
- govet
|
||||
|
||||
# make sure names and comments are used according to the conventions
|
||||
- golint
|
||||
|
||||
# detect when assignments to existing variables are not used
|
||||
- ineffassign
|
||||
|
||||
# run static analysis and find errors
|
||||
- staticcheck
|
||||
|
||||
# find unused variables, functions, structs, types, etc.
|
||||
- unused
|
||||
|
||||
# find unused struct fields
|
||||
- structcheck
|
||||
|
||||
# find unused global variables
|
||||
- varcheck
|
||||
|
||||
# parse and typecheck code
|
||||
- typecheck
|
||||
|
||||
issues:
|
||||
# don't use the default exclude rules, this hides (among others) ignored
|
||||
# errors from Close() calls
|
||||
exclude-use-default: false
|
||||
|
||||
# list of things to not warn about
|
||||
exclude:
|
||||
# golint: do not warn about missing comments for exported stuff
|
||||
- exported (function|method|var|type|const) `.*` should have comment or be unexported
|
||||
# golint: ignore constants in all caps
|
||||
- don't use ALL_CAPS in Go names; use CamelCase
|
||||
2
.hound.yml
Normal file
2
.hound.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
go:
|
||||
enabled: true
|
||||
58
.travis.yml
Normal file
58
.travis.yml
Normal file
@@ -0,0 +1,58 @@
|
||||
language: go
|
||||
sudo: false
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
go: "1.13.x"
|
||||
env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
- os: linux
|
||||
go: "1.14.x"
|
||||
env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
# only run fuse and cloud backends tests on Travis for the latest Go on Linux
|
||||
- os: linux
|
||||
go: "1.15.x"
|
||||
sudo: true
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
- os: osx
|
||||
go: "1.15.x"
|
||||
env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/Library/Caches/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
||||
notifications:
|
||||
irc:
|
||||
channels:
|
||||
- "chat.freenode.net#restic"
|
||||
on_success: change
|
||||
on_failure: change
|
||||
skip_join: true
|
||||
|
||||
install:
|
||||
- go version
|
||||
- export GOBIN="$GOPATH/bin"
|
||||
- export PATH="$PATH:$GOBIN"
|
||||
- go env
|
||||
|
||||
script:
|
||||
- go run run_integration_tests.go
|
||||
743
CHANGELOG.md
743
CHANGELOG.md
@@ -1,746 +1,3 @@
|
||||
Changelog for restic 0.12.1 (2021-08-03)
|
||||
=======================================
|
||||
|
||||
The following sections list the changes in restic 0.12.1 relevant to
|
||||
restic users. The changes are ordered by importance.
|
||||
|
||||
Summary
|
||||
-------
|
||||
|
||||
* Fix #2742: Improve error handling for rclone and REST backend over HTTP2
|
||||
* Fix #3111: Fix terminal output redirection for PowerShell
|
||||
* Fix #3214: Treat an empty password as a fatal error for repository init
|
||||
* Fix #3267: `copy` failed to copy snapshots in rare cases
|
||||
* Fix #3184: `backup --quiet` no longer prints status information
|
||||
* Fix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
|
||||
* Fix #3302: Fix `fdopendir: not a directory` error for local backend
|
||||
* Fix #3334: Print `created new cache` message only on a terminal
|
||||
* Fix #3380: Fix crash of `backup --exclude='**'`
|
||||
* Fix #3305: Fix possibly missing backup summary of JSON output in case of error
|
||||
* Fix #3439: Correctly handle download errors during `restore`
|
||||
* Chg #3247: Empty files now have size of 0 in `ls --json` output
|
||||
* Enh #2780: Add release binaries for s390x architecture on Linux
|
||||
* Enh #3293: Add `--repository-file2` option to `init` and `copy` command
|
||||
* Enh #3312: Add auto-completion support for fish
|
||||
* Enh #3336: SFTP backend now checks for disk space
|
||||
* Enh #3377: Add release binaries for Apple Silicon
|
||||
* Enh #3414: Add `--keep-within-hourly` option to restic forget
|
||||
* Enh #3456: Support filtering and specifying untagged snapshots
|
||||
* Enh #3167: Allow specifying limit of `snapshots` list
|
||||
* Enh #3426: Optimize read performance of mount command
|
||||
* Enh #3427: `find --pack` fallback to index if data file is missing
|
||||
|
||||
Details
|
||||
-------
|
||||
|
||||
* Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2
|
||||
|
||||
When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect
|
||||
when no data was returned at all. This could cause for example the `check` command to report the
|
||||
following error:
|
||||
|
||||
Pack ID does not match, want [...], got e3b0c442
|
||||
|
||||
This has been fixed by correctly detecting and retrying the incomplete download.
|
||||
|
||||
https://github.com/restic/restic/issues/2742
|
||||
https://github.com/restic/restic/pull/3453
|
||||
https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
|
||||
|
||||
* Bugfix #3111: Fix terminal output redirection for PowerShell
|
||||
|
||||
When redirecting the output of restic using PowerShell on Windows, the output contained
|
||||
terminal escape characters. This has been fixed by properly detecting the terminal type.
|
||||
|
||||
In addition, the mintty terminal now shows progress output for the backup command.
|
||||
|
||||
https://github.com/restic/restic/issues/3111
|
||||
https://github.com/restic/restic/pull/3325
|
||||
|
||||
* Bugfix #3214: Treat an empty password as a fatal error for repository init
|
||||
|
||||
When attempting to initialize a new repository, if an empty password was supplied, the
|
||||
repository would be created but the init command would return an error with a stack trace. Now,
|
||||
if an empty password is provided, it is treated as a fatal error, and no repository is created.
|
||||
|
||||
https://github.com/restic/restic/issues/3214
|
||||
https://github.com/restic/restic/pull/3283
|
||||
|
||||
* Bugfix #3267: `copy` failed to copy snapshots in rare cases
|
||||
|
||||
The `copy` command could in rare cases fail with the error message `SaveTree(...) returned
|
||||
unexpected id ...`. This has been fixed.
|
||||
|
||||
On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths.
|
||||
Note that, due to limitations in the repository format, these are not stored properly and
|
||||
should be avoided if possible.
|
||||
|
||||
https://github.com/restic/restic/issues/3267
|
||||
https://github.com/restic/restic/pull/3310
|
||||
|
||||
* Bugfix #3184: `backup --quiet` no longer prints status information
|
||||
|
||||
A regression in the latest restic version caused the output of `backup --quiet` to contain
|
||||
large amounts of backup progress information when run using an interactive terminal. This is
|
||||
fixed now.
|
||||
|
||||
A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
|
||||
|
||||
https://github.com/restic/restic/issues/3184
|
||||
https://github.com/restic/restic/pull/3186
|
||||
|
||||
* Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
|
||||
|
||||
The command `restic check --read-data-subset=x%` crashed when run for an empty repository.
|
||||
This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/3296
|
||||
https://github.com/restic/restic/pull/3309
|
||||
|
||||
* Bugfix #3302: Fix `fdopendir: not a directory` error for local backend
|
||||
|
||||
The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend
|
||||
when the `data` folder in the repository contained files. This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/3302
|
||||
https://github.com/restic/restic/pull/3308
|
||||
|
||||
* Bugfix #3334: Print `created new cache` message only on a terminal
|
||||
|
||||
The message `created new cache` was printed even when the output wasn't a terminal. That broke
|
||||
piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now
|
||||
only printed on a terminal.
|
||||
|
||||
https://github.com/restic/restic/issues/3334
|
||||
https://github.com/restic/restic/pull/3343
|
||||
|
||||
* Bugfix #3380: Fix crash of `backup --exclude='**'`
|
||||
|
||||
The exclude filter `**`, which excludes all files, caused restic to crash. This has been
|
||||
corrected.
|
||||
|
||||
https://github.com/restic/restic/issues/3380
|
||||
https://github.com/restic/restic/pull/3393
|
||||
|
||||
* Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
|
||||
|
||||
When using `--json` output it happened from time to time that the summary output was missing in
|
||||
case an error occurred. This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/pull/3305
|
||||
|
||||
* Bugfix #3439: Correctly handle download errors during `restore`
|
||||
|
||||
Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download
|
||||
errors and only printed a warning. This has been fixed by retrying incomplete data downloads.
|
||||
|
||||
https://github.com/restic/restic/issues/3439
|
||||
https://github.com/restic/restic/pull/3449
|
||||
|
||||
* Change #3247: Empty files now have size of 0 in `ls --json` output
|
||||
|
||||
The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size
|
||||
of zero explicitly for regular files, while omitting the size field for all other types.
|
||||
|
||||
https://github.com/restic/restic/issues/3247
|
||||
https://github.com/restic/restic/pull/3257
|
||||
|
||||
* Enhancement #2780: Add release binaries for s390x architecture on Linux
|
||||
|
||||
We've added release binaries for Linux using the s390x architecture.
|
||||
|
||||
https://github.com/restic/restic/issues/2780
|
||||
https://github.com/restic/restic/pull/3452
|
||||
|
||||
* Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command
|
||||
|
||||
The `init` and `copy` command can now be used with the `--repository-file2` option or the
|
||||
`$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the
|
||||
`--repo2` flag and allow you to read the destination repository from a file.
|
||||
|
||||
Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or
|
||||
`init` command. The handling of this combination of options has been fixed. A workaround for
|
||||
this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`.
|
||||
|
||||
https://github.com/restic/restic/issues/3293
|
||||
https://github.com/restic/restic/pull/3294
|
||||
|
||||
* Enhancement #3312: Add auto-completion support for fish
|
||||
|
||||
The `generate` command now supports fish auto completion.
|
||||
|
||||
https://github.com/restic/restic/pull/3312
|
||||
|
||||
* Enhancement #3336: SFTP backend now checks for disk space
|
||||
|
||||
Backing up over SFTP previously spewed multiple generic "failure" messages when the remote
|
||||
disk was full. It now checks for disk space before writing a file and fails immediately with a "no
|
||||
space left on device" message.
|
||||
|
||||
https://github.com/restic/restic/issues/3336
|
||||
https://github.com/restic/restic/pull/3345
|
||||
|
||||
* Enhancement #3377: Add release binaries for Apple Silicon
|
||||
|
||||
We've added release binaries for macOS on Apple Silicon (M1).
|
||||
|
||||
https://github.com/restic/restic/issues/3377
|
||||
https://github.com/restic/restic/pull/3394
|
||||
|
||||
* Enhancement #3414: Add `--keep-within-hourly` option to restic forget
|
||||
|
||||
The `forget` command allowed keeping a given number of hourly backups or to keep all backups
|
||||
within a given interval, but it was not possible to specify keeping hourly backups within a
|
||||
given interval.
|
||||
|
||||
The new `--keep-within-hourly` option now offers this functionality. Similar options for
|
||||
daily/weekly/monthly/yearly are also implemented, the new options are:
|
||||
|
||||
--keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly
|
||||
<1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h>
|
||||
|
||||
https://github.com/restic/restic/issues/3414
|
||||
https://github.com/restic/restic/pull/3416
|
||||
https://forum.restic.net/t/forget-policy/4014/11
|
||||
|
||||
* Enhancement #3456: Support filtering and specifying untagged snapshots
|
||||
|
||||
It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
|
||||
options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
|
||||
snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
|
||||
operate on untagged snapshots.
|
||||
|
||||
https://github.com/restic/restic/issues/3456
|
||||
https://github.com/restic/restic/pull/3457
|
||||
|
||||
* Enhancement #3167: Allow specifying limit of `snapshots` list
|
||||
|
||||
The `--last` option allowed limiting the output of the `snapshots` command to the latest
|
||||
snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
|
||||
snapshots.
|
||||
|
||||
This change deprecates the option `--last` in favour of `--latest 1`.
|
||||
|
||||
https://github.com/restic/restic/pull/3167
|
||||
|
||||
* Enhancement #3426: Optimize read performance of mount command
|
||||
|
||||
Reading large files in a mounted repository may be up to five times faster. This improvement
|
||||
primarily applies to repositories stored at a backend that can be accessed with low latency,
|
||||
like e.g. the local backend.
|
||||
|
||||
https://github.com/restic/restic/pull/3426
|
||||
|
||||
* Enhancement #3427: `find --pack` fallback to index if data file is missing
|
||||
|
||||
When investigating a repository with missing data files, it might be useful to determine
|
||||
affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id`
|
||||
returned no data as it required accessing the data file. Now, if the necessary data is still
|
||||
available in the repository index, it gets retrieved from there.
|
||||
|
||||
The command now also supports looking up multiple pack files in a single `find` run.
|
||||
|
||||
https://github.com/restic/restic/pull/3427
|
||||
https://forum.restic.net/t/missing-packs-not-found/2600
|
||||
|
||||
|
||||
Changelog for restic 0.12.0 (2021-02-14)
|
||||
=======================================
|
||||
|
||||
The following sections list the changes in restic 0.12.0 relevant to
|
||||
restic users. The changes are ordered by importance.
|
||||
|
||||
Summary
|
||||
-------
|
||||
|
||||
* Fix #1681: Make `mount` not create missing mount point directory
|
||||
* Fix #1800: Ignore `no data available` filesystem error during backup
|
||||
* Fix #2563: Report the correct owner of directories in FUSE mounts
|
||||
* Fix #2688: Make `backup` and `tag` commands separate tags by comma
|
||||
* Fix #2739: Make the `cat` command respect the `--no-lock` option
|
||||
* Fix #3087: The `--use-fs-snapshot` option now works on windows/386
|
||||
* Fix #3100: Do not require gs bucket permissions when running `init`
|
||||
* Fix #3111: Correctly detect output redirection for `backup` command on Windows
|
||||
* Fix #3151: Don't create invalid snapshots when `backup` is interrupted
|
||||
* Fix #3166: Improve error handling in the `restore` command
|
||||
* Fix #3232: Correct statistics for overlapping targets
|
||||
* Fix #3014: Fix sporadic stream reset between rclone and restic
|
||||
* Fix #3152: Do not hang until foregrounded when completed in background
|
||||
* Fix #3249: Improve error handling in `gs` backend
|
||||
* Chg #3095: Deleting files on Google Drive now moves them to the trash
|
||||
* Enh #2186: Allow specifying percentage in `check --read-data-subset`
|
||||
* Enh #2453: Report permanent/fatal backend errors earlier
|
||||
* Enh #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
|
||||
* Enh #2706: Configurable progress reports for non-interactive terminals
|
||||
* Enh #2944: Add `backup` options `--files-from-{verbatim,raw}`
|
||||
* Enh #3083: Allow usage of deprecated S3 `ListObjects` API
|
||||
* Enh #3147: Support additional environment variables for Swift authentication
|
||||
* Enh #3191: Add release binaries for MIPS architectures
|
||||
* Enh #909: Back up mountpoints as empty directories
|
||||
* Enh #3250: Add several more error checks
|
||||
* Enh #2718: Improve `prune` performance and make it more customizable
|
||||
* Enh #2495: Add option to let `backup` trust mtime without checking ctime
|
||||
* Enh #2941: Speed up the repacking step of the `prune` command
|
||||
* Enh #3006: Speed up the `rebuild-index` command
|
||||
* Enh #3048: Add more checks for index and pack files in the `check` command
|
||||
* Enh #2433: Make the `dump` command support `zip` format
|
||||
* Enh #3099: Reduce memory usage of `check` command
|
||||
* Enh #3106: Parallelize scan of snapshot content in `copy` and `prune`
|
||||
* Enh #3130: Parallelize reading of locks and snapshots
|
||||
* Enh #3254: Enable HTTP/2 for backend connections
|
||||
|
||||
Details
|
||||
-------
|
||||
|
||||
* Bugfix #1681: Make `mount` not create missing mount point directory
|
||||
|
||||
When specifying a non-existent directory as mount point for the `mount` command, restic used
|
||||
to create the specified directory automatically.
|
||||
|
||||
This has now changed such that restic instead gives an error when the specified directory for
|
||||
the mount point does not exist.
|
||||
|
||||
https://github.com/restic/restic/issues/1681
|
||||
https://github.com/restic/restic/pull/3008
|
||||
|
||||
* Bugfix #1800: Ignore `no data available` filesystem error during backup
|
||||
|
||||
Restic was unable to backup files on some filesystems, for example certain configurations of
|
||||
CIFS on Linux which return a `no data available` error when reading extended attributes. These
|
||||
errors are now ignored.
|
||||
|
||||
https://github.com/restic/restic/issues/1800
|
||||
https://github.com/restic/restic/pull/3034
|
||||
|
||||
* Bugfix #2563: Report the correct owner of directories in FUSE mounts
|
||||
|
||||
Restic 0.10.0 changed the FUSE mount to always report the current user as the owner of
|
||||
directories within the FUSE mount, which is incorrect.
|
||||
|
||||
This is now changed back to reporting the correct owner of a directory.
|
||||
|
||||
https://github.com/restic/restic/issues/2563
|
||||
https://github.com/restic/restic/pull/3141
|
||||
|
||||
* Bugfix #2688: Make `backup` and `tag` commands separate tags by comma
|
||||
|
||||
Running `restic backup --tag foo,bar` previously created snapshots with one single tag
|
||||
containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`).
|
||||
|
||||
Similarly, the `tag` command's `--set`, `--add` and `--remove` options would treat
|
||||
`foo,bar` as one tag instead of two tags. This was inconsistent with other commands and often
|
||||
unexpected when one intended `foo,bar` to mean two tags.
|
||||
|
||||
To be consistent in all commands, restic now interprets `foo,bar` to mean two separate tags
|
||||
(`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, including in the `backup` and
|
||||
`tag` commands.
|
||||
|
||||
NOTE: This change might result in unexpected behavior in cases where you use the `forget`
|
||||
command and filter on tags like `foo,bar`. Snapshots previously backed up with `--tag
|
||||
foo,bar` will still not match that filter, but snapshots saved from now on will match that
|
||||
filter.
|
||||
|
||||
To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can first generate a
|
||||
list of the relevant snapshots using a command like:
|
||||
|
||||
Restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id'
|
||||
|
||||
And then use `restic tag --set foo --set bar snapshotID [...]` to set the new tags. Please adjust
|
||||
the commands to include real tag names and any additional tags, as well as the list of snapshots
|
||||
to process.
|
||||
|
||||
https://github.com/restic/restic/issues/2688
|
||||
https://github.com/restic/restic/pull/2690
|
||||
https://github.com/restic/restic/pull/3197
|
||||
|
||||
* Bugfix #2739: Make the `cat` command respect the `--no-lock` option
|
||||
|
||||
The `cat` command would not respect the `--no-lock` flag. This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/2739
|
||||
|
||||
* Bugfix #3087: The `--use-fs-snapshot` option now works on windows/386
|
||||
|
||||
Restic failed to create VSS snapshots on windows/386 with the following error:
|
||||
|
||||
GetSnapshotProperties() failed: E_INVALIDARG (0x80070057)
|
||||
|
||||
This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/3087
|
||||
https://github.com/restic/restic/pull/3090
|
||||
|
||||
* Bugfix #3100: Do not require gs bucket permissions when running `init`
|
||||
|
||||
Restic used to require bucket level permissions for the `gs` backend in order to initialize a
|
||||
restic repository.
|
||||
|
||||
It now allows a `gs` service account to initialize a repository if the bucket does exist and the
|
||||
service account has permissions to write/read to that bucket.
|
||||
|
||||
https://github.com/restic/restic/issues/3100
|
||||
|
||||
* Bugfix #3111: Correctly detect output redirection for `backup` command on Windows
|
||||
|
||||
On Windows, since restic 0.10.0 the `backup` command did not properly detect when the output
|
||||
was redirected to a file. This caused restic to output terminal control characters. This has
|
||||
been fixed by correcting the terminal detection.
|
||||
|
||||
https://github.com/restic/restic/issues/3111
|
||||
https://github.com/restic/restic/pull/3150
|
||||
|
||||
* Bugfix #3151: Don't create invalid snapshots when `backup` is interrupted
|
||||
|
||||
When canceling a backup run at a certain moment it was possible that restic created a snapshot
|
||||
with an invalid "null" tree. This caused `check` and other operations to fail. The `backup`
|
||||
command now properly handles interruptions and never saves a snapshot when interrupted.
|
||||
|
||||
https://github.com/restic/restic/issues/3151
|
||||
https://github.com/restic/restic/pull/3164
|
||||
|
||||
* Bugfix #3166: Improve error handling in the `restore` command
|
||||
|
||||
The `restore` command used to not print errors while downloading file contents from the
|
||||
repository. It also incorrectly exited with a zero error code even when there were errors
|
||||
during the restore process. This has all been fixed and `restore` now returns with a non-zero
|
||||
exit code when there's an error.
|
||||
|
||||
https://github.com/restic/restic/issues/3166
|
||||
https://github.com/restic/restic/pull/3207
|
||||
|
||||
* Bugfix #3232: Correct statistics for overlapping targets
|
||||
|
||||
A user reported that restic's statistics and progress information during backup was not
|
||||
correctly calculated when the backup targets (files/dirs to save) overlap. For example,
|
||||
consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic
|
||||
backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the
|
||||
completeness percentage as well as the number of files was wrong. This is now corrected.
|
||||
|
||||
https://github.com/restic/restic/issues/3232
|
||||
https://github.com/restic/restic/pull/3243
|
||||
|
||||
* Bugfix #3014: Fix sporadic stream reset between rclone and restic
|
||||
|
||||
Sometimes when using restic with the `rclone` backend, an error message similar to the
|
||||
following would be printed:
|
||||
|
||||
Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
|
||||
|
||||
It was found that this was caused by restic closing the connection to rclone to soon when
|
||||
downloading data. A workaround has been added which waits for the end of the download before
|
||||
closing the connection.
|
||||
|
||||
https://github.com/rclone/rclone/issues/2598
|
||||
https://github.com/restic/restic/pull/3014
|
||||
|
||||
* Bugfix #3152: Do not hang until foregrounded when completed in background
|
||||
|
||||
On Linux, when running in the background restic failed to stop the terminal output of the
|
||||
`backup` command after it had completed. This caused restic to hang until moved to the
|
||||
foreground. This has now been fixed.
|
||||
|
||||
https://github.com/restic/restic/pull/3152
|
||||
https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334
|
||||
|
||||
* Bugfix #3249: Improve error handling in `gs` backend
|
||||
|
||||
The `gs` backend did not notice when the last step of completing a file upload failed. Under rare
|
||||
circumstances, this could cause missing files in the backup repository. This has now been
|
||||
fixed.
|
||||
|
||||
https://github.com/restic/restic/pull/3249
|
||||
|
||||
* Change #3095: Deleting files on Google Drive now moves them to the trash
|
||||
|
||||
When deleting files on Google Drive via the `rclone` backend, restic used to bypass the trash
|
||||
folder required that one used the `-o rclone.args` option to enable usage of the trash folder.
|
||||
This ensured that deleted files in Google Drive were not kept indefinitely in the trash folder.
|
||||
However, since Google Drive's trash retention policy changed to deleting trashed files after
|
||||
30 days, this is no longer needed.
|
||||
|
||||
Restic now leaves it up to rclone and its configuration to use or not use the trash folder when
|
||||
deleting files. The default is to use the trash folder, as of rclone 1.53.2. To re-enable the
|
||||
restic 0.11 behavior, set the `RCLONE_DRIVE_USE_TRASH` environment variable or change the
|
||||
rclone configuration. See the rclone documentation for more details.
|
||||
|
||||
https://github.com/restic/restic/issues/3095
|
||||
https://github.com/restic/restic/pull/3102
|
||||
|
||||
* Enhancement #2186: Allow specifying percentage in `check --read-data-subset`
|
||||
|
||||
We've enhanced the `check` command's `--read-data-subset` option to also accept a
|
||||
percentage (e.g. `2.5%` or `10%`). This will check the given percentage of pack files (which
|
||||
are randomly selected on each run).
|
||||
|
||||
https://github.com/restic/restic/issues/2186
|
||||
https://github.com/restic/restic/pull/3038
|
||||
|
||||
* Enhancement #2453: Report permanent/fatal backend errors earlier
|
||||
|
||||
When encountering errors in reading from or writing to storage backends, restic retries the
|
||||
failing operation up to nine times (for a total of ten attempts). It used to retry all backend
|
||||
operations, but now detects some permanent error conditions so that it can report fatal errors
|
||||
earlier.
|
||||
|
||||
Permanent failures include local disks being full, SSH connections dropping and permission
|
||||
errors.
|
||||
|
||||
https://github.com/restic/restic/issues/2453
|
||||
https://github.com/restic/restic/issues/3180
|
||||
https://github.com/restic/restic/pull/3170
|
||||
https://github.com/restic/restic/pull/3181
|
||||
|
||||
* Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
|
||||
|
||||
A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the
|
||||
`s3` backend. The option can be set to one of the following values:
|
||||
|
||||
- `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - Use path style
|
||||
bucket access
|
||||
|
||||
To make the `s3` backend work with Alibaba/Aliyun OSS you must set `s3.bucket-lookup` to `dns`
|
||||
and set the `s3.region` parameter. For example:
|
||||
|
||||
Restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r
|
||||
s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
|
||||
|
||||
Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up and it seems that
|
||||
Alibaba doesn't support that properly.
|
||||
|
||||
https://github.com/restic/restic/issues/2528
|
||||
https://github.com/restic/restic/pull/2535
|
||||
|
||||
* Enhancement #2706: Configurable progress reports for non-interactive terminals
|
||||
|
||||
The `backup`, `check` and `prune` commands never printed any progress reports on
|
||||
non-interactive terminals. This behavior is now configurable using the
|
||||
`RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for an update
|
||||
every second, or `0.01666` for an update every minute.
|
||||
|
||||
The `backup` command now also prints the current progress when restic receives a `SIGUSR1`
|
||||
signal.
|
||||
|
||||
Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` signal
|
||||
prints a status report even when `--quiet` was specified.
|
||||
|
||||
https://github.com/restic/restic/issues/2706
|
||||
https://github.com/restic/restic/issues/3194
|
||||
https://github.com/restic/restic/pull/3199
|
||||
|
||||
* Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}`
|
||||
|
||||
The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of
|
||||
files to back up from a file. Unlike the existing `--files-from` option, these options do not
|
||||
interpret the listed filenames as glob patterns; instead, whitespace in filenames is
|
||||
preserved as-is and no pattern expansion is done. Please see the documentation for specifics.
|
||||
|
||||
These new options are highly recommended over `--files-from`, when using a script to generate
|
||||
the list of files to back up.
|
||||
|
||||
https://github.com/restic/restic/issues/2944
|
||||
https://github.com/restic/restic/issues/3013
|
||||
|
||||
* Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API
|
||||
|
||||
Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2`
|
||||
implementation which causes problems for restic when using their API endpoints. When a broken
|
||||
server implementation is used, restic prints errors similar to the following:
|
||||
|
||||
List() returned error: Truncated response should have continuation token set
|
||||
|
||||
As a temporary workaround, restic now allows using the older `ListObjects` endpoint by
|
||||
setting the `s3.list-objects-v1` extended option, for instance:
|
||||
|
||||
Restic -o s3.list-objects-v1=true snapshots
|
||||
|
||||
Please note that this option may be removed in future versions of restic.
|
||||
|
||||
https://github.com/restic/restic/issues/3083
|
||||
https://github.com/restic/restic/pull/3085
|
||||
|
||||
* Enhancement #3147: Support additional environment variables for Swift authentication
|
||||
|
||||
The `swift` backend now supports the following additional environment variables for passing
|
||||
authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`,
|
||||
`OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
|
||||
|
||||
Depending on the `openrc` configuration file these might be required when the user and project
|
||||
domains differ from one another.
|
||||
|
||||
https://github.com/restic/restic/issues/3147
|
||||
https://github.com/restic/restic/pull/3158
|
||||
|
||||
* Enhancement #3191: Add release binaries for MIPS architectures
|
||||
|
||||
We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`,
|
||||
`mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems.
|
||||
|
||||
https://github.com/restic/restic/issues/3191
|
||||
https://github.com/restic/restic/pull/3208
|
||||
|
||||
* Enhancement #909: Back up mountpoints as empty directories
|
||||
|
||||
When the `--one-file-system` option is specified to `restic backup`, it ignores all file
|
||||
systems mounted below one of the target directories. This means that when a snapshot is
|
||||
restored, users needed to manually recreate the mountpoint directories.
|
||||
|
||||
Restic now backs up mountpoints as empty directories and therefore implements the same
|
||||
approach as `tar`.
|
||||
|
||||
https://github.com/restic/restic/issues/909
|
||||
https://github.com/restic/restic/pull/3119
|
||||
|
||||
* Enhancement #3250: Add several more error checks
|
||||
|
||||
We've added a lot more error checks in places where errors were previously ignored (as hinted by
|
||||
the static analysis program `errcheck` via `golangci-lint`).
|
||||
|
||||
https://github.com/restic/restic/pull/3250
|
||||
|
||||
* Enhancement #2718: Improve `prune` performance and make it more customizable
|
||||
|
||||
The `prune` command is now much faster. This is especially the case for remote repositories or
|
||||
repositories with not much data to remove. Also the memory usage of the `prune` command is now
|
||||
reduced.
|
||||
|
||||
Restic used to rebuild the index from scratch after pruning. This could lead to missing packs in
|
||||
the index in some cases for eventually consistent backends such as e.g. AWS S3. This behavior is
|
||||
now changed and the index rebuilding uses the information already known by `prune`.
|
||||
|
||||
By default, the `prune` command no longer removes all unused data. This behavior can be
|
||||
fine-tuned by new options, like the acceptable amount of unused space or the maximum size of
|
||||
data to reorganize. For more details, please see
|
||||
https://restic.readthedocs.io/en/stable/060_forget.html .
|
||||
|
||||
Moreover, `prune` now accepts the `--dry-run` option and also running `forget --dry-run
|
||||
--prune` will show what `prune` would do.
|
||||
|
||||
This enhancement also fixes several open issues, e.g.: -
|
||||
https://github.com/restic/restic/issues/1140 -
|
||||
https://github.com/restic/restic/issues/1599 -
|
||||
https://github.com/restic/restic/issues/1985 -
|
||||
https://github.com/restic/restic/issues/2112 -
|
||||
https://github.com/restic/restic/issues/2227 -
|
||||
https://github.com/restic/restic/issues/2305
|
||||
|
||||
https://github.com/restic/restic/pull/2718
|
||||
https://github.com/restic/restic/pull/2842
|
||||
|
||||
* Enhancement #2495: Add option to let `backup` trust mtime without checking ctime
|
||||
|
||||
The `backup` command used to require that both `ctime` and `mtime` of a file matched with a
|
||||
previously backed up version to determine that the file was unchanged. In other words, if
|
||||
either `ctime` or `mtime` of the file had changed, it would be considered changed and restic
|
||||
would read the file's content again to back up the relevant (changed) parts of it.
|
||||
|
||||
The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes
|
||||
for a file does not cause restic to read the file's contents again.
|
||||
|
||||
The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more
|
||||
reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned
|
||||
out to often be expensive because it made restic read file contents even if only the metadata
|
||||
(owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user
|
||||
restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned
|
||||
off this behavior, but also removed a different check.
|
||||
|
||||
Please note that changes in files' metadata are still recorded, regardless of the command line
|
||||
options provided to the backup command.
|
||||
|
||||
https://github.com/restic/restic/issues/2495
|
||||
https://github.com/restic/restic/issues/2558
|
||||
https://github.com/restic/restic/issues/2819
|
||||
https://github.com/restic/restic/pull/2823
|
||||
|
||||
* Enhancement #2941: Speed up the repacking step of the `prune` command
|
||||
|
||||
The repack step of the `prune` command, which moves still used file parts into new pack files
|
||||
such that the old ones can be garbage collected later on, now processes multiple pack files in
|
||||
parallel. This is especially beneficial for high latency backends or when using a fast network
|
||||
connection.
|
||||
|
||||
https://github.com/restic/restic/pull/2941
|
||||
|
||||
* Enhancement #3006: Speed up the `rebuild-index` command
|
||||
|
||||
We've optimized the `rebuild-index` command. Now, existing index entries are used to
|
||||
minimize the number of pack files that must be read. This speeds up the index rebuild a lot.
|
||||
|
||||
Additionally, the option `--read-all-packs` has been added, implementing the previous
|
||||
behavior.
|
||||
|
||||
https://github.com/restic/restic/pull/3006
|
||||
https://github.com/restic/restic/issue/2547
|
||||
|
||||
* Enhancement #3048: Add more checks for index and pack files in the `check` command
|
||||
|
||||
The `check` command run with the `--read-data` or `--read-data-subset` options used to only
|
||||
verify only the pack file content - it did not check if the blobs within the pack are correctly
|
||||
contained in the index.
|
||||
|
||||
A check for the latter is now in place, which can print the following error:
|
||||
|
||||
Blob ID is not contained in index or position is incorrect
|
||||
|
||||
Another test is also added, which compares pack file sizes computed from the index and the pack
|
||||
header with the actual file size. This test is able to detect truncated pack files.
|
||||
|
||||
If the index is not correct, it can be rebuilt by using the `rebuild-index` command.
|
||||
|
||||
Having added these tests, `restic check` is now able to detect non-existing blobs which are
|
||||
wrongly referenced in the index. This situation could have lead to missing data.
|
||||
|
||||
https://github.com/restic/restic/pull/3048
|
||||
https://github.com/restic/restic/pull/3082
|
||||
|
||||
* Enhancement #2433: Make the `dump` command support `zip` format
|
||||
|
||||
Previously, restic could dump the contents of a whole folder structure only in the `tar`
|
||||
format. The `dump` command now has a new flag to change output format to `zip`. Just pass
|
||||
`--archive zip` as an option to `restic dump`.
|
||||
|
||||
https://github.com/restic/restic/pull/2433
|
||||
https://github.com/restic/restic/pull/3081
|
||||
|
||||
* Enhancement #3099: Reduce memory usage of `check` command
|
||||
|
||||
The `check` command now requires less memory if it is run without the `--check-unused` option.
|
||||
|
||||
https://github.com/restic/restic/pull/3099
|
||||
|
||||
* Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune`
|
||||
|
||||
The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find
|
||||
used data. This snapshot traversal is now parallized which can speed up this step several
|
||||
times.
|
||||
|
||||
In addition the `check` command now reports how many snapshots have already been processed.
|
||||
|
||||
https://github.com/restic/restic/pull/3106
|
||||
|
||||
* Enhancement #3130: Parallelize reading of locks and snapshots
|
||||
|
||||
Restic used to read snapshots sequentially. For repositories containing many snapshots this
|
||||
slowed down commands which have to read all snapshots.
|
||||
|
||||
Now the reading of snapshots is parallelized. This speeds up for example `prune`, `backup` and
|
||||
other commands that search for snapshots with certain properties or which have to find the
|
||||
`latest` snapshot.
|
||||
|
||||
The speed up also applies to locks stored in the backup repository.
|
||||
|
||||
https://github.com/restic/restic/pull/3130
|
||||
https://github.com/restic/restic/pull/3174
|
||||
|
||||
* Enhancement #3254: Enable HTTP/2 for backend connections
|
||||
|
||||
Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on
|
||||
what the server supports. But for compatibility this mechanism is disabled if DialContext is
|
||||
used (which is the case for restic). This change allows restic's HTTP client to negotiate
|
||||
HTTP/2 if supported by the server.
|
||||
|
||||
https://github.com/restic/restic/pull/3254
|
||||
|
||||
|
||||
Changelog for restic 0.11.0 (2020-11-05)
|
||||
=======================================
|
||||
|
||||
|
||||
@@ -13,10 +13,11 @@ bug fixes are most welcome. However even "minor" details as fixing spelling
|
||||
errors, improving documentation or pointing out usability issues are a great
|
||||
help also.
|
||||
|
||||
|
||||
The restic project uses the GitHub infrastructure (see the
|
||||
[project page](https://github.com/restic/restic)) for all related discussions
|
||||
as well as the [forum](https://forum.restic.net/) and the `#restic` channel
|
||||
on [irc.libera.chat](https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:6697/#restic).
|
||||
on [irc.freenode.net](https://kiwiirc.com/nextclient/irc.freenode.net/restic).
|
||||
|
||||
If you want to find an area that currently needs improving have a look at the
|
||||
open issues listed at the
|
||||
@@ -140,14 +141,6 @@ Installing the script `fmt-check` from https://github.com/edsrzf/gofmt-git-hook
|
||||
locally as a pre-commit hook checks formatting before committing automatically,
|
||||
just copy this script to `.git/hooks/pre-commit`.
|
||||
|
||||
The project is using the program
|
||||
[`golangci-lint`](https://github.com/golangci/golangci-lint) to run a list of
|
||||
linters and checkers. It will be run on the code when you submit a PR. In order
|
||||
to check your code beforehand, you can run `golangci-lint run` manually.
|
||||
Eventually, we will enable `golangci-lint` for the whole code base. For now,
|
||||
you can ignore warnings printed for lines you did not modify, those will be
|
||||
ignored by the CI.
|
||||
|
||||
For each pull request, several different systems run the integration tests on
|
||||
Linux, macOS and Windows. We won't merge any code that does not pass all tests
|
||||
for all systems, so when a tests fails, try to find out what's wrong and fix
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
[](https://restic.readthedocs.io/en/latest/?badge=latest)
|
||||
[](https://github.com/restic/restic/actions?query=workflow%3Atest)
|
||||
[](https://travis-ci.com/restic/restic)
|
||||
[](https://ci.appveyor.com/project/fd0/restic/branch/master)
|
||||
[](https://goreportcard.com/report/github.com/restic/restic)
|
||||
|
||||
# Introduction
|
||||
|
||||
32
appveyor.yml
Normal file
32
appveyor.yml
Normal file
@@ -0,0 +1,32 @@
|
||||
clone_folder: c:\restic
|
||||
|
||||
environment:
|
||||
GOPATH: c:\gopath
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
||||
cache:
|
||||
- '%LocalAppData%\go-build'
|
||||
|
||||
init:
|
||||
- ps: >-
|
||||
$app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
|
||||
|
||||
if ($app) {
|
||||
$app.Uninstall()
|
||||
}
|
||||
|
||||
install:
|
||||
- rmdir c:\go /s /q
|
||||
- appveyor DownloadFile https://dl.google.com/go/go1.15.2.windows-amd64.msi
|
||||
- msiexec /i go1.15.2.windows-amd64.msi /q
|
||||
- go version
|
||||
- go env
|
||||
- appveyor DownloadFile https://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
|
||||
- 7z x tar.zip bin/tar.exe
|
||||
- set PATH=bin/;%PATH%
|
||||
|
||||
build_script:
|
||||
- go run run_integration_tests.go
|
||||
@@ -1,10 +0,0 @@
|
||||
Bugfix: Make `mount` not create missing mount point directory
|
||||
|
||||
When specifying a non-existent directory as mount point for the `mount`
|
||||
command, restic used to create the specified directory automatically.
|
||||
|
||||
This has now changed such that restic instead gives an error when the
|
||||
specified directory for the mount point does not exist.
|
||||
|
||||
https://github.com/restic/restic/issues/1681
|
||||
https://github.com/restic/restic/pull/3008
|
||||
@@ -1,8 +0,0 @@
|
||||
Bugfix: Ignore `no data available` filesystem error during backup
|
||||
|
||||
Restic was unable to backup files on some filesystems, for example certain
|
||||
configurations of CIFS on Linux which return a `no data available` error
|
||||
when reading extended attributes. These errors are now ignored.
|
||||
|
||||
https://github.com/restic/restic/issues/1800
|
||||
https://github.com/restic/restic/pull/3034
|
||||
@@ -1,8 +0,0 @@
|
||||
Enhancement: Allow specifying percentage in `check --read-data-subset`
|
||||
|
||||
We've enhanced the `check` command's `--read-data-subset` option to also accept
|
||||
a percentage (e.g. `2.5%` or `10%`). This will check the given percentage of
|
||||
pack files (which are randomly selected on each run).
|
||||
|
||||
https://github.com/restic/restic/issues/2186
|
||||
https://github.com/restic/restic/pull/3038
|
||||
@@ -1,14 +0,0 @@
|
||||
Enhancement: Report permanent/fatal backend errors earlier
|
||||
|
||||
When encountering errors in reading from or writing to storage backends,
|
||||
restic retries the failing operation up to nine times (for a total of ten
|
||||
attempts). It used to retry all backend operations, but now detects some
|
||||
permanent error conditions so that it can report fatal errors earlier.
|
||||
|
||||
Permanent failures include local disks being full, SSH connections
|
||||
dropping and permission errors.
|
||||
|
||||
https://github.com/restic/restic/issues/2453
|
||||
https://github.com/restic/restic/pull/3170
|
||||
https://github.com/restic/restic/issues/3180
|
||||
https://github.com/restic/restic/pull/3181
|
||||
@@ -1,21 +0,0 @@
|
||||
Enhancement: Add Alibaba/Aliyun OSS support in the `s3` backend
|
||||
|
||||
A new extended option `s3.bucket-lookup` has been added to support
|
||||
Alibaba/Aliyun OSS in the `s3` backend. The option can be set to one
|
||||
of the following values:
|
||||
|
||||
- `auto` - Existing behaviour
|
||||
- `dns` - Use DNS style bucket access
|
||||
- `path` - Use path style bucket access
|
||||
|
||||
To make the `s3` backend work with Alibaba/Aliyun OSS you must set
|
||||
`s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For
|
||||
example:
|
||||
|
||||
restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
|
||||
|
||||
Note that `s3.region` must be set, otherwise the MinIO SDK tries to
|
||||
look it up and it seems that Alibaba doesn't support that properly.
|
||||
|
||||
https://github.com/restic/restic/issues/2528
|
||||
https://github.com/restic/restic/pull/2535
|
||||
@@ -1,9 +0,0 @@
|
||||
Bugfix: Report the correct owner of directories in FUSE mounts
|
||||
|
||||
Restic 0.10.0 changed the FUSE mount to always report the current user
|
||||
as the owner of directories within the FUSE mount, which is incorrect.
|
||||
|
||||
This is now changed back to reporting the correct owner of a directory.
|
||||
|
||||
https://github.com/restic/restic/issues/2563
|
||||
https://github.com/restic/restic/pull/3141
|
||||
@@ -1,31 +0,0 @@
|
||||
Bugfix: Make `backup` and `tag` commands separate tags by comma
|
||||
|
||||
Running `restic backup --tag foo,bar` previously created snapshots with one
|
||||
single tag containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`).
|
||||
|
||||
Similarly, the `tag` command's `--set`, `--add` and `--remove` options would
|
||||
treat `foo,bar` as one tag instead of two tags. This was inconsistent with
|
||||
other commands and often unexpected when one intended `foo,bar` to mean two
|
||||
tags.
|
||||
|
||||
To be consistent in all commands, restic now interprets `foo,bar` to mean two
|
||||
separate tags (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere,
|
||||
including in the `backup` and `tag` commands.
|
||||
|
||||
NOTE: This change might result in unexpected behavior in cases where you use
|
||||
the `forget` command and filter on tags like `foo,bar`. Snapshots previously
|
||||
backed up with `--tag foo,bar` will still not match that filter, but snapshots
|
||||
saved from now on will match that filter.
|
||||
|
||||
To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can
|
||||
first generate a list of the relevant snapshots using a command like:
|
||||
|
||||
restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id'
|
||||
|
||||
and then use `restic tag --set foo --set bar snapshotID [...]` to set the new
|
||||
tags. Please adjust the commands to include real tag names and any additional
|
||||
tags, as well as the list of snapshots to process.
|
||||
|
||||
https://github.com/restic/restic/issues/2688
|
||||
https://github.com/restic/restic/pull/2690
|
||||
https://github.com/restic/restic/pull/3197
|
||||
@@ -1,17 +0,0 @@
|
||||
Enhancement: Configurable progress reports for non-interactive terminals
|
||||
|
||||
The `backup`, `check` and `prune` commands never printed any progress
|
||||
reports on non-interactive terminals. This behavior is now configurable
|
||||
using the `RESTIC_PROGRESS_FPS` environment variable. Use for example a
|
||||
value of `1` for an update every second, or `0.01666` for an update every
|
||||
minute.
|
||||
|
||||
The `backup` command now also prints the current progress when restic
|
||||
receives a `SIGUSR1` signal.
|
||||
|
||||
Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1`
|
||||
signal prints a status report even when `--quiet` was specified.
|
||||
|
||||
https://github.com/restic/restic/issues/2706
|
||||
https://github.com/restic/restic/issues/3194
|
||||
https://github.com/restic/restic/pull/3199
|
||||
@@ -1,5 +0,0 @@
|
||||
Bugfix: Make the `cat` command respect the `--no-lock` option
|
||||
|
||||
The `cat` command would not respect the `--no-lock` flag. This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/2739
|
||||
@@ -1,13 +0,0 @@
|
||||
Enhancement: Add `backup` options `--files-from-{verbatim,raw}`
|
||||
|
||||
The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a
|
||||
list of files to back up from a file. Unlike the existing `--files-from`
|
||||
option, these options do not interpret the listed filenames as glob patterns;
|
||||
instead, whitespace in filenames is preserved as-is and no pattern expansion is
|
||||
done. Please see the documentation for specifics.
|
||||
|
||||
These new options are highly recommended over `--files-from`, when using a
|
||||
script to generate the list of files to back up.
|
||||
|
||||
https://github.com/restic/restic/issues/2944
|
||||
https://github.com/restic/restic/issues/3013
|
||||
@@ -1,18 +0,0 @@
|
||||
Enhancement: Allow usage of deprecated S3 `ListObjects` API
|
||||
|
||||
Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken
|
||||
`ListObjectsV2` implementation which causes problems for restic when using
|
||||
their API endpoints. When a broken server implementation is used, restic prints
|
||||
errors similar to the following:
|
||||
|
||||
List() returned error: Truncated response should have continuation token set
|
||||
|
||||
As a temporary workaround, restic now allows using the older `ListObjects`
|
||||
endpoint by setting the `s3.list-objects-v1` extended option, for instance:
|
||||
|
||||
restic -o s3.list-objects-v1=true snapshots
|
||||
|
||||
Please note that this option may be removed in future versions of restic.
|
||||
|
||||
https://github.com/restic/restic/issues/3083
|
||||
https://github.com/restic/restic/pull/3085
|
||||
@@ -1,10 +0,0 @@
|
||||
Bugfix: The `--use-fs-snapshot` option now works on windows/386
|
||||
|
||||
Restic failed to create VSS snapshots on windows/386 with the following error:
|
||||
|
||||
GetSnapshotProperties() failed: E_INVALIDARG (0x80070057)
|
||||
|
||||
This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/3087
|
||||
https://github.com/restic/restic/pull/3090
|
||||
@@ -1,17 +0,0 @@
|
||||
Change: Deleting files on Google Drive now moves them to the trash
|
||||
|
||||
When deleting files on Google Drive via the `rclone` backend, restic used to
|
||||
bypass the trash folder required that one used the `-o rclone.args` option to
|
||||
enable usage of the trash folder. This ensured that deleted files in Google
|
||||
Drive were not kept indefinitely in the trash folder. However, since Google
|
||||
Drive's trash retention policy changed to deleting trashed files after 30 days,
|
||||
this is no longer needed.
|
||||
|
||||
Restic now leaves it up to rclone and its configuration to use or not use the
|
||||
trash folder when deleting files. The default is to use the trash folder, as
|
||||
of rclone 1.53.2. To re-enable the restic 0.11 behavior, set the
|
||||
`RCLONE_DRIVE_USE_TRASH` environment variable or change the rclone
|
||||
configuration. See the rclone documentation for more details.
|
||||
|
||||
https://github.com/restic/restic/issues/3095
|
||||
https://github.com/restic/restic/pull/3102
|
||||
@@ -1,10 +0,0 @@
|
||||
Bugfix: Do not require gs bucket permissions when running `init`
|
||||
|
||||
Restic used to require bucket level permissions for the `gs` backend
|
||||
in order to initialize a restic repository.
|
||||
|
||||
It now allows a `gs` service account to initialize a repository if the
|
||||
bucket does exist and the service account has permissions to write/read
|
||||
to that bucket.
|
||||
|
||||
https://github.com/restic/restic/issues/3100
|
||||
@@ -1,9 +0,0 @@
|
||||
Bugfix: Correctly detect output redirection for `backup` command on Windows
|
||||
|
||||
On Windows, since restic 0.10.0 the `backup` command did not properly detect
|
||||
when the output was redirected to a file. This caused restic to output
|
||||
terminal control characters. This has been fixed by correcting the terminal
|
||||
detection.
|
||||
|
||||
https://github.com/restic/restic/issues/3111
|
||||
https://github.com/restic/restic/pull/3150
|
||||
@@ -1,11 +0,0 @@
|
||||
Enhancement: Support additional environment variables for Swift authentication
|
||||
|
||||
The `swift` backend now supports the following additional environment variables
|
||||
for passing authentication details to restic:
|
||||
`OS_USER_ID`, `OS_USER_DOMAIN_ID`, `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
|
||||
|
||||
Depending on the `openrc` configuration file these might be required when the
|
||||
user and project domains differ from one another.
|
||||
|
||||
https://github.com/restic/restic/issues/3147
|
||||
https://github.com/restic/restic/pull/3158
|
||||
@@ -1,9 +0,0 @@
|
||||
Bugfix: Don't create invalid snapshots when `backup` is interrupted
|
||||
|
||||
When canceling a backup run at a certain moment it was possible that
|
||||
restic created a snapshot with an invalid "null" tree. This caused
|
||||
`check` and other operations to fail. The `backup` command now properly
|
||||
handles interruptions and never saves a snapshot when interrupted.
|
||||
|
||||
https://github.com/restic/restic/issues/3151
|
||||
https://github.com/restic/restic/pull/3164
|
||||
@@ -1,9 +0,0 @@
|
||||
Bugfix: Improve error handling in the `restore` command
|
||||
|
||||
The `restore` command used to not print errors while downloading file contents
|
||||
from the repository. It also incorrectly exited with a zero error code even
|
||||
when there were errors during the restore process. This has all been fixed and
|
||||
`restore` now returns with a non-zero exit code when there's an error.
|
||||
|
||||
https://github.com/restic/restic/issues/3166
|
||||
https://github.com/restic/restic/pull/3207
|
||||
@@ -1,8 +0,0 @@
|
||||
Enhancement: Add release binaries for MIPS architectures
|
||||
|
||||
We've added a few new architectures for Linux to the release binaries: `mips`,
|
||||
`mipsle`, `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded
|
||||
systems.
|
||||
|
||||
https://github.com/restic/restic/issues/3191
|
||||
https://github.com/restic/restic/pull/3208
|
||||
@@ -1,11 +0,0 @@
|
||||
Bugfix: Correct statistics for overlapping targets
|
||||
|
||||
A user reported that restic's statistics and progress information during backup
|
||||
was not correctly calculated when the backup targets (files/dirs to save)
|
||||
overlap. For example, consider a directory `foo` which contains (among others)
|
||||
a file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the
|
||||
size of the file `foo/bar` twice, so the completeness percentage as well as the
|
||||
number of files was wrong. This is now corrected.
|
||||
|
||||
https://github.com/restic/restic/issues/3232
|
||||
https://github.com/restic/restic/pull/3243
|
||||
@@ -1,12 +0,0 @@
|
||||
Enhancement: Back up mountpoints as empty directories
|
||||
|
||||
When the `--one-file-system` option is specified to `restic backup`, it
|
||||
ignores all file systems mounted below one of the target directories. This
|
||||
means that when a snapshot is restored, users needed to manually recreate
|
||||
the mountpoint directories.
|
||||
|
||||
Restic now backs up mountpoints as empty directories and therefore implements
|
||||
the same approach as `tar`.
|
||||
|
||||
https://github.com/restic/restic/issues/909
|
||||
https://github.com/restic/restic/pull/3119
|
||||
@@ -1,6 +0,0 @@
|
||||
Enhancement: Add several more error checks
|
||||
|
||||
We've added a lot more error checks in places where errors were previously
|
||||
ignored (as hinted by the static analysis program `errcheck` via `golangci-lint`).
|
||||
|
||||
https://github.com/restic/restic/pull/3250
|
||||
@@ -1,29 +0,0 @@
|
||||
Enhancement: Improve `prune` performance and make it more customizable
|
||||
|
||||
The `prune` command is now much faster. This is especially the case for remote
|
||||
repositories or repositories with not much data to remove. Also the memory
|
||||
usage of the `prune` command is now reduced.
|
||||
|
||||
Restic used to rebuild the index from scratch after pruning. This could lead
|
||||
to missing packs in the index in some cases for eventually consistent backends
|
||||
such as e.g. AWS S3. This behavior is now changed and the index rebuilding
|
||||
uses the information already known by `prune`.
|
||||
|
||||
By default, the `prune` command no longer removes all unused data. This
|
||||
behavior can be fine-tuned by new options, like the acceptable amount of
|
||||
unused space or the maximum size of data to reorganize. For more details,
|
||||
please see https://restic.readthedocs.io/en/stable/060_forget.html .
|
||||
|
||||
Moreover, `prune` now accepts the `--dry-run` option and also running
|
||||
`forget --dry-run --prune` will show what `prune` would do.
|
||||
|
||||
This enhancement also fixes several open issues, e.g.:
|
||||
- https://github.com/restic/restic/issues/1140
|
||||
- https://github.com/restic/restic/issues/1599
|
||||
- https://github.com/restic/restic/issues/1985
|
||||
- https://github.com/restic/restic/issues/2112
|
||||
- https://github.com/restic/restic/issues/2227
|
||||
- https://github.com/restic/restic/issues/2305
|
||||
|
||||
https://github.com/restic/restic/pull/2718
|
||||
https://github.com/restic/restic/pull/2842
|
||||
@@ -1,27 +0,0 @@
|
||||
Enhancement: Add option to let `backup` trust mtime without checking ctime
|
||||
|
||||
The `backup` command used to require that both `ctime` and `mtime` of a file
|
||||
matched with a previously backed up version to determine that the file was
|
||||
unchanged. In other words, if either `ctime` or `mtime` of the file had
|
||||
changed, it would be considered changed and restic would read the file's
|
||||
content again to back up the relevant (changed) parts of it.
|
||||
|
||||
The new option `--ignore-ctime` makes restic look at `mtime` only, such that
|
||||
`ctime` changes for a file does not cause restic to read the file's contents
|
||||
again.
|
||||
|
||||
The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make
|
||||
backups more reliable in the face of programs that reset `mtime` (some Unix
|
||||
archivers do that), but it turned out to often be expensive because it made
|
||||
restic read file contents even if only the metadata (owner, permissions) of
|
||||
a file had changed. The new `--ignore-ctime` option lets the user restore the
|
||||
0.9.5 behavior when needed. The existing `--ignore-inode` option already
|
||||
turned off this behavior, but also removed a different check.
|
||||
|
||||
Please note that changes in files' metadata are still recorded, regardless of
|
||||
the command line options provided to the backup command.
|
||||
|
||||
https://github.com/restic/restic/issues/2495
|
||||
https://github.com/restic/restic/issues/2558
|
||||
https://github.com/restic/restic/issues/2819
|
||||
https://github.com/restic/restic/pull/2823
|
||||
@@ -1,8 +0,0 @@
|
||||
Enhancement: Speed up the repacking step of the `prune` command
|
||||
|
||||
The repack step of the `prune` command, which moves still used file parts into
|
||||
new pack files such that the old ones can be garbage collected later on, now
|
||||
processes multiple pack files in parallel. This is especially beneficial for
|
||||
high latency backends or when using a fast network connection.
|
||||
|
||||
https://github.com/restic/restic/pull/2941
|
||||
@@ -1,11 +0,0 @@
|
||||
Enhancement: Speed up the `rebuild-index` command
|
||||
|
||||
We've optimized the `rebuild-index` command. Now, existing index entries are used
|
||||
to minimize the number of pack files that must be read. This speeds up the index
|
||||
rebuild a lot.
|
||||
|
||||
Additionally, the option `--read-all-packs` has been added, implementing the
|
||||
previous behavior.
|
||||
|
||||
https://github.com/restic/restic/issue/2547
|
||||
https://github.com/restic/restic/pull/3006
|
||||
@@ -1,13 +0,0 @@
|
||||
Bugfix: Fix sporadic stream reset between rclone and restic
|
||||
|
||||
Sometimes when using restic with the `rclone` backend, an error message
|
||||
similar to the following would be printed:
|
||||
|
||||
Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
|
||||
|
||||
It was found that this was caused by restic closing the connection to rclone
|
||||
to soon when downloading data. A workaround has been added which waits for
|
||||
the end of the download before closing the connection.
|
||||
|
||||
https://github.com/restic/restic/pull/3014
|
||||
https://github.com/rclone/rclone/issues/2598
|
||||
@@ -1,23 +0,0 @@
|
||||
Enhancement: Add more checks for index and pack files in the `check` command
|
||||
|
||||
The `check` command run with the `--read-data` or `--read-data-subset` options
|
||||
used to only verify only the pack file content - it did not check if the blobs
|
||||
within the pack are correctly contained in the index.
|
||||
|
||||
A check for the latter is now in place, which can print the following error:
|
||||
|
||||
Blob ID is not contained in index or position is incorrect
|
||||
|
||||
Another test is also added, which compares pack file sizes computed from the
|
||||
index and the pack header with the actual file size. This test is able to
|
||||
detect truncated pack files.
|
||||
|
||||
If the index is not correct, it can be rebuilt by using the `rebuild-index`
|
||||
command.
|
||||
|
||||
Having added these tests, `restic check` is now able to detect non-existing
|
||||
blobs which are wrongly referenced in the index. This situation could have
|
||||
lead to missing data.
|
||||
|
||||
https://github.com/restic/restic/pull/3048
|
||||
https://github.com/restic/restic/pull/3082
|
||||
@@ -1,8 +0,0 @@
|
||||
Enhancement: Make the `dump` command support `zip` format
|
||||
|
||||
Previously, restic could dump the contents of a whole folder structure only
|
||||
in the `tar` format. The `dump` command now has a new flag to change output
|
||||
format to `zip`. Just pass `--archive zip` as an option to `restic dump`.
|
||||
|
||||
https://github.com/restic/restic/pull/2433
|
||||
https://github.com/restic/restic/pull/3081
|
||||
@@ -1,6 +0,0 @@
|
||||
Enhancement: Reduce memory usage of `check` command
|
||||
|
||||
The `check` command now requires less memory if it is run without the
|
||||
`--check-unused` option.
|
||||
|
||||
https://github.com/restic/restic/pull/3099
|
||||
@@ -1,10 +0,0 @@
|
||||
Enhancement: Parallelize scan of snapshot content in `copy` and `prune`
|
||||
|
||||
The `copy` and `prune` commands used to traverse the directories of
|
||||
snapshots one by one to find used data. This snapshot traversal is
|
||||
now parallized which can speed up this step several times.
|
||||
|
||||
In addition the `check` command now reports how many snapshots have
|
||||
already been processed.
|
||||
|
||||
https://github.com/restic/restic/pull/3106
|
||||
@@ -1,13 +0,0 @@
|
||||
Enhancement: Parallelize reading of locks and snapshots
|
||||
|
||||
Restic used to read snapshots sequentially. For repositories containing
|
||||
many snapshots this slowed down commands which have to read all snapshots.
|
||||
|
||||
Now the reading of snapshots is parallelized. This speeds up for example
|
||||
`prune`, `backup` and other commands that search for snapshots with certain
|
||||
properties or which have to find the `latest` snapshot.
|
||||
|
||||
The speed up also applies to locks stored in the backup repository.
|
||||
|
||||
https://github.com/restic/restic/pull/3130
|
||||
https://github.com/restic/restic/pull/3174
|
||||
@@ -1,8 +0,0 @@
|
||||
Bugfix: Do not hang until foregrounded when completed in background
|
||||
|
||||
On Linux, when running in the background restic failed to stop the terminal
|
||||
output of the `backup` command after it had completed. This caused restic to
|
||||
hang until moved to the foreground. This has now been fixed.
|
||||
|
||||
https://github.com/restic/restic/pull/3152
|
||||
https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334
|
||||
@@ -1,7 +0,0 @@
|
||||
Bugfix: Improve error handling in `gs` backend
|
||||
|
||||
The `gs` backend did not notice when the last step of completing a
|
||||
file upload failed. Under rare circumstances, this could cause
|
||||
missing files in the backup repository. This has now been fixed.
|
||||
|
||||
https://github.com/restic/restic/pull/3249
|
||||
@@ -1,8 +0,0 @@
|
||||
Enhancement: Enable HTTP/2 for backend connections
|
||||
|
||||
Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2
|
||||
depending on what the server supports. But for compatibility this mechanism
|
||||
is disabled if DialContext is used (which is the case for restic). This change
|
||||
allows restic's HTTP client to negotiate HTTP/2 if supported by the server.
|
||||
|
||||
https://github.com/restic/restic/pull/3254
|
||||
@@ -1,13 +0,0 @@
|
||||
Bugfix: Improve error handling for rclone and REST backend over HTTP2
|
||||
|
||||
When retrieving data from the rclone / REST backend while also using HTTP2
|
||||
restic did not detect when no data was returned at all. This could cause
|
||||
for example the `check` command to report the following error:
|
||||
|
||||
Pack ID does not match, want [...], got e3b0c442
|
||||
|
||||
This has been fixed by correctly detecting and retrying the incomplete download.
|
||||
|
||||
https://github.com/restic/restic/issues/2742
|
||||
https://github.com/restic/restic/pull/3453
|
||||
https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
|
||||
@@ -1,6 +0,0 @@
|
||||
Enhancement: Add release binaries for s390x architecture on Linux
|
||||
|
||||
We've added release binaries for Linux using the s390x architecture.
|
||||
|
||||
https://github.com/restic/restic/issues/2780
|
||||
https://github.com/restic/restic/pull/3452
|
||||
@@ -1,11 +0,0 @@
|
||||
Bugfix: Fix terminal output redirection for PowerShell
|
||||
|
||||
When redirecting the output of restic using PowerShell on Windows, the
|
||||
output contained terminal escape characters. This has been fixed by
|
||||
properly detecting the terminal type.
|
||||
|
||||
In addition, the mintty terminal now shows progress output for the backup
|
||||
command.
|
||||
|
||||
https://github.com/restic/restic/issues/3111
|
||||
https://github.com/restic/restic/pull/3325
|
||||
@@ -1,9 +0,0 @@
|
||||
Bugfix: Treat an empty password as a fatal error for repository init
|
||||
|
||||
When attempting to initialize a new repository, if an empty password was
|
||||
supplied, the repository would be created but the init command would return
|
||||
an error with a stack trace. Now, if an empty password is provided, it is
|
||||
treated as a fatal error, and no repository is created.
|
||||
|
||||
https://github.com/restic/restic/issues/3214
|
||||
https://github.com/restic/restic/pull/3283
|
||||
@@ -1,8 +0,0 @@
|
||||
Change: Empty files now have size of 0 in `ls --json` output
|
||||
|
||||
The `ls --json` command used to omit the sizes of empty files in its
|
||||
output. It now reports a size of zero explicitly for regular files,
|
||||
while omitting the size field for all other types.
|
||||
|
||||
https://github.com/restic/restic/issues/3247
|
||||
https://github.com/restic/restic/pull/3257
|
||||
@@ -1,11 +0,0 @@
|
||||
Bugfix: `copy` failed to copy snapshots in rare cases
|
||||
|
||||
The `copy` command could in rare cases fail with the error message `SaveTree(...)
|
||||
returned unexpected id ...`. This has been fixed.
|
||||
|
||||
On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8
|
||||
target paths. Note that, due to limitations in the repository format, these are
|
||||
not stored properly and should be avoided if possible.
|
||||
|
||||
https://github.com/restic/restic/issues/3267
|
||||
https://github.com/restic/restic/pull/3310
|
||||
@@ -1,11 +0,0 @@
|
||||
Bugfix: `backup --quiet` no longer prints status information
|
||||
|
||||
A regression in the latest restic version caused the output of `backup --quiet`
|
||||
to contain large amounts of backup progress information when run using an
|
||||
interactive terminal. This is fixed now.
|
||||
|
||||
A workaround for this bug is to run restic as follows:
|
||||
`restic backup --quiet [..] | cat -`.
|
||||
|
||||
https://github.com/restic/restic/issues/3184
|
||||
https://github.com/restic/restic/pull/3186
|
||||
@@ -1,14 +0,0 @@
|
||||
Enhancement: Add `--repository-file2` option to `init` and `copy` command
|
||||
|
||||
The `init` and `copy` command can now be used with the `--repository-file2`
|
||||
option or the `$RESTIC_REPOSITORY_FILE2` environment variable.
|
||||
These to options are in addition to the `--repo2` flag and allow you to read
|
||||
the destination repository from a file.
|
||||
|
||||
Using both `--repository-file` and `--repo2` options resulted in an error for
|
||||
the `copy` or `init` command. The handling of this combination of options has
|
||||
been fixed. A workaround for this issue is to only use `--repo` or `-r` and
|
||||
`--repo2` for `init` or `copy`.
|
||||
|
||||
https://github.com/restic/restic/issues/3293
|
||||
https://github.com/restic/restic/pull/3294
|
||||
@@ -1,7 +0,0 @@
|
||||
Bugfix: Fix crash of `check --read-data-subset=x%` run for an empty repository
|
||||
|
||||
The command `restic check --read-data-subset=x%` crashed when run for an empty
|
||||
repository. This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/3296
|
||||
https://github.com/restic/restic/pull/3309
|
||||
@@ -1,8 +0,0 @@
|
||||
Bugfix: Fix `fdopendir: not a directory` error for local backend
|
||||
|
||||
The `check`, `list packs`, `prune` and `rebuild-index` commands failed
|
||||
for the local backend when the `data` folder in the repository contained
|
||||
files. This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/3302
|
||||
https://github.com/restic/restic/pull/3308
|
||||
@@ -1,5 +0,0 @@
|
||||
Enhancement: Add auto-completion support for fish
|
||||
|
||||
The `generate` command now supports fish auto completion.
|
||||
|
||||
https://github.com/restic/restic/pull/3312
|
||||
@@ -1,8 +0,0 @@
|
||||
Bugfix: Print `created new cache` message only on a terminal
|
||||
|
||||
The message `created new cache` was printed even when the output wasn't a
|
||||
terminal. That broke piping `restic dump` output to tar or zip if cache
|
||||
directory didn't exist. The message is now only printed on a terminal.
|
||||
|
||||
https://github.com/restic/restic/issues/3334
|
||||
https://github.com/restic/restic/pull/3343
|
||||
@@ -1,8 +0,0 @@
|
||||
Enhancement: SFTP backend now checks for disk space
|
||||
|
||||
Backing up over SFTP previously spewed multiple generic "failure" messages
|
||||
when the remote disk was full. It now checks for disk space before writing
|
||||
a file and fails immediately with a "no space left on device" message.
|
||||
|
||||
https://github.com/restic/restic/issues/3336
|
||||
https://github.com/restic/restic/pull/3345
|
||||
@@ -1,6 +0,0 @@
|
||||
Enhancement: Add release binaries for Apple Silicon
|
||||
|
||||
We've added release binaries for macOS on Apple Silicon (M1).
|
||||
|
||||
https://github.com/restic/restic/issues/3377
|
||||
https://github.com/restic/restic/pull/3394
|
||||
@@ -1,7 +0,0 @@
|
||||
Bugfix: Fix crash of `backup --exclude='**'`
|
||||
|
||||
The exclude filter `**`, which excludes all files, caused restic to crash. This
|
||||
has been corrected.
|
||||
|
||||
https://github.com/restic/restic/issues/3380
|
||||
https://github.com/restic/restic/pull/3393
|
||||
@@ -1,20 +0,0 @@
|
||||
Enhancement: Add `--keep-within-hourly` option to restic forget
|
||||
|
||||
The `forget` command allowed keeping a given number of hourly
|
||||
backups or to keep all backups within a given interval, but it
|
||||
was not possible to specify keeping hourly backups within a given
|
||||
interval.
|
||||
|
||||
The new `--keep-within-hourly` option now offers this functionality.
|
||||
Similar options for daily/weekly/monthly/yearly are also implemented,
|
||||
the new options are:
|
||||
|
||||
--keep-within-hourly <1y2m3d4h>
|
||||
--keep-within-daily <1y2m3d4h>
|
||||
--keep-within-weekly <1y2m3d4h>
|
||||
--keep-within-monthly <1y2m3d4h>
|
||||
--keep-within-yearly <1y2m3d4h>
|
||||
|
||||
https://github.com/restic/restic/issues/3414
|
||||
https://github.com/restic/restic/pull/3416
|
||||
https://forum.restic.net/t/forget-policy/4014/11
|
||||
@@ -1,9 +0,0 @@
|
||||
Enhancement: Support filtering and specifying untagged snapshots
|
||||
|
||||
It was previously not possible to specify an empty tag with the `--tag` and
|
||||
`--keep-tag` options. This has now been fixed, such that `--tag ''` and
|
||||
`--keep-tag ''` now matches snapshots without tags. This allows e.g. the
|
||||
`snapshots` and `forget` commands to only operate on untagged snapshots.
|
||||
|
||||
https://github.com/restic/restic/issues/3456
|
||||
https://github.com/restic/restic/pull/3457
|
||||
@@ -1,9 +0,0 @@
|
||||
Enhancement: Allow specifying limit of `snapshots` list
|
||||
|
||||
The `--last` option allowed limiting the output of the `snapshots`
|
||||
command to the latest snapshot for each host. The new `--latest n`
|
||||
option allows limiting the output to the latest `n` snapshots.
|
||||
|
||||
This change deprecates the option `--last` in favour of `--latest 1`.
|
||||
|
||||
https://github.com/restic/restic/pull/3167
|
||||
@@ -1,6 +0,0 @@
|
||||
Bugfix: Fix possibly missing backup summary of JSON output in case of error
|
||||
|
||||
When using `--json` output it happened from time to time that the summary
|
||||
output was missing in case an error occurred. This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/pull/3305
|
||||
@@ -1,7 +0,0 @@
|
||||
Enhancement: Optimize read performance of mount command
|
||||
|
||||
Reading large files in a mounted repository may be up to five times faster.
|
||||
This improvement primarily applies to repositories stored at a backend that can
|
||||
be accessed with low latency, like e.g. the local backend.
|
||||
|
||||
https://github.com/restic/restic/pull/3426
|
||||
@@ -1,13 +0,0 @@
|
||||
Enhancement: `find --pack` fallback to index if data file is missing
|
||||
|
||||
When investigating a repository with missing data files, it might be useful to
|
||||
determine affected snapshots before running `rebuild-index`. Previously,
|
||||
`find --pack pack-id` returned no data as it required accessing the data file.
|
||||
Now, if the necessary data is still available in the repository index, it gets
|
||||
retrieved from there.
|
||||
|
||||
The command now also supports looking up multiple pack files in a single `find`
|
||||
run.
|
||||
|
||||
https://github.com/restic/restic/pull/3427
|
||||
https://forum.restic.net/t/missing-packs-not-found/2600
|
||||
@@ -1,8 +0,0 @@
|
||||
Bugfix: Correctly handle download errors during `restore`
|
||||
|
||||
Due to a regression in restic 0.12.0, the `restore` command in some cases did
|
||||
not retry download errors and only printed a warning. This has been fixed by
|
||||
retrying incomplete data downloads.
|
||||
|
||||
https://github.com/restic/restic/issues/3439
|
||||
https://github.com/restic/restic/pull/3449
|
||||
11
changelog/unreleased/issue-1212
Normal file
11
changelog/unreleased/issue-1212
Normal file
@@ -0,0 +1,11 @@
|
||||
Bugfix: Restore timestamps and permissions on intermediate directories
|
||||
|
||||
When using the `--include` option of the restore command, restic restored
|
||||
timestamps and permissions only on directories selected by the include pattern.
|
||||
Intermediate directories, which are necessary to restore files located in sub-
|
||||
directories, were created with default permissions. We've fixed the restore
|
||||
command to restore timestamps and permissions for these directories as well.
|
||||
|
||||
https://github.com/restic/restic/issues/1212
|
||||
https://github.com/restic/restic/issues/1402
|
||||
https://github.com/restic/restic/pull/2906
|
||||
15
changelog/unreleased/issue-1756
Normal file
15
changelog/unreleased/issue-1756
Normal file
@@ -0,0 +1,15 @@
|
||||
Bugfix: Mark repository files as read-only when using the local backend
|
||||
|
||||
Files stored in a local repository were marked as writeable on the
|
||||
filesystem for non-Windows systems, which did not prevent accidental file
|
||||
modifications outside of restic. In addition, the local backend did not work
|
||||
with certain filesystems and network mounts which do not permit modifications
|
||||
of file permissions.
|
||||
|
||||
restic now marks files stored in a local repository as read-only on the
|
||||
filesystem on non-Windows systems. The error handling is improved to support
|
||||
more filesystems.
|
||||
|
||||
https://github.com/restic/restic/issues/1756
|
||||
https://github.com/restic/restic/issues/2157
|
||||
https://github.com/restic/restic/pull/2989
|
||||
10
changelog/unreleased/issue-2241
Normal file
10
changelog/unreleased/issue-2241
Normal file
@@ -0,0 +1,10 @@
|
||||
Bugfix: Hide password in REST backend repository URLs
|
||||
|
||||
When using a password in the REST backend repository URL,
|
||||
the password could in some cases be included in the output
|
||||
from restic, e.g. when initializing a repo or during an error.
|
||||
|
||||
The password is now replaced with "***" where applicable.
|
||||
|
||||
https://github.com/restic/restic/issues/2241
|
||||
https://github.com/restic/restic/pull/2658
|
||||
12
changelog/unreleased/issue-2319
Normal file
12
changelog/unreleased/issue-2319
Normal file
@@ -0,0 +1,12 @@
|
||||
Bugfix: Correctly dump directories into tar files
|
||||
|
||||
The dump command previously wrote directories in a tar file in a way which
|
||||
can cause compatibility problems. This caused, for example, 7zip on Windows
|
||||
to not open tar files containing directories. In addition it was not possible
|
||||
to dump directories with extended attributes. These compatibility problems
|
||||
are now corrected.
|
||||
|
||||
In addition, a tar file now includes the name of the owner and group of a file.
|
||||
|
||||
https://github.com/restic/restic/issues/2319
|
||||
https://github.com/restic/restic/pull/3039
|
||||
9
changelog/unreleased/issue-2491
Normal file
9
changelog/unreleased/issue-2491
Normal file
@@ -0,0 +1,9 @@
|
||||
Bugfix: Don't require `self-update --output` placeholder file
|
||||
|
||||
`restic self-update --output /path/to/new-restic` used to require that
|
||||
new-restic was an existing file, to be overwritten. Now it's possible
|
||||
to download an updated restic binary to a new path, without first
|
||||
having to create a placeholder file.
|
||||
|
||||
https://github.com/restic/restic/issues/2491
|
||||
https://github.com/restic/restic/pull/2937
|
||||
7
changelog/unreleased/issue-2834
Normal file
7
changelog/unreleased/issue-2834
Normal file
@@ -0,0 +1,7 @@
|
||||
Bugfix: Fix rare cases of backup command hanging forever
|
||||
|
||||
We've fixed an issue with the backup progress reporting which could cause
|
||||
restic to hang forever right before finishing a backup.
|
||||
|
||||
https://github.com/restic/restic/issues/2834
|
||||
https://github.com/restic/restic/pull/2963
|
||||
6
changelog/unreleased/issue-2938
Normal file
6
changelog/unreleased/issue-2938
Normal file
@@ -0,0 +1,6 @@
|
||||
Bugfix: Fix manpage formatting
|
||||
|
||||
The manpage formatting in restic v0.10.0 was garbled, which is fixed now.
|
||||
|
||||
https://github.com/restic/restic/issues/2938
|
||||
https://github.com/restic/restic/pull/2977
|
||||
7
changelog/unreleased/issue-2942
Normal file
7
changelog/unreleased/issue-2942
Normal file
@@ -0,0 +1,7 @@
|
||||
Bugfix: Make --exclude-larger-than handle disappearing files
|
||||
|
||||
There was a small bug in the backup command's --exclude-larger-than
|
||||
option where files that disappeared between scanning and actually
|
||||
backing them up to the repository caused a panic. This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/2942
|
||||
9
changelog/unreleased/issue-2951
Normal file
9
changelog/unreleased/issue-2951
Normal file
@@ -0,0 +1,9 @@
|
||||
Bugfix: restic generate, help and self-update no longer check passwords
|
||||
|
||||
The commands `restic cache`, `generate`, `help` and `self-update` don't need
|
||||
passwords, but they previously did run the RESTIC_PASSWORD_COMMAND (if set in
|
||||
the environment), prompting users to authenticate for no reason. They now skip
|
||||
running the password command.
|
||||
|
||||
https://github.com/restic/restic/issues/2951
|
||||
https://github.com/restic/restic/pull/2987
|
||||
9
changelog/unreleased/issue-2969
Normal file
9
changelog/unreleased/issue-2969
Normal file
@@ -0,0 +1,9 @@
|
||||
Enhancement: Optimize check for unchanged files during backup
|
||||
|
||||
During a backup restic skips processing files which have not changed since the last backup run.
|
||||
Previously this required opening each file once which can be slow on network filesystems. The
|
||||
backup command now checks for file changes before opening a file. This considerably reduces
|
||||
the time to create a backup on network filesystems.
|
||||
|
||||
https://github.com/restic/restic/issues/2969
|
||||
https://github.com/restic/restic/pull/2970
|
||||
9
changelog/unreleased/issue-2979
Normal file
9
changelog/unreleased/issue-2979
Normal file
@@ -0,0 +1,9 @@
|
||||
Bugfix: Make snapshots --json output [] instead of null when no snapshots
|
||||
|
||||
Restic previously output `null` instead of `[]` for the `--json snapshots`
|
||||
command, when there were no snapshots in the repository. This caused some
|
||||
minor problems when parsing the output, but is now fixed such that `[]` is
|
||||
output when the list of snapshots is empty.
|
||||
|
||||
https://github.com/restic/restic/issues/2979
|
||||
https://github.com/restic/restic/pull/2984
|
||||
12
changelog/unreleased/issue-340
Normal file
12
changelog/unreleased/issue-340
Normal file
@@ -0,0 +1,12 @@
|
||||
Enhancement: Add support for Volume Shadow Copy Service (VSS) on Windows
|
||||
|
||||
Volume Shadow Copy Service allows read access to files that are locked by
|
||||
another process using an exclusive lock through a filesystem snapshot. Restic
|
||||
was unable to backup those files before. This update enables backing up these
|
||||
files.
|
||||
|
||||
This needs to be enabled explicitely using the --use-fs-snapshot option of the
|
||||
backup command.
|
||||
|
||||
https://github.com/restic/restic/issues/340
|
||||
https://github.com/restic/restic/pull/2274
|
||||
7
changelog/unreleased/pull-2849
Normal file
7
changelog/unreleased/pull-2849
Normal file
@@ -0,0 +1,7 @@
|
||||
Enhancement: Authenticate to Google Cloud Storage with access token
|
||||
|
||||
When using the GCS backend, it is now possible to authenticate with OAuth2
|
||||
access tokens instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN
|
||||
environment variable.
|
||||
|
||||
https://github.com/restic/restic/pull/2849
|
||||
10
changelog/unreleased/pull-2910
Normal file
10
changelog/unreleased/pull-2910
Normal file
@@ -0,0 +1,10 @@
|
||||
Enhancement: New option --repository-file
|
||||
|
||||
We've added a new command-line option --repository-file as an alternative
|
||||
to -r. This allows to read the repository URL from a file in order to
|
||||
prevent certain types of information leaks, especially for URLs containing
|
||||
credentials.
|
||||
|
||||
https://github.com/restic/restic/issues/1458
|
||||
https://github.com/restic/restic/issues/2900
|
||||
https://github.com/restic/restic/pull/2910
|
||||
8
changelog/unreleased/pull-2978
Normal file
8
changelog/unreleased/pull-2978
Normal file
@@ -0,0 +1,8 @@
|
||||
Enhancement: Warn if parent snapshot cannot be loaded during backup
|
||||
|
||||
During a backup restic uses the parent snapshot to check whether a file was
|
||||
changed and has to be backed up again. For this check the backup has to read
|
||||
the directories contained in the old snapshot. If a tree blob cannot be
|
||||
loaded, restic now warns about this problem with the backup repository.
|
||||
|
||||
https://github.com/restic/restic/pull/2978
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -55,16 +56,24 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
|
||||
},
|
||||
DisableAutoGenTag: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if backupOptions.Stdin {
|
||||
for _, filename := range backupOptions.FilesFrom {
|
||||
if filename == "-" {
|
||||
return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var t tomb.Tomb
|
||||
term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
|
||||
t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
|
||||
|
||||
err := runBackup(backupOptions, globalOptions, term, args)
|
||||
t.Kill(nil)
|
||||
if werr := t.Wait(); werr != nil {
|
||||
panic(fmt.Sprintf("term.Run() returned err: %v", err))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
t.Kill(nil)
|
||||
return t.Wait()
|
||||
},
|
||||
}
|
||||
|
||||
@@ -82,22 +91,19 @@ type BackupOptions struct {
|
||||
ExcludeLargerThan string
|
||||
Stdin bool
|
||||
StdinFilename string
|
||||
Tags restic.TagLists
|
||||
Tags []string
|
||||
Host string
|
||||
FilesFrom []string
|
||||
FilesFromVerbatim []string
|
||||
FilesFromRaw []string
|
||||
TimeStamp string
|
||||
WithAtime bool
|
||||
IgnoreInode bool
|
||||
IgnoreCtime bool
|
||||
UseFsSnapshot bool
|
||||
}
|
||||
|
||||
var backupOptions BackupOptions
|
||||
|
||||
// ErrInvalidSourceData is used to report an incomplete backup
|
||||
var ErrInvalidSourceData = errors.New("at least one source file could not be read")
|
||||
var ErrInvalidSourceData = errors.New("failed to read all source data during backup")
|
||||
|
||||
func init() {
|
||||
cmdRoot.AddCommand(cmdBackup)
|
||||
@@ -109,29 +115,22 @@ func init() {
|
||||
f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
|
||||
f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
|
||||
f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
|
||||
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
|
||||
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
|
||||
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
|
||||
f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
|
||||
f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
|
||||
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
|
||||
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
|
||||
f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
|
||||
f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
|
||||
|
||||
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
|
||||
f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
|
||||
err := f.MarkDeprecated("hostname", "use --host")
|
||||
if err != nil {
|
||||
// MarkDeprecated only returns an error when the flag could not be found
|
||||
panic(err)
|
||||
}
|
||||
f.MarkDeprecated("hostname", "use --host")
|
||||
|
||||
f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
|
||||
f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
|
||||
f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
|
||||
f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args/can be specified multiple times)")
|
||||
f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
|
||||
f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
|
||||
f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
|
||||
f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
|
||||
if runtime.GOOS == "windows" {
|
||||
f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
|
||||
}
|
||||
@@ -157,13 +156,11 @@ func filterExisting(items []string) (result []string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// readLines reads all lines from the named file and returns them as a
|
||||
// string slice.
|
||||
//
|
||||
// If filename is empty, readPatternsFromFile returns an empty slice.
|
||||
// If filename is a dash (-), readPatternsFromFile will read the lines from the
|
||||
// readFromFile will read all lines from the given filename and return them as
|
||||
// a string array, if filename is empty readFromFile returns and empty string
|
||||
// array. If filename is a dash (-), readFromFile will read the lines from the
|
||||
// standard input.
|
||||
func readLines(filename string) ([]string, error) {
|
||||
func readLinesFromFile(filename string) ([]string, error) {
|
||||
if filename == "" {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -187,72 +184,29 @@ func readLines(filename string) ([]string, error) {
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
// ignore empty lines
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
// strip comments
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// readFilenamesFromFileRaw reads a list of filenames from the given file,
|
||||
// or stdin if filename is "-". Each filename is terminated by a zero byte,
|
||||
// which is stripped off.
|
||||
func readFilenamesFromFileRaw(filename string) (names []string, err error) {
|
||||
f := os.Stdin
|
||||
if filename != "-" {
|
||||
if f, err = os.Open(filename); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
names, err = readFilenamesRaw(f)
|
||||
if err != nil {
|
||||
// ignore subsequent errors
|
||||
_ = f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func readFilenamesRaw(r io.Reader) (names []string, err error) {
|
||||
br := bufio.NewReader(r)
|
||||
for {
|
||||
name, err := br.ReadString(0)
|
||||
switch err {
|
||||
case nil:
|
||||
case io.EOF:
|
||||
if name == "" {
|
||||
return names, nil
|
||||
}
|
||||
return nil, errors.Fatal("--files-from-raw: trailing zero byte missing")
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
name = name[:len(name)-1]
|
||||
if name == "" {
|
||||
// The empty filename is never valid. Handle this now to
|
||||
// prevent downstream code from erroneously backing up
|
||||
// filepath.Clean("") == ".".
|
||||
return nil, errors.Fatal("--files-from-raw: empty filename in listing")
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
|
||||
// Check returns an error when an invalid combination of options was set.
|
||||
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
||||
if gopts.password == "" {
|
||||
filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
|
||||
for _, filename := range filesFrom {
|
||||
for _, filename := range opts.FilesFrom {
|
||||
if filename == "-" {
|
||||
return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
|
||||
}
|
||||
@@ -263,12 +217,6 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
||||
if len(opts.FilesFrom) > 0 {
|
||||
return errors.Fatal("--stdin and --files-from cannot be used together")
|
||||
}
|
||||
if len(opts.FilesFromVerbatim) > 0 {
|
||||
return errors.Fatal("--stdin and --files-from-verbatim cannot be used together")
|
||||
}
|
||||
if len(opts.FilesFromRaw) > 0 {
|
||||
return errors.Fatal("--stdin and --files-from-raw cannot be used together")
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
|
||||
@@ -408,19 +356,15 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var lines []string
|
||||
for _, file := range opts.FilesFrom {
|
||||
fromfile, err := readLines(file)
|
||||
fromfile, err := readLinesFromFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// expand wildcards
|
||||
for _, line := range fromfile {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || line[0] == '#' { // '#' marks a comment.
|
||||
continue
|
||||
}
|
||||
|
||||
var expanded []string
|
||||
expanded, err := filepath.Glob(line)
|
||||
if err != nil {
|
||||
@@ -429,38 +373,19 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
|
||||
if len(expanded) == 0 {
|
||||
Warnf("pattern %q does not match any files, skipping\n", line)
|
||||
}
|
||||
targets = append(targets, expanded...)
|
||||
lines = append(lines, expanded...)
|
||||
}
|
||||
}
|
||||
|
||||
for _, file := range opts.FilesFromVerbatim {
|
||||
fromfile, err := readLines(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range fromfile {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
targets = append(targets, line)
|
||||
}
|
||||
}
|
||||
|
||||
for _, file := range opts.FilesFromRaw {
|
||||
fromfile, err := readFilenamesFromFileRaw(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets = append(targets, fromfile...)
|
||||
}
|
||||
|
||||
// Merge args into files-from so we can reuse the normal args checks
|
||||
// and have the ability to use both files-from and args at the same time.
|
||||
targets = append(targets, args...)
|
||||
if len(targets) == 0 && !opts.Stdin {
|
||||
// merge files from files-from into normal args so we can reuse the normal
|
||||
// args checks and have the ability to use both files-from and args at the
|
||||
// same time
|
||||
args = append(args, lines...)
|
||||
if len(args) == 0 && !opts.Stdin {
|
||||
return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
|
||||
}
|
||||
|
||||
targets = args
|
||||
targets, err = filterExisting(targets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -561,7 +486,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
|
||||
}()
|
||||
gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
|
||||
|
||||
p.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet))
|
||||
if s, ok := os.LookupEnv("RESTIC_PROGRESS_FPS"); ok {
|
||||
fps, err := strconv.Atoi(s)
|
||||
if err == nil && fps >= 1 {
|
||||
if fps > 60 {
|
||||
fps = 60
|
||||
}
|
||||
p.SetMinUpdatePause(time.Second / time.Duration(fps))
|
||||
}
|
||||
}
|
||||
|
||||
t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
|
||||
|
||||
@@ -599,12 +532,8 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
|
||||
return err
|
||||
}
|
||||
|
||||
if !gopts.JSON {
|
||||
if parentSnapshotID != nil {
|
||||
p.P("using parent snapshot %v\n", parentSnapshotID.Str())
|
||||
} else {
|
||||
p.P("no parent snapshot found, will read all files\n")
|
||||
}
|
||||
if !gopts.JSON && parentSnapshotID != nil {
|
||||
p.V("using parent snapshot %v\n", parentSnapshotID.Str())
|
||||
}
|
||||
|
||||
selectByNameFilter := func(item string) bool {
|
||||
@@ -682,15 +611,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
|
||||
arch.CompleteItem = p.CompleteItem
|
||||
arch.StartFile = p.StartFile
|
||||
arch.CompleteBlob = p.CompleteBlob
|
||||
|
||||
if opts.IgnoreInode {
|
||||
// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
|
||||
// reliable either.
|
||||
arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime | archiver.ChangeIgnoreInode
|
||||
}
|
||||
if opts.IgnoreCtime {
|
||||
arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
|
||||
}
|
||||
arch.IgnoreInode = opts.IgnoreInode
|
||||
|
||||
if parentSnapshotID == nil {
|
||||
parentSnapshotID = &restic.ID{}
|
||||
@@ -698,7 +619,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
|
||||
|
||||
snapshotOpts := archiver.SnapshotOptions{
|
||||
Excludes: opts.Excludes,
|
||||
Tags: opts.Tags.Flatten(),
|
||||
Tags: opts.Tags,
|
||||
Time: timeStamp,
|
||||
Hostname: opts.Host,
|
||||
ParentSnapshot: *parentSnapshotID,
|
||||
@@ -708,17 +629,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
|
||||
p.V("start backup on %v", targets)
|
||||
}
|
||||
_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
|
||||
if err != nil {
|
||||
return errors.Fatalf("unable to save snapshot: %v", err)
|
||||
}
|
||||
|
||||
// cleanly shutdown all running goroutines
|
||||
t.Kill(nil)
|
||||
|
||||
// let's see if one returned an error
|
||||
werr := t.Wait()
|
||||
|
||||
// return original error
|
||||
if err != nil {
|
||||
return errors.Fatalf("unable to save snapshot: %v", err)
|
||||
}
|
||||
err = t.Wait()
|
||||
|
||||
// Report finished execution
|
||||
p.Finish(id)
|
||||
@@ -730,5 +649,5 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
|
||||
}
|
||||
|
||||
// Return error if any
|
||||
return werr
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestCollectTargets(t *testing.T) {
|
||||
dir, cleanup := rtest.TempDir(t)
|
||||
defer cleanup()
|
||||
|
||||
fooSpace := "foo "
|
||||
barStar := "bar*" // Must sort before the others, below.
|
||||
if runtime.GOOS == "windows" { // Doesn't allow "*" or trailing space.
|
||||
fooSpace = "foo"
|
||||
barStar = "bar"
|
||||
}
|
||||
|
||||
var expect []string
|
||||
for _, filename := range []string{
|
||||
barStar, "baz", "cmdline arg", fooSpace,
|
||||
"fromfile", "fromfile-raw", "fromfile-verbatim", "quux",
|
||||
} {
|
||||
// All mentioned files must exist for collectTargets.
|
||||
f, err := os.Create(filepath.Join(dir, filename))
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, f.Close())
|
||||
|
||||
expect = append(expect, f.Name())
|
||||
}
|
||||
|
||||
f1, err := os.Create(filepath.Join(dir, "fromfile"))
|
||||
rtest.OK(t, err)
|
||||
// Empty lines should be ignored. A line starting with '#' is a comment.
|
||||
fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
|
||||
rtest.OK(t, f1.Close())
|
||||
|
||||
f2, err := os.Create(filepath.Join(dir, "fromfile-verbatim"))
|
||||
rtest.OK(t, err)
|
||||
for _, filename := range []string{fooSpace, barStar} {
|
||||
// Empty lines should be ignored. CR+LF is allowed.
|
||||
fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
|
||||
}
|
||||
rtest.OK(t, f2.Close())
|
||||
|
||||
f3, err := os.Create(filepath.Join(dir, "fromfile-raw"))
|
||||
rtest.OK(t, err)
|
||||
for _, filename := range []string{"baz", "quux"} {
|
||||
fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
|
||||
}
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, f3.Close())
|
||||
|
||||
opts := BackupOptions{
|
||||
FilesFrom: []string{f1.Name()},
|
||||
FilesFromVerbatim: []string{f2.Name()},
|
||||
FilesFromRaw: []string{f3.Name()},
|
||||
}
|
||||
|
||||
targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
|
||||
rtest.OK(t, err)
|
||||
sort.Strings(targets)
|
||||
rtest.Equals(t, expect, targets)
|
||||
}
|
||||
|
||||
func TestReadFilenamesRaw(t *testing.T) {
|
||||
// These should all be returned exactly as-is.
|
||||
expected := []string{
|
||||
"\xef\xbb\xbf/utf-8-bom",
|
||||
"/absolute",
|
||||
"../.././relative",
|
||||
"\t\t leading and trailing space \t\t",
|
||||
"newline\nin filename",
|
||||
"not UTF-8: \x80\xff/simple",
|
||||
` / *[]* \ `,
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
for _, name := range expected {
|
||||
buf.WriteString(name)
|
||||
buf.WriteByte(0)
|
||||
}
|
||||
|
||||
got, err := readFilenamesRaw(&buf)
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, expected, got)
|
||||
|
||||
// Empty input is ok.
|
||||
got, err = readFilenamesRaw(strings.NewReader(""))
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, 0, len(got))
|
||||
|
||||
// An empty filename is an error.
|
||||
_, err = readFilenamesRaw(strings.NewReader("foo\x00\x00"))
|
||||
rtest.Assert(t, err != nil, "no error for zero byte")
|
||||
rtest.Assert(t, strings.Contains(err.Error(), "empty filename"),
|
||||
"wrong error message: %v", err.Error())
|
||||
|
||||
// No trailing NUL byte is an error, because it likely means we're
|
||||
// reading a line-oriented text file (someone forgot -print0).
|
||||
_, err = readFilenamesRaw(strings.NewReader("simple.txt"))
|
||||
rtest.Assert(t, err != nil, "no error for zero byte")
|
||||
rtest.Assert(t, strings.Contains(err.Error(), "zero byte"),
|
||||
"wrong error message: %v", err.Error())
|
||||
}
|
||||
@@ -148,7 +148,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
|
||||
})
|
||||
}
|
||||
|
||||
_ = tab.Write(gopts.stdout)
|
||||
tab.Write(gopts.stdout)
|
||||
Printf("%d cache dirs in %s\n", len(dirs), cachedir)
|
||||
|
||||
return nil
|
||||
|
||||
@@ -42,13 +42,10 @@ func runCat(gopts GlobalOptions, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if !gopts.NoLock {
|
||||
lock, err := lockRepo(gopts.ctx, repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer unlockRepo(lock)
|
||||
lock, err := lockRepo(gopts.ctx, repo)
|
||||
defer unlockRepo(lock)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tpe := args[0]
|
||||
@@ -69,6 +66,7 @@ func runCat(gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// handle all types that don't need an index
|
||||
switch tpe {
|
||||
case "config":
|
||||
buf, err := json.MarshalIndent(repo.Config(), "", " ")
|
||||
@@ -141,7 +139,15 @@ func runCat(gopts GlobalOptions, args []string) error {
|
||||
|
||||
Println(string(buf))
|
||||
return nil
|
||||
}
|
||||
|
||||
// load index, handle all the other types
|
||||
err = repo.LoadIndex(gopts.ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch tpe {
|
||||
case "pack":
|
||||
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
|
||||
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
|
||||
@@ -158,14 +164,8 @@ func runCat(gopts GlobalOptions, args []string) error {
|
||||
return err
|
||||
|
||||
case "blob":
|
||||
err = repo.LoadIndex(gopts.ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
|
||||
bh := restic.BlobHandle{ID: id, Type: t}
|
||||
if !repo.Index().Has(bh) {
|
||||
if !repo.Index().Has(id, t) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
@@ -54,38 +53,25 @@ func init() {
|
||||
|
||||
f := cmdCheck.Flags()
|
||||
f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
|
||||
f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
|
||||
f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read subset n of m data packs (format: `n/m`)")
|
||||
f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
|
||||
f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
|
||||
}
|
||||
|
||||
func checkFlags(opts CheckOptions) error {
|
||||
if opts.ReadData && opts.ReadDataSubset != "" {
|
||||
return errors.Fatal("check flags --read-data and --read-data-subset cannot be used together")
|
||||
return errors.Fatalf("check flags --read-data and --read-data-subset cannot be used together")
|
||||
}
|
||||
if opts.ReadDataSubset != "" {
|
||||
dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
|
||||
argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
|
||||
if err == nil {
|
||||
if len(dataSubset) != 2 {
|
||||
return argumentError
|
||||
}
|
||||
if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
|
||||
return errors.Fatal("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
|
||||
}
|
||||
if dataSubset[1] > totalBucketsMax {
|
||||
return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
|
||||
}
|
||||
} else {
|
||||
percentage, err := parsePercentage(opts.ReadDataSubset)
|
||||
if err != nil {
|
||||
return argumentError
|
||||
}
|
||||
|
||||
if percentage <= 0.0 || percentage > 100.0 {
|
||||
return errors.Fatal(
|
||||
"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
|
||||
}
|
||||
if err != nil || len(dataSubset) != 2 {
|
||||
return errors.Fatalf("check flag --read-data-subset must have two positive integer values, e.g. --read-data-subset=1/2")
|
||||
}
|
||||
if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
|
||||
return errors.Fatalf("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
|
||||
}
|
||||
if dataSubset[1] > totalBucketsMax {
|
||||
return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,21 +98,6 @@ func stringToIntSlice(param string) (split []uint, err error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ParsePercentage parses a percentage string of the form "X%" where X is a float constant,
|
||||
// and returns the value of that constant. It does not check the range of the value.
|
||||
func parsePercentage(s string) (float64, error) {
|
||||
if !strings.HasSuffix(s, "%") {
|
||||
return 0, errors.Errorf(`parsePercentage: %q does not end in "%%"`, s)
|
||||
}
|
||||
s = s[:len(s)-1]
|
||||
|
||||
p, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, errors.Errorf("parsePercentage: %v", err)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// prepareCheckCache configures a special cache directory for check.
|
||||
//
|
||||
// * if --with-cache is specified, the default cache is used
|
||||
@@ -194,7 +165,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
chkr := checker.New(repo, opts.CheckUnused)
|
||||
chkr := checker.New(repo)
|
||||
|
||||
Verbosef("load indexes\n")
|
||||
hints, errs := chkr.LoadIndex(gopts.ctx)
|
||||
@@ -241,11 +212,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
|
||||
Verbosef("check snapshots, trees and blobs\n")
|
||||
errChan = make(chan error)
|
||||
go func() {
|
||||
bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
|
||||
defer bar.Done()
|
||||
chkr.Structure(gopts.ctx, bar, errChan)
|
||||
}()
|
||||
go chkr.Structure(gopts.ctx, errChan)
|
||||
|
||||
for err := range errChan {
|
||||
errorsFound = true
|
||||
@@ -260,15 +227,29 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
|
||||
if opts.CheckUnused {
|
||||
for _, id := range chkr.UnusedBlobs(gopts.ctx) {
|
||||
for _, id := range chkr.UnusedBlobs() {
|
||||
Verbosef("unused blob %v\n", id)
|
||||
errorsFound = true
|
||||
}
|
||||
}
|
||||
|
||||
doReadData := func(packs map[restic.ID]int64) {
|
||||
doReadData := func(bucket, totalBuckets uint) {
|
||||
packs := restic.IDSet{}
|
||||
for pack := range chkr.GetPacks() {
|
||||
// If we ever check more than the first byte
|
||||
// of pack, update totalBucketsMax.
|
||||
if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
|
||||
packs.Insert(pack)
|
||||
}
|
||||
}
|
||||
packCount := uint64(len(packs))
|
||||
|
||||
if packCount < chkr.CountPacks() {
|
||||
Verbosef(fmt.Sprintf("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets))
|
||||
} else {
|
||||
Verbosef("read all data\n")
|
||||
}
|
||||
|
||||
p := newProgressMax(!gopts.Quiet, packCount, "packs")
|
||||
errChan := make(chan error)
|
||||
|
||||
@@ -278,31 +259,14 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
errorsFound = true
|
||||
Warnf("%v\n", err)
|
||||
}
|
||||
p.Done()
|
||||
}
|
||||
|
||||
switch {
|
||||
case opts.ReadData:
|
||||
Verbosef("read all data\n")
|
||||
doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
|
||||
doReadData(1, 1)
|
||||
case opts.ReadDataSubset != "":
|
||||
var packs map[restic.ID]int64
|
||||
dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
|
||||
if err == nil {
|
||||
bucket := dataSubset[0]
|
||||
totalBuckets := dataSubset[1]
|
||||
packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
|
||||
packCount := uint64(len(packs))
|
||||
Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
|
||||
} else {
|
||||
percentage, _ := parsePercentage(opts.ReadDataSubset)
|
||||
packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
|
||||
Verbosef("read %.1f%% of data packs\n", percentage)
|
||||
}
|
||||
if packs == nil {
|
||||
return errors.Fatal("internal error: failed to select packs to check")
|
||||
}
|
||||
doReadData(packs)
|
||||
dataSubset, _ := stringToIntSlice(opts.ReadDataSubset)
|
||||
doReadData(dataSubset[0], dataSubset[1])
|
||||
}
|
||||
|
||||
if errorsFound {
|
||||
@@ -313,42 +277,3 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// selectPacksByBucket selects subsets of packs by ranges of buckets.
|
||||
func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
|
||||
packs := make(map[restic.ID]int64)
|
||||
for pack, size := range allPacks {
|
||||
// If we ever check more than the first byte
|
||||
// of pack, update totalBucketsMax.
|
||||
if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
|
||||
packs[pack] = size
|
||||
}
|
||||
}
|
||||
return packs
|
||||
}
|
||||
|
||||
// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
|
||||
func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
|
||||
packCount := len(allPacks)
|
||||
packsToCheck := int(float64(packCount) * (percentage / 100.0))
|
||||
if packCount > 0 && packsToCheck < 1 {
|
||||
packsToCheck = 1
|
||||
}
|
||||
timeNs := time.Now().UnixNano()
|
||||
r := rand.New(rand.NewSource(timeNs))
|
||||
idx := r.Perm(packCount)
|
||||
|
||||
var keys []restic.ID
|
||||
for k := range allPacks {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
packs := make(map[restic.ID]int64)
|
||||
|
||||
for i := 0; i < packsToCheck; i++ {
|
||||
id := keys[idx[i]]
|
||||
packs[id] = allPacks[id]
|
||||
}
|
||||
|
||||
return packs
|
||||
}
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestParsePercentage(t *testing.T) {
|
||||
testCases := []struct {
|
||||
input string
|
||||
output float64
|
||||
expectError bool
|
||||
}{
|
||||
{"0%", 0.0, false},
|
||||
{"1%", 1.0, false},
|
||||
{"100%", 100.0, false},
|
||||
{"123%", 123.0, false},
|
||||
{"123.456%", 123.456, false},
|
||||
{"0.742%", 0.742, false},
|
||||
{"-100%", -100.0, false},
|
||||
{" 1%", 0.0, true},
|
||||
{"1 %", 0.0, true},
|
||||
{"1% ", 0.0, true},
|
||||
}
|
||||
for _, testCase := range testCases {
|
||||
output, err := parsePercentage(testCase.input)
|
||||
|
||||
if testCase.expectError {
|
||||
rtest.Assert(t, err != nil, "Expected error for case %s", testCase.input)
|
||||
rtest.Assert(t, output == 0.0, "Expected output to be 0.0, got %s", output)
|
||||
} else {
|
||||
rtest.Assert(t, err == nil, "Expected no error for case %s", testCase.input)
|
||||
rtest.Assert(t, math.Abs(testCase.output-output) < 0.00001, "Expected %f, got %f",
|
||||
testCase.output, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStringToIntSlice(t *testing.T) {
|
||||
testCases := []struct {
|
||||
input string
|
||||
output []uint
|
||||
expectError bool
|
||||
}{
|
||||
{"3/5", []uint{3, 5}, false},
|
||||
{"1/100", []uint{1, 100}, false},
|
||||
{"abc", nil, true},
|
||||
{"1/a", nil, true},
|
||||
{"/", nil, true},
|
||||
}
|
||||
for _, testCase := range testCases {
|
||||
output, err := stringToIntSlice(testCase.input)
|
||||
|
||||
if testCase.expectError {
|
||||
rtest.Assert(t, err != nil, "Expected error for case %s", testCase.input)
|
||||
rtest.Assert(t, output == nil, "Expected output to be nil, got %s", output)
|
||||
} else {
|
||||
rtest.Assert(t, err == nil, "Expected no error for case %s", testCase.input)
|
||||
rtest.Assert(t, len(output) == 2, "Invalid output length for case %s", testCase.input)
|
||||
rtest.Assert(t, reflect.DeepEqual(output, testCase.output), "Expected %f, got %f",
|
||||
testCase.output, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectPacksByBucket(t *testing.T) {
|
||||
var testPacks = make(map[restic.ID]int64)
|
||||
for i := 1; i <= 10; i++ {
|
||||
id := restic.NewRandomID()
|
||||
// ensure relevant part of generated id is reproducable
|
||||
id[0] = byte(i)
|
||||
testPacks[id] = 0
|
||||
}
|
||||
|
||||
selectedPacks := selectPacksByBucket(testPacks, 0, 10)
|
||||
rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
|
||||
|
||||
for i := uint(1); i <= 5; i++ {
|
||||
selectedPacks = selectPacksByBucket(testPacks, i, 5)
|
||||
rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
|
||||
}
|
||||
|
||||
selectedPacks = selectPacksByBucket(testPacks, 1, 1)
|
||||
rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
|
||||
for testPack := range testPacks {
|
||||
_, ok := selectedPacks[testPack]
|
||||
rtest.Assert(t, ok, "Expected input and output to be equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectRandomPacksByPercentage(t *testing.T) {
|
||||
var testPacks = make(map[restic.ID]int64)
|
||||
for i := 1; i <= 10; i++ {
|
||||
testPacks[restic.NewRandomID()] = 0
|
||||
}
|
||||
|
||||
selectedPacks := selectRandomPacksByPercentage(testPacks, 0.0)
|
||||
rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
|
||||
|
||||
selectedPacks = selectRandomPacksByPercentage(testPacks, 10.0)
|
||||
rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected pack")
|
||||
for pack := range selectedPacks {
|
||||
_, ok := testPacks[pack]
|
||||
rtest.Assert(t, ok, "Unexpected selection")
|
||||
}
|
||||
|
||||
selectedPacks = selectRandomPacksByPercentage(testPacks, 50.0)
|
||||
rtest.Assert(t, len(selectedPacks) == 5, "Expected 5 selected packs")
|
||||
for pack := range selectedPacks {
|
||||
_, ok := testPacks[pack]
|
||||
rtest.Assert(t, ok, "Unexpected item in selection")
|
||||
}
|
||||
|
||||
selectedPacks = selectRandomPacksByPercentage(testPacks, 100.0)
|
||||
rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
|
||||
for testPack := range testPacks {
|
||||
_, ok := selectedPacks[testPack]
|
||||
rtest.Assert(t, ok, "Expected input and output to be equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectNoRandomPacksByPercentage(t *testing.T) {
|
||||
// that the a repository without pack files works
|
||||
var testPacks = make(map[restic.ID]int64)
|
||||
selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
|
||||
rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -15,19 +14,12 @@ var cmdCopy = &cobra.Command{
|
||||
Use: "copy [flags] [snapshotID ...]",
|
||||
Short: "Copy snapshots from one repository to another",
|
||||
Long: `
|
||||
The "copy" command copies one or more snapshots from one repository to another.
|
||||
|
||||
NOTE: This process will have to both download (read) and upload (write) the
|
||||
entire snapshot(s) due to the different encryption keys used in the source and
|
||||
destination repositories. This /may incur higher bandwidth usage and costs/ than
|
||||
expected during normal backup runs.
|
||||
|
||||
NOTE: The copying process does not re-chunk files, which may break deduplication
|
||||
between the files copied and files already stored in the destination repository.
|
||||
This means that copied files, which existed in both the source and destination
|
||||
repository, /may occupy up to twice their space/ in the destination repository.
|
||||
This can be mitigated by the "--copy-chunker-params" option when initializing a
|
||||
new destination repository using the "init" command.
|
||||
The "copy" command copies one or more snapshots from one repository to another
|
||||
repository. Note that this will have to read (download) and write (upload) the
|
||||
entire snapshot(s) due to the different encryption keys on the source and
|
||||
destination, and that transferred files are not re-chunked, which may break
|
||||
their deduplication. This can be mitigated by the "--copy-chunker-params"
|
||||
option when initializing a new destination repository using the "init" command.
|
||||
`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runCopy(copyOptions, globalOptions, args)
|
||||
@@ -104,8 +96,12 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
|
||||
dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
|
||||
}
|
||||
|
||||
// remember already processed trees across all snapshots
|
||||
visitedTrees := restic.NewIDSet()
|
||||
cloner := &treeCloner{
|
||||
srcRepo: srcRepo,
|
||||
dstRepo: dstRepo,
|
||||
visitedTrees: restic.NewIDSet(),
|
||||
buf: nil,
|
||||
}
|
||||
|
||||
for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
|
||||
Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
|
||||
@@ -130,7 +126,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
Verbosef(" copy started, this may take a while...\n")
|
||||
|
||||
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil {
|
||||
if err := cloner.copyTree(ctx, *sn.Tree); err != nil {
|
||||
return err
|
||||
}
|
||||
debug.Log("tree copied")
|
||||
@@ -174,67 +170,64 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
|
||||
visitedTrees restic.IDSet, rootTreeID restic.ID) error {
|
||||
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
|
||||
treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
|
||||
visited := visitedTrees.Has(treeID)
|
||||
visitedTrees.Insert(treeID)
|
||||
return visited
|
||||
}, nil)
|
||||
|
||||
wg.Go(func() error {
|
||||
// reused buffer
|
||||
var buf []byte
|
||||
|
||||
for tree := range treeStream {
|
||||
if tree.Error != nil {
|
||||
return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
|
||||
}
|
||||
|
||||
// Do we already have this tree blob?
|
||||
if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
|
||||
// copy raw tree bytes to avoid problems if the serialization changes
|
||||
var err error
|
||||
buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
|
||||
}
|
||||
|
||||
_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: parallelize blob down/upload
|
||||
|
||||
for _, entry := range tree.Nodes {
|
||||
// Recursion into directories is handled by StreamTrees
|
||||
// Copy the blobs for this file.
|
||||
for _, blobID := range entry.Content {
|
||||
// Do we already have this data blob?
|
||||
if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
|
||||
continue
|
||||
}
|
||||
debug.Log("Copying blob %s\n", blobID.Str())
|
||||
var err error
|
||||
buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
|
||||
}
|
||||
|
||||
_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return wg.Wait()
|
||||
type treeCloner struct {
|
||||
srcRepo restic.Repository
|
||||
dstRepo restic.Repository
|
||||
visitedTrees restic.IDSet
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func (t *treeCloner) copyTree(ctx context.Context, treeID restic.ID) error {
|
||||
// We have already processed this tree
|
||||
if t.visitedTrees.Has(treeID) {
|
||||
return nil
|
||||
}
|
||||
|
||||
tree, err := t.srcRepo.LoadTree(ctx, treeID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
|
||||
}
|
||||
t.visitedTrees.Insert(treeID)
|
||||
|
||||
// Do we already have this tree blob?
|
||||
if !t.dstRepo.Index().Has(treeID, restic.TreeBlob) {
|
||||
newTreeID, err := t.dstRepo.SaveTree(ctx, tree)
|
||||
if err != nil {
|
||||
return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err)
|
||||
}
|
||||
// Assurance only.
|
||||
if newTreeID != treeID {
|
||||
return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str())
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: parellize this stuff, likely only needed inside a tree.
|
||||
|
||||
for _, entry := range tree.Nodes {
|
||||
// If it is a directory, recurse
|
||||
if entry.Type == "dir" && entry.Subtree != nil {
|
||||
if err := t.copyTree(ctx, *entry.Subtree); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Copy the blobs for this file.
|
||||
for _, blobID := range entry.Content {
|
||||
// Do we already have this data blob?
|
||||
if t.dstRepo.Index().Has(blobID, restic.DataBlob) {
|
||||
continue
|
||||
}
|
||||
debug.Log("Copying blob %s\n", blobID.Str())
|
||||
t.buf, err = t.srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, t.buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
|
||||
}
|
||||
|
||||
_, _, err = t.dstRepo.SaveBlob(ctx, restic.DataBlob, t.buf, blobID, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,21 +4,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/pack"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
@@ -48,17 +39,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||
},
|
||||
}
|
||||
|
||||
var tryRepair bool
|
||||
var repairByte bool
|
||||
var extractPack bool
|
||||
|
||||
func init() {
|
||||
cmdRoot.AddCommand(cmdDebug)
|
||||
cmdDebug.AddCommand(cmdDebugDump)
|
||||
cmdDebug.AddCommand(cmdDebugExamine)
|
||||
cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
|
||||
cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
|
||||
cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
|
||||
}
|
||||
|
||||
func prettyPrintJSON(wr io.Writer, item interface{}) error {
|
||||
@@ -72,7 +55,8 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
|
||||
}
|
||||
|
||||
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
|
||||
return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
|
||||
return repo.List(ctx, restic.SnapshotFile, func(id restic.ID, size int64) error {
|
||||
snapshot, err := restic.LoadSnapshot(ctx, repo, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -103,7 +87,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
|
||||
return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
|
||||
|
||||
blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
|
||||
blobs, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
|
||||
if err != nil {
|
||||
Warnf("error for pack %v: %v\n", id.Str(), err)
|
||||
return nil
|
||||
@@ -127,8 +111,10 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
|
||||
}
|
||||
|
||||
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
|
||||
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
|
||||
return repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
|
||||
Printf("index_id: %v\n", id)
|
||||
|
||||
idx, err := repository.LoadIndex(ctx, repo, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -182,367 +168,3 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
|
||||
return errors.Fatalf("no such type %q", tpe)
|
||||
}
|
||||
}
|
||||
|
||||
var cmdDebugExamine = &cobra.Command{
|
||||
Use: "examine pack-ID...",
|
||||
Short: "Examine a pack file",
|
||||
DisableAutoGenTag: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runDebugExamine(globalOptions, args)
|
||||
},
|
||||
}
|
||||
|
||||
func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
|
||||
if bytewise {
|
||||
Printf(" trying to repair blob by finding a broken byte\n")
|
||||
} else {
|
||||
Printf(" trying to repair blob with single bit flip\n")
|
||||
}
|
||||
|
||||
ch := make(chan int)
|
||||
var wg errgroup.Group
|
||||
done := make(chan struct{})
|
||||
var fixed []byte
|
||||
var found bool
|
||||
|
||||
workers := runtime.GOMAXPROCS(0)
|
||||
Printf(" spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Go(func() error {
|
||||
// make a local copy of the buffer
|
||||
buf := make([]byte, len(input))
|
||||
copy(buf, input)
|
||||
|
||||
testFlip := func(idx int, pattern byte) bool {
|
||||
// flip bits
|
||||
buf[idx] ^= pattern
|
||||
|
||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
|
||||
if err == nil {
|
||||
Printf("\n")
|
||||
Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
|
||||
Printf(" hash is %v\n", restic.Hash(plaintext))
|
||||
close(done)
|
||||
found = true
|
||||
fixed = plaintext
|
||||
return true
|
||||
}
|
||||
|
||||
// flip bits back
|
||||
buf[idx] ^= pattern
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range ch {
|
||||
if bytewise {
|
||||
for j := 0; j < 255; j++ {
|
||||
if testFlip(i, byte(j)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for j := 0; j < 7; j++ {
|
||||
// flip each bit once
|
||||
if testFlip(i, (1 << uint(j))) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
wg.Go(func() error {
|
||||
defer close(ch)
|
||||
|
||||
start := time.Now()
|
||||
info := time.Now()
|
||||
for i := range input {
|
||||
select {
|
||||
case ch <- i:
|
||||
case <-done:
|
||||
Printf(" done after %v\n", time.Since(start))
|
||||
return nil
|
||||
}
|
||||
|
||||
if time.Since(info) > time.Second {
|
||||
secs := time.Since(start).Seconds()
|
||||
gps := float64(i) / secs
|
||||
remaining := len(input) - i
|
||||
eta := time.Duration(float64(remaining)/gps) * time.Second
|
||||
|
||||
Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
|
||||
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
|
||||
info = time.Now()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
err := wg.Wait()
|
||||
if err != nil {
|
||||
panic("all go rountines can only return nil")
|
||||
}
|
||||
|
||||
if !found {
|
||||
Printf("\n blob could not be repaired\n")
|
||||
}
|
||||
return fixed
|
||||
}
|
||||
|
||||
func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
|
||||
// strip signature at the end
|
||||
l := len(buf)
|
||||
nonce, ct := buf[:16], buf[16:l-16]
|
||||
out := make([]byte, len(ct))
|
||||
|
||||
c, err := aes.NewCipher(k.EncryptionKey[:])
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("unable to create cipher: %v", err))
|
||||
}
|
||||
e := cipher.NewCTR(c, nonce)
|
||||
e.XORKeyStream(out, ct)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
|
||||
be := repo.Backend()
|
||||
h := restic.Handle{
|
||||
Name: pack.String(),
|
||||
Type: restic.PackFile,
|
||||
}
|
||||
for _, blob := range list {
|
||||
Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
|
||||
buf := make([]byte, blob.Length)
|
||||
err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
|
||||
n, err := io.ReadFull(rd, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read error after %d bytes: %v", n, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
Warnf("error read: %v\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
key := repo.Key()
|
||||
|
||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
|
||||
if err != nil {
|
||||
Warnf("error decrypting blob: %v\n", err)
|
||||
var plain []byte
|
||||
if tryRepair || repairByte {
|
||||
plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
|
||||
}
|
||||
var prefix string
|
||||
if plain != nil {
|
||||
id := restic.Hash(plain)
|
||||
if !id.Equal(blob.ID) {
|
||||
Printf(" repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
|
||||
prefix = "repaired-wrong-hash-"
|
||||
} else {
|
||||
Printf(" successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
|
||||
prefix = "repaired-"
|
||||
}
|
||||
} else {
|
||||
plain = decryptUnsigned(ctx, key, buf)
|
||||
prefix = "damaged-"
|
||||
}
|
||||
err = storePlainBlob(blob.ID, prefix, plain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
id := restic.Hash(plaintext)
|
||||
var prefix string
|
||||
if !id.Equal(blob.ID) {
|
||||
Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
|
||||
prefix = "wrong-hash-"
|
||||
} else {
|
||||
Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
|
||||
prefix = "correct-"
|
||||
}
|
||||
if extractPack {
|
||||
err = storePlainBlob(id, prefix, plaintext)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
|
||||
filename := fmt.Sprintf("%s%s.bin", prefix, id)
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = f.Write(plain)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Printf("decrypt of blob %v stored at %v\n", id, filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runDebugExamine(gopts GlobalOptions, args []string) error {
|
||||
ids := make([]restic.ID, 0)
|
||||
for _, name := range args {
|
||||
id, err := restic.ParseID(name)
|
||||
if err != nil {
|
||||
Warnf("error: %v\n", err)
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
if len(ids) == 0 {
|
||||
return errors.Fatal("no pack files to examine")
|
||||
}
|
||||
|
||||
repo, err := OpenRepository(gopts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !gopts.NoLock {
|
||||
lock, err := lockRepo(gopts.ctx, repo)
|
||||
defer unlockRepo(lock)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(gopts.ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range ids {
|
||||
err := examinePack(gopts.ctx, repo, id)
|
||||
if err != nil {
|
||||
Warnf("error: %v\n", err)
|
||||
}
|
||||
if err == context.Canceled {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
|
||||
Printf("examine %v\n", id)
|
||||
|
||||
h := restic.Handle{
|
||||
Type: restic.PackFile,
|
||||
Name: id.String(),
|
||||
}
|
||||
fi, err := repo.Backend().Stat(ctx, h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
Printf(" file size is %v\n", fi.Size)
|
||||
|
||||
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gotID := restic.Hash(buf)
|
||||
if !id.Equal(gotID) {
|
||||
Printf(" wanted hash %v, got %v\n", id, gotID)
|
||||
} else {
|
||||
Printf(" hash for file content matches\n")
|
||||
}
|
||||
|
||||
Printf(" ========================================\n")
|
||||
Printf(" looking for info in the indexes\n")
|
||||
|
||||
blobsLoaded := false
|
||||
// examine all data the indexes have for the pack file
|
||||
for _, idx := range repo.Index().(*repository.MasterIndex).All() {
|
||||
idxIDs, err := idx.IDs()
|
||||
if err != nil {
|
||||
idxIDs = restic.IDs{}
|
||||
}
|
||||
|
||||
blobs := idx.ListPack(id)
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
Printf(" index %v:\n", idxIDs)
|
||||
|
||||
// convert list of blobs to []restic.Blob
|
||||
var list []restic.Blob
|
||||
for _, b := range blobs {
|
||||
list = append(list, b.Blob)
|
||||
}
|
||||
checkPackSize(list, fi.Size)
|
||||
|
||||
err = loadBlobs(ctx, repo, id, list)
|
||||
if err != nil {
|
||||
Warnf("error: %v\n", err)
|
||||
} else {
|
||||
blobsLoaded = true
|
||||
}
|
||||
}
|
||||
|
||||
Printf(" ========================================\n")
|
||||
Printf(" inspect the pack itself\n")
|
||||
|
||||
blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
||||
}
|
||||
checkPackSize(blobs, fi.Size)
|
||||
|
||||
if !blobsLoaded {
|
||||
return loadBlobs(ctx, repo, id, blobs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkPackSize(blobs []restic.Blob, fileSize int64) {
|
||||
// track current size and offset
|
||||
var size, offset uint64
|
||||
|
||||
sort.Slice(blobs, func(i, j int) bool {
|
||||
return blobs[i].Offset < blobs[j].Offset
|
||||
})
|
||||
|
||||
for _, pb := range blobs {
|
||||
Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
|
||||
if offset != uint64(pb.Offset) {
|
||||
Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset)
|
||||
}
|
||||
offset += uint64(pb.Length)
|
||||
size += uint64(pb.Length)
|
||||
}
|
||||
|
||||
// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
|
||||
size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
|
||||
// length in uint32 little endian
|
||||
size += 4
|
||||
|
||||
if uint64(fileSize) != size {
|
||||
Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
|
||||
} else {
|
||||
Printf(" file sizes match\n")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,8 +55,9 @@ func init() {
|
||||
func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
|
||||
id, err := restic.FindSnapshot(ctx, repo, desc)
|
||||
if err != nil {
|
||||
return nil, errors.Fatal(err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return restic.LoadSnapshot(ctx, repo, id)
|
||||
}
|
||||
|
||||
@@ -364,8 +365,6 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
|
||||
stats := NewDiffStats()
|
||||
stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
|
||||
stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
|
||||
|
||||
err = c.diffTree(ctx, stats, "/", *sn1.Tree, *sn2.Tree)
|
||||
if err != nil {
|
||||
|
||||
@@ -21,8 +21,8 @@ var cmdDump = &cobra.Command{
|
||||
Long: `
|
||||
The "dump" command extracts files from a snapshot from the repository. If a
|
||||
single file is selected, it prints its contents to stdout. Folders are output
|
||||
as a tar (default) or zip file containing the contents of the specified folder.
|
||||
Pass "/" as file name to dump the whole snapshot as an archive file.
|
||||
as a tar file containing the contents of the specified folder. Pass "/" as
|
||||
file name to dump the whole snapshot as a tar file.
|
||||
|
||||
The special snapshot "latest" can be used to use the latest snapshot in the
|
||||
repository.
|
||||
@@ -40,10 +40,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||
|
||||
// DumpOptions collects all options for the dump command.
|
||||
type DumpOptions struct {
|
||||
Hosts []string
|
||||
Paths []string
|
||||
Tags restic.TagLists
|
||||
Archive string
|
||||
Hosts []string
|
||||
Paths []string
|
||||
Tags restic.TagLists
|
||||
}
|
||||
|
||||
var dumpOptions DumpOptions
|
||||
@@ -55,7 +54,6 @@ func init() {
|
||||
flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
|
||||
flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
|
||||
flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
|
||||
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
@@ -67,7 +65,8 @@ func splitPath(p string) []string {
|
||||
return append(s, f)
|
||||
}
|
||||
|
||||
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
|
||||
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error {
|
||||
|
||||
if tree == nil {
|
||||
return fmt.Errorf("called with a nil tree")
|
||||
}
|
||||
@@ -82,10 +81,10 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
|
||||
// If we print / we need to assume that there are multiple nodes at that
|
||||
// level in the tree.
|
||||
if pathComponents[0] == "" {
|
||||
if err := checkStdoutArchive(); err != nil {
|
||||
if err := checkStdoutTar(); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeDump(ctx, repo, tree, "/", os.Stdout)
|
||||
return dump.WriteTar(ctx, repo, tree, "/", os.Stdout)
|
||||
}
|
||||
|
||||
item := filepath.Join(prefix, pathComponents[0])
|
||||
@@ -101,16 +100,16 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "cannot load subtree for %q", item)
|
||||
}
|
||||
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
|
||||
return printFromTree(ctx, subtree, repo, item, pathComponents[1:])
|
||||
case dump.IsDir(node):
|
||||
if err := checkStdoutArchive(); err != nil {
|
||||
if err := checkStdoutTar(); err != nil {
|
||||
return err
|
||||
}
|
||||
subtree, err := repo.LoadTree(ctx, *node.Subtree)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeDump(ctx, repo, subtree, item, os.Stdout)
|
||||
return dump.WriteTar(ctx, repo, subtree, item, os.Stdout)
|
||||
case l > 1:
|
||||
return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
|
||||
case !dump.IsFile(node):
|
||||
@@ -128,16 +127,6 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
|
||||
return errors.Fatal("no file and no snapshot ID specified")
|
||||
}
|
||||
|
||||
var wd dump.WriteDump
|
||||
switch opts.Archive {
|
||||
case "tar":
|
||||
wd = dump.WriteTar
|
||||
case "zip":
|
||||
wd = dump.WriteZip
|
||||
default:
|
||||
return fmt.Errorf("unknown archive format %q", opts.Archive)
|
||||
}
|
||||
|
||||
snapshotIDString := args[0]
|
||||
pathToPrint := args[1]
|
||||
|
||||
@@ -187,7 +176,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
|
||||
Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
|
||||
}
|
||||
|
||||
err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
|
||||
err = printFromTree(ctx, tree, repo, "/", splittedPath)
|
||||
if err != nil {
|
||||
Exitf(2, "cannot dump file: %v", err)
|
||||
}
|
||||
@@ -195,7 +184,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkStdoutArchive() error {
|
||||
func checkStdoutTar() error {
|
||||
if stdoutIsTerminal() {
|
||||
return fmt.Errorf("stdout is the terminal, please redirect output")
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -41,6 +40,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||
},
|
||||
}
|
||||
|
||||
const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
|
||||
|
||||
// FindOptions bundles all options for the find command.
|
||||
type FindOptions struct {
|
||||
Oldest string
|
||||
@@ -385,14 +386,15 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
||||
idStr := id.String()
|
||||
if _, ok := f.blobIDs[idStr]; !ok {
|
||||
// Look for short ID form
|
||||
if _, ok := f.blobIDs[id.Str()]; !ok {
|
||||
if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
|
||||
continue
|
||||
}
|
||||
// Replace the short ID with the long one
|
||||
f.blobIDs[idStr] = struct{}{}
|
||||
delete(f.blobIDs, id.Str())
|
||||
delete(f.blobIDs, idStr[:shortStr])
|
||||
}
|
||||
f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,8 +402,6 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
||||
})
|
||||
}
|
||||
|
||||
var errAllPacksFound = errors.New("all packs found")
|
||||
|
||||
// packsToBlobs converts the list of pack IDs to a list of blob IDs that
|
||||
// belong to those packs.
|
||||
func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
||||
@@ -413,18 +413,20 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
||||
f.blobIDs = make(map[string]struct{})
|
||||
}
|
||||
|
||||
allPacksFound := false
|
||||
packsFound := 0
|
||||
|
||||
debug.Log("Looking for packs...")
|
||||
err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
if allPacksFound {
|
||||
return nil
|
||||
}
|
||||
idStr := id.String()
|
||||
if _, ok := packIDs[idStr]; !ok {
|
||||
// Look for short ID form
|
||||
if _, ok := packIDs[id.Str()]; !ok {
|
||||
if _, ok := packIDs[idStr[:shortStr]]; !ok {
|
||||
return nil
|
||||
}
|
||||
delete(packIDs, id.Str())
|
||||
} else {
|
||||
// forget found id
|
||||
delete(packIDs, idStr)
|
||||
}
|
||||
debug.Log("Found pack %s", idStr)
|
||||
blobs, _, err := f.repo.ListPack(ctx, id, size)
|
||||
@@ -435,75 +437,25 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
||||
f.blobIDs[b.ID.String()] = struct{}{}
|
||||
}
|
||||
// Stop searching when all packs have been found
|
||||
if len(packIDs) == 0 {
|
||||
return errAllPacksFound
|
||||
packsFound++
|
||||
if packsFound >= len(packIDs) {
|
||||
allPacksFound = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil && err != errAllPacksFound {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != errAllPacksFound {
|
||||
// try to resolve unknown pack ids from the index
|
||||
packIDs = f.indexPacksToBlobs(ctx, packIDs)
|
||||
}
|
||||
|
||||
if len(packIDs) > 0 {
|
||||
list := make([]string, 0, len(packIDs))
|
||||
for h := range packIDs {
|
||||
list = append(list, h)
|
||||
}
|
||||
|
||||
sort.Strings(list)
|
||||
return errors.Fatalf("unable to find pack(s): %v", list)
|
||||
if !allPacksFound {
|
||||
return errors.Fatal("unable to find all specified pack(s)")
|
||||
}
|
||||
|
||||
debug.Log("%d blobs found", len(f.blobIDs))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
|
||||
wctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
// remember which packs were found in the index
|
||||
indexPackIDs := make(map[string]struct{})
|
||||
for pb := range f.repo.Index().Each(wctx) {
|
||||
idStr := pb.PackID.String()
|
||||
// keep entry in packIDs as Each() returns individual index entries
|
||||
matchingID := false
|
||||
if _, ok := packIDs[idStr]; ok {
|
||||
matchingID = true
|
||||
} else {
|
||||
if _, ok := packIDs[pb.PackID.Str()]; ok {
|
||||
// expand id
|
||||
delete(packIDs, pb.PackID.Str())
|
||||
packIDs[idStr] = struct{}{}
|
||||
matchingID = true
|
||||
}
|
||||
}
|
||||
if matchingID {
|
||||
f.blobIDs[pb.ID.String()] = struct{}{}
|
||||
indexPackIDs[idStr] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
for id := range indexPackIDs {
|
||||
delete(packIDs, id)
|
||||
}
|
||||
|
||||
if len(indexPackIDs) > 0 {
|
||||
list := make([]string, 0, len(indexPackIDs))
|
||||
for h := range indexPackIDs {
|
||||
list = append(list, h)
|
||||
}
|
||||
Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
|
||||
}
|
||||
return packIDs
|
||||
}
|
||||
|
||||
func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) {
|
||||
idx := f.repo.Index()
|
||||
|
||||
@@ -513,7 +465,7 @@ func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobTyp
|
||||
return
|
||||
}
|
||||
|
||||
blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t})
|
||||
blobs := idx.Lookup(rid, t)
|
||||
if len(blobs) == 0 {
|
||||
Printf("Object %s not found in the index\n", rid.Str())
|
||||
return
|
||||
@@ -612,10 +564,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
|
||||
if opts.PackID {
|
||||
err := f.packsToBlobs(ctx, f.pat.pattern)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
|
||||
}
|
||||
|
||||
for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
|
||||
|
||||
@@ -31,19 +31,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||
|
||||
// ForgetOptions collects all options for the forget command.
|
||||
type ForgetOptions struct {
|
||||
Last int
|
||||
Hourly int
|
||||
Daily int
|
||||
Weekly int
|
||||
Monthly int
|
||||
Yearly int
|
||||
Within restic.Duration
|
||||
WithinHourly restic.Duration
|
||||
WithinDaily restic.Duration
|
||||
WithinWeekly restic.Duration
|
||||
WithinMonthly restic.Duration
|
||||
WithinYearly restic.Duration
|
||||
KeepTags restic.TagLists
|
||||
Last int
|
||||
Hourly int
|
||||
Daily int
|
||||
Weekly int
|
||||
Monthly int
|
||||
Yearly int
|
||||
Within restic.Duration
|
||||
KeepTags restic.TagLists
|
||||
|
||||
Hosts []string
|
||||
Tags restic.TagLists
|
||||
@@ -69,20 +64,11 @@ func init() {
|
||||
f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
|
||||
f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
|
||||
f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||
f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||
f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||
f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||
f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||
f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||
|
||||
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
|
||||
f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
|
||||
f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
|
||||
err := f.MarkDeprecated("hostname", "use --host")
|
||||
if err != nil {
|
||||
// MarkDeprecated only returns an error when the flag is not found
|
||||
panic(err)
|
||||
}
|
||||
f.MarkDeprecated("hostname", "use --host")
|
||||
|
||||
f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
|
||||
|
||||
@@ -94,15 +80,9 @@ func init() {
|
||||
f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
|
||||
|
||||
f.SortFlags = false
|
||||
addPruneOptions(cmdForget)
|
||||
}
|
||||
|
||||
func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
|
||||
err := verifyPruneOptions(&pruneOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
repo, err := OpenRepository(gopts)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -138,19 +118,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
|
||||
policy := restic.ExpirePolicy{
|
||||
Last: opts.Last,
|
||||
Hourly: opts.Hourly,
|
||||
Daily: opts.Daily,
|
||||
Weekly: opts.Weekly,
|
||||
Monthly: opts.Monthly,
|
||||
Yearly: opts.Yearly,
|
||||
Within: opts.Within,
|
||||
WithinHourly: opts.WithinHourly,
|
||||
WithinDaily: opts.WithinDaily,
|
||||
WithinWeekly: opts.WithinWeekly,
|
||||
WithinMonthly: opts.WithinMonthly,
|
||||
WithinYearly: opts.WithinYearly,
|
||||
Tags: opts.KeepTags,
|
||||
Last: opts.Last,
|
||||
Hourly: opts.Hourly,
|
||||
Daily: opts.Daily,
|
||||
Weekly: opts.Weekly,
|
||||
Monthly: opts.Monthly,
|
||||
Yearly: opts.Yearly,
|
||||
Within: opts.Within,
|
||||
Tags: opts.KeepTags,
|
||||
}
|
||||
|
||||
if policy.Empty() && len(args) == 0 {
|
||||
@@ -229,12 +204,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if len(removeSnIDs) > 0 && opts.Prune {
|
||||
if !gopts.JSON {
|
||||
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
|
||||
}
|
||||
pruneOptions.DryRun = opts.DryRun
|
||||
return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
|
||||
if len(removeSnIDs) > 0 && opts.Prune && !opts.DryRun {
|
||||
return pruneRepository(gopts, repo)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
|
||||
var cmdGenerate = &cobra.Command{
|
||||
Use: "generate [flags]",
|
||||
Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
|
||||
Short: "Generate manual pages and auto-completion files (bash, zsh)",
|
||||
Long: `
|
||||
The "generate" command writes automatically generated files (like the man pages
|
||||
and the auto-completion files for bash, fish and zsh).
|
||||
and the auto-completion files for bash and zsh).
|
||||
|
||||
EXIT STATUS
|
||||
===========
|
||||
@@ -27,7 +27,6 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||
type generateOptions struct {
|
||||
ManDir string
|
||||
BashCompletionFile string
|
||||
FishCompletionFile string
|
||||
ZSHCompletionFile string
|
||||
}
|
||||
|
||||
@@ -38,7 +37,6 @@ func init() {
|
||||
fs := cmdGenerate.Flags()
|
||||
fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
|
||||
fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
|
||||
fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
|
||||
fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
|
||||
}
|
||||
|
||||
@@ -65,11 +63,6 @@ func writeBashCompletion(file string) error {
|
||||
return cmdRoot.GenBashCompletionFile(file)
|
||||
}
|
||||
|
||||
func writeFishCompletion(file string) error {
|
||||
Verbosef("writing fish completion file to %v\n", file)
|
||||
return cmdRoot.GenFishCompletionFile(file, true)
|
||||
}
|
||||
|
||||
func writeZSHCompletion(file string) error {
|
||||
Verbosef("writing zsh completion file to %v\n", file)
|
||||
return cmdRoot.GenZshCompletionFile(file)
|
||||
@@ -90,13 +83,6 @@ func runGenerate(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if genOpts.FishCompletionFile != "" {
|
||||
err := writeFishCompletion(genOpts.FishCompletionFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if genOpts.ZSHCompletionFile != "" {
|
||||
err := writeZSHCompletion(genOpts.ZSHCompletionFile)
|
||||
if err != nil {
|
||||
|
||||
@@ -43,6 +43,10 @@ func init() {
|
||||
}
|
||||
|
||||
func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
|
||||
if gopts.Repo == "" {
|
||||
return errors.Fatal("Please specify repository location (-r)")
|
||||
}
|
||||
|
||||
chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -53,6 +57,11 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
be, err := create(repo, gopts.extended)
|
||||
if err != nil {
|
||||
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
|
||||
}
|
||||
|
||||
gopts.password, err = ReadPasswordTwice(gopts,
|
||||
"enter password for new repository: ",
|
||||
"enter password again: ")
|
||||
@@ -60,11 +69,6 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
be, err := create(repo, gopts.extended)
|
||||
if err != nil {
|
||||
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
|
||||
}
|
||||
|
||||
s := repository.New(be)
|
||||
|
||||
err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
|
||||
|
||||
@@ -60,7 +60,8 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
|
||||
case "locks":
|
||||
t = restic.LockFile
|
||||
case "blobs":
|
||||
return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
|
||||
return repo.List(opts.ctx, restic.IndexFile, func(id restic.ID, size int64) error {
|
||||
idx, err := repository.LoadIndex(opts.ctx, repo, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -69,6 +70,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
default:
|
||||
return errors.Fatal("invalid type")
|
||||
}
|
||||
|
||||
@@ -74,42 +74,18 @@ type lsSnapshot struct {
|
||||
StructType string `json:"struct_type"` // "snapshot"
|
||||
}
|
||||
|
||||
// Print node in our custom JSON format, followed by a newline.
|
||||
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
|
||||
n := &struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
UID uint32 `json:"uid"`
|
||||
GID uint32 `json:"gid"`
|
||||
Size *uint64 `json:"size,omitempty"`
|
||||
Mode os.FileMode `json:"mode,omitempty"`
|
||||
ModTime time.Time `json:"mtime,omitempty"`
|
||||
AccessTime time.Time `json:"atime,omitempty"`
|
||||
ChangeTime time.Time `json:"ctime,omitempty"`
|
||||
StructType string `json:"struct_type"` // "node"
|
||||
|
||||
size uint64 // Target for Size pointer.
|
||||
}{
|
||||
Name: node.Name,
|
||||
Type: node.Type,
|
||||
Path: path,
|
||||
UID: node.UID,
|
||||
GID: node.GID,
|
||||
size: node.Size,
|
||||
Mode: node.Mode,
|
||||
ModTime: node.ModTime,
|
||||
AccessTime: node.AccessTime,
|
||||
ChangeTime: node.ChangeTime,
|
||||
StructType: "node",
|
||||
}
|
||||
// Always print size for regular files, even when empty,
|
||||
// but never for other types.
|
||||
if node.Type == "file" {
|
||||
n.Size = &n.size
|
||||
}
|
||||
|
||||
return enc.Encode(n)
|
||||
type lsNode struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
UID uint32 `json:"uid"`
|
||||
GID uint32 `json:"gid"`
|
||||
Size uint64 `json:"size,omitempty"`
|
||||
Mode os.FileMode `json:"mode,omitempty"`
|
||||
ModTime time.Time `json:"mtime,omitempty"`
|
||||
AccessTime time.Time `json:"atime,omitempty"`
|
||||
ChangeTime time.Time `json:"ctime,omitempty"`
|
||||
StructType string `json:"struct_type"` // "node"
|
||||
}
|
||||
|
||||
func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
|
||||
@@ -183,22 +159,28 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
|
||||
enc := json.NewEncoder(gopts.stdout)
|
||||
|
||||
printSnapshot = func(sn *restic.Snapshot) {
|
||||
err := enc.Encode(lsSnapshot{
|
||||
enc.Encode(lsSnapshot{
|
||||
Snapshot: sn,
|
||||
ID: sn.ID(),
|
||||
ShortID: sn.ID().Str(),
|
||||
StructType: "snapshot",
|
||||
})
|
||||
if err != nil {
|
||||
Warnf("JSON encode failed: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
printNode = func(path string, node *restic.Node) {
|
||||
err := lsNodeJSON(enc, path, node)
|
||||
if err != nil {
|
||||
Warnf("JSON encode failed: %v\n", err)
|
||||
}
|
||||
enc.Encode(lsNode{
|
||||
Name: node.Name,
|
||||
Type: node.Type,
|
||||
Path: path,
|
||||
UID: node.UID,
|
||||
GID: node.GID,
|
||||
Size: node.Size,
|
||||
Mode: node.Mode,
|
||||
ModTime: node.ModTime,
|
||||
AccessTime: node.AccessTime,
|
||||
ChangeTime: node.ChangeTime,
|
||||
StructType: "node",
|
||||
})
|
||||
}
|
||||
} else {
|
||||
printSnapshot = func(sn *restic.Snapshot) {
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestLsNodeJSON(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
path string
|
||||
restic.Node
|
||||
expect string
|
||||
}{
|
||||
// Mode is omitted when zero.
|
||||
{
|
||||
path: "/bar/baz",
|
||||
Node: restic.Node{
|
||||
Name: "baz",
|
||||
Type: "file",
|
||||
Size: 12345,
|
||||
UID: 10000000,
|
||||
GID: 20000000,
|
||||
|
||||
User: "nobody",
|
||||
Group: "nobodies",
|
||||
Links: 1,
|
||||
},
|
||||
expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
|
||||
},
|
||||
|
||||
// Even empty files get an explicit size.
|
||||
{
|
||||
path: "/foo/empty",
|
||||
Node: restic.Node{
|
||||
Name: "empty",
|
||||
Type: "file",
|
||||
Size: 0,
|
||||
UID: 1001,
|
||||
GID: 1001,
|
||||
|
||||
User: "not printed",
|
||||
Group: "not printed",
|
||||
Links: 0xF00,
|
||||
},
|
||||
expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
|
||||
},
|
||||
|
||||
// Non-regular files do not get a size.
|
||||
// Mode is printed in decimal, including the type bits.
|
||||
{
|
||||
path: "/foo/link",
|
||||
Node: restic.Node{
|
||||
Name: "link",
|
||||
Type: "symlink",
|
||||
Mode: os.ModeSymlink | 0777,
|
||||
LinkTarget: "not printed",
|
||||
},
|
||||
expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
|
||||
},
|
||||
|
||||
{
|
||||
path: "/some/directory",
|
||||
Node: restic.Node{
|
||||
Name: "directory",
|
||||
Type: "dir",
|
||||
Mode: os.ModeDir | 0755,
|
||||
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
|
||||
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
|
||||
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
|
||||
},
|
||||
expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
|
||||
},
|
||||
} {
|
||||
buf := new(bytes.Buffer)
|
||||
enc := json.NewEncoder(buf)
|
||||
err := lsNodeJSON(enc, c.path, &c.Node)
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, c.expect+"\n", buf.String())
|
||||
|
||||
// Sanity check: output must be valid JSON.
|
||||
var v interface{}
|
||||
err = json.NewDecoder(buf).Decode(&v)
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
}
|
||||
@@ -116,13 +116,15 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
|
||||
mountpoint := args[0]
|
||||
|
||||
if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) {
|
||||
Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
|
||||
return err
|
||||
Verbosef("Mountpoint %s doesn't exist, creating it\n", mountpoint)
|
||||
err = resticfs.Mkdir(mountpoint, os.ModeDir|0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
mountOptions := []systemFuse.MountOption{
|
||||
systemFuse.ReadOnly(),
|
||||
systemFuse.FSName("restic"),
|
||||
systemFuse.MaxReadahead(128 * 1024),
|
||||
}
|
||||
|
||||
if opts.AllowOther {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user