amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-08-06 21:53:18 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #898 from restic/prepare-cloud-backends

Browse Source 
Prepare more cloud backends, add backend layouts
This commit is contained in:
Alexander Neumann
2017-04-15 11:30:25 +02:00
parent e73038cbcb be06983c80
commit 525db875b0
38 changed files with 1791 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cmds/restic/cmd_options.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"fmt"
"restic/options"
"github.com/spf13/cobra"
)
var optionsCmd = &cobra.Command{
Use: "options",
Short: "print list of extended options",
Long: `
The "options" command prints a list of extended options.
`,
Hidden: true,
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("All Extended Options:\n")
for _, opt := range options.List() {
fmt.Printf(" %-15s %s\n", opt.Namespace+"."+opt.Name, opt.Text)
}
},
}
func init() {
cmdRoot.AddCommand(optionsCmd)
}
src/cmds/restic/global.go
+2 -2
View File
@@ -388,7 +388,7 @@ func open(s string, opts options.Options) (restic.Backend, error) {
case "local":
be, err = local.Open(cfg.(local.Config))
case "sftp":
be, err = sftp.OpenWithConfig(cfg.(sftp.Config))
be, err = sftp.Open(cfg.(sftp.Config))
case "s3":
be, err = s3.Open(cfg.(s3.Config))
case "rest":
@@ -422,7 +422,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
case "local":
return local.Create(cfg.(local.Config))
case "sftp":
return sftp.CreateWithConfig(cfg.(sftp.Config))
return sftp.Create(cfg.(sftp.Config))
case "s3":
return s3.Open(cfg.(s3.Config))
case "rest":
src/cmds/restic/global_debug.go
+13
View File
@@ -8,6 +8,7 @@ import (
_ "net/http/pprof"
"os"
"restic/errors"
"restic/repository"
"github.com/pkg/profile"
)
@@ -16,6 +17,7 @@ var (
listenMemoryProfile string
memProfilePath string
cpuProfilePath string
insecure bool
prof interface {
Stop()
@@ -27,6 +29,13 @@ func init() {
f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
f.BoolVar(&insecure, "insecure-kdf", false, "use insecure KDF settings")
}
type fakeTestingTB struct{}
func (fakeTestingTB) Logf(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg, args...)
}
func runDebug() error {
@@ -50,6 +59,10 @@ func runDebug() error {
prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath))
}
if insecure {
repository.TestUseLowSecurityKDFParameters(fakeTestingTB{})
}
return nil
}
src/cmds/restic/integration_helpers_test.go
+2
View File
@@ -9,6 +9,7 @@ import (
"runtime"
"testing"
"restic/options"
"restic/repository"
. "restic/test"
)
@@ -199,6 +200,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
password: TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,
extended: make(options.Options),
}
// always overwrite global options
src/cmds/restic/local_layout_test.go
+41
View File
@@ -0,0 +1,41 @@
package main
import (
"path/filepath"
. "restic/test"
"testing"
)
func TestRestoreLocalLayout(t *testing.T) {
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
var tests = []struct {
filename string
layout string
}{
{"repo-layout-cloud.tar.gz", ""},
{"repo-layout-local.tar.gz", ""},
{"repo-layout-s3-old.tar.gz", ""},
{"repo-layout-cloud.tar.gz", "cloud"},
{"repo-layout-local.tar.gz", "default"},
{"repo-layout-s3-old.tar.gz", "s3"},
}
for _, test := range tests {
datafile := filepath.Join("..", "..", "restic", "backend", "testdata", test.filename)
SetupTarTestFixture(t, env.base, datafile)
gopts.extended["local.layout"] = test.layout
// check the repo
testRunCheck(t, gopts)
// restore latest snapshot
target := filepath.Join(env.base, "restore")
testRunRestoreLatest(t, gopts, target, nil, "")
RemoveAll(t, filepath.Join(env.base, "repo"))
RemoveAll(t, target)
}
})
}
src/restic/backend/layout.go
+221
View File
@@ -0,0 +1,221 @@
package backend
import (
"fmt"
"os"
"path/filepath"
"regexp"
"restic"
"restic/debug"
"restic/errors"
"restic/fs"
)
// Layout computes paths for file name storage.
type Layout interface {
Filename(restic.Handle) string
Dirname(restic.Handle) string
Basedir(restic.FileType) string
Paths() []string
}
// Filesystem is the abstraction of a file system used for a backend.
type Filesystem interface {
Join(...string) string
ReadDir(string) ([]os.FileInfo, error)
IsNotExist(error) bool
}
// ensure statically that *LocalFilesystem implements Filesystem.
var _ Filesystem = &LocalFilesystem{}
// LocalFilesystem implements Filesystem in a local path.
type LocalFilesystem struct {
}
// ReadDir returns all entries of a directory.
func (l *LocalFilesystem) ReadDir(dir string) ([]os.FileInfo, error) {
f, err := fs.Open(dir)
if err != nil {
return nil, err
}
entries, err := f.Readdir(-1)
if err != nil {
return nil, errors.Wrap(err, "Readdir")
}
err = f.Close()
if err != nil {
return nil, errors.Wrap(err, "Close")
}
return entries, nil
}
// Join combines several path components to one.
func (l *LocalFilesystem) Join(paths ...string) string {
return filepath.Join(paths...)
}
// IsNotExist returns true for errors that are caused by not existing files.
func (l *LocalFilesystem) IsNotExist(err error) bool {
return os.IsNotExist(err)
}
var backendFilenameLength = len(restic.ID{}) * 2
var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength))
func hasBackendFile(fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(dir)
if err != nil && fs.IsNotExist(errors.Cause(err)) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "ReadDir")
}
for _, e := range entries {
if backendFilename.MatchString(e.Name()) {
return true, nil
}
}
return false, nil
}
var dataSubdirName = regexp.MustCompile("^[a-fA-F0-9]{2}$")
func hasSubdirBackendFile(fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(dir)
if err != nil && fs.IsNotExist(errors.Cause(err)) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "ReadDir")
}
for _, subdir := range entries {
if !dataSubdirName.MatchString(subdir.Name()) {
continue
}
present, err := hasBackendFile(fs, fs.Join(dir, subdir.Name()))
if err != nil {
return false, err
}
if present {
return true, nil
}
}
return false, nil
}
// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
// cannot be detected automatically.
var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
// DetectLayout tries to find out which layout is used in a local (or sftp)
// filesystem at the given path. If repo is nil, an instance of LocalFilesystem
// is used.
func DetectLayout(repo Filesystem, dir string) (Layout, error) {
debug.Log("detect layout at %v", dir)
if repo == nil {
repo = &LocalFilesystem{}
}
// key file in the "keys" dir (DefaultLayout or CloudLayout)
foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3Layout)
foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// data file in "data" directory (S3Layout or CloudLayout)
foundDataFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
if err != nil {
return nil, err
}
// data file in subdir of "data" directory (DefaultLayout)
foundDataSubdirFile, err := hasSubdirBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
if err != nil {
return nil, err
}
if foundKeysFile && foundDataFile && !foundKeyFile && !foundDataSubdirFile {
debug.Log("found cloud layout at %v", dir)
return &CloudLayout{
Path: dir,
Join: repo.Join,
}, nil
}
if foundKeysFile && foundDataSubdirFile && !foundKeyFile && !foundDataFile {
debug.Log("found default layout at %v", dir)
return &DefaultLayout{
Path: dir,
Join: repo.Join,
}, nil
}
if foundKeyFile && foundDataFile && !foundKeysFile && !foundDataSubdirFile {
debug.Log("found s3 layout at %v", dir)
return &S3Layout{
Path: dir,
Join: repo.Join,
}, nil
}
debug.Log("layout detection failed")
return nil, ErrLayoutDetectionFailed
}
// ParseLayout parses the config string and returns a Layout. When layout is
// the empty string, DetectLayout is used. If that fails, defaultLayout is used.
func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
debug.Log("parse layout string %q for backend at %v", layout, path)
switch layout {
case "default":
l = &DefaultLayout{
Path: path,
Join: repo.Join,
}
case "cloud":
l = &CloudLayout{
Path: path,
Join: repo.Join,
}
case "s3":
l = &S3Layout{
Path: path,
Join: repo.Join,
}
case "":
l, err = DetectLayout(repo, path)
// use the default layout if auto detection failed
if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
debug.Log("error: %v, use default layout %v", defaultLayout)
return ParseLayout(repo, defaultLayout, "", path)
}
if err != nil {
return nil, err
}
debug.Log("layout detected: %v", l)
default:
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, cloud, s3", layout)
}
return l, nil
}
src/restic/backend/layout_cloud.go
+46
View File
@@ -0,0 +1,46 @@
package backend
import "restic"
// CloudLayout implements the default layout for cloud storage backends, as
// described in the Design document.
type CloudLayout struct {
URL string
Path string
Join func(...string) string
}
var cloudLayoutPaths = defaultLayoutPaths
// Dirname returns the directory path for a given file type and name.
func (l *CloudLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *CloudLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *CloudLayout) Paths() (dirs []string) {
for _, p := range cloudLayoutPaths {
dirs = append(dirs, l.URL+l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for files of type t.
func (l *CloudLayout) Basedir(t restic.FileType) string {
return l.URL + l.Join(l.Path, cloudLayoutPaths[t])
}
src/restic/backend/layout_default.go
+54
View File
@@ -0,0 +1,54 @@
package backend
import "restic"
// DefaultLayout implements the default layout for local and sftp backends, as
// described in the Design document. The `data` directory has one level of
// subdirs, two characters each (taken from the first two characters of the
// file name).
type DefaultLayout struct {
Path string
Join func(...string) string
}
var defaultLayoutPaths = map[restic.FileType]string{
restic.DataFile: "data",
restic.SnapshotFile: "snapshots",
restic.IndexFile: "index",
restic.LockFile: "locks",
restic.KeyFile: "keys",
}
// Dirname returns the directory path for a given file type and name.
func (l *DefaultLayout) Dirname(h restic.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == restic.DataFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2])
}
return l.Join(l.Path, p)
}
// Filename returns a path to a file, including its name.
func (l *DefaultLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.Join(l.Dirname(h), name)
}
// Paths returns all directory names
func (l *DefaultLayout) Paths() (dirs []string) {
for _, p := range defaultLayoutPaths {
dirs = append(dirs, l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *DefaultLayout) Basedir(t restic.FileType) string {
return l.Join(l.Path, defaultLayoutPaths[t])
}
src/restic/backend/layout_s3.go
+52
View File
@@ -0,0 +1,52 @@
package backend
import "restic"
// S3Layout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
type S3Layout struct {
URL string
Path string
Join func(...string) string
}
var s3LayoutPaths = map[restic.FileType]string{
restic.DataFile: "data",
restic.SnapshotFile: "snapshot",
restic.IndexFile: "index",
restic.LockFile: "lock",
restic.KeyFile: "key",
}
// Dirname returns the directory path for a given file type and name.
func (l *S3Layout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *S3Layout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *S3Layout) Paths() (dirs []string) {
for _, p := range s3LayoutPaths {
dirs = append(dirs, l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *S3Layout) Basedir(t restic.FileType) string {
return l.Join(l.Path, s3LayoutPaths[t])
}
src/restic/backend/layout_test.go
+374
View File
@@ -0,0 +1,374 @@
package backend
import (
"fmt"
"path"
"path/filepath"
"reflect"
"restic"
. "restic/test"
"sort"
"testing"
)
func TestDefaultLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
restic.Handle
filename string
}{
{
restic.Handle{Type: restic.DataFile, Name: "0123456"},
filepath.Join(path, "data", "01", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshots", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
filepath.Join(path, "locks", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
filepath.Join(path, "keys", "123456"),
},
}
l := &DefaultLayout{
Path: path,
Join: filepath.Join,
}
t.Run("Paths", func(t *testing.T) {
dirs := l.Paths()
want := []string{
filepath.Join(path, "data"),
filepath.Join(path, "snapshots"),
filepath.Join(path, "index"),
filepath.Join(path, "locks"),
filepath.Join(path, "keys"),
}
sort.Sort(sort.StringSlice(want))
sort.Sort(sort.StringSlice(dirs))
if !reflect.DeepEqual(dirs, want) {
t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs)
}
})
for _, test := range tests {
t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
filename := l.Filename(test.Handle)
if filename != test.filename {
t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
}
})
}
}
func TestCloudLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
restic.Handle
filename string
}{
{
restic.Handle{Type: restic.DataFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshots", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
filepath.Join(path, "locks", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
filepath.Join(path, "keys", "123456"),
},
}
l := &CloudLayout{
Path: path,
Join: filepath.Join,
}
t.Run("Paths", func(t *testing.T) {
dirs := l.Paths()
want := []string{
filepath.Join(path, "data"),
filepath.Join(path, "snapshots"),
filepath.Join(path, "index"),
filepath.Join(path, "locks"),
filepath.Join(path, "keys"),
}
sort.Sort(sort.StringSlice(want))
sort.Sort(sort.StringSlice(dirs))
if !reflect.DeepEqual(dirs, want) {
t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs)
}
})
for _, test := range tests {
t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
filename := l.Filename(test.Handle)
if filename != test.filename {
t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
}
})
}
}
func TestCloudLayoutURLs(t *testing.T) {
var tests = []struct {
l Layout
h restic.Handle
fn string
dir string
}{
{
&CloudLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3Layout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%T", test.l), func(t *testing.T) {
fn := test.l.Filename(test.h)
if fn != test.fn {
t.Fatalf("wrong filename, want %v, got %v", test.fn, fn)
}
dir := test.l.Dirname(test.h)
if dir != test.dir {
t.Fatalf("wrong dirname, want %v, got %v", test.dir, dir)
}
})
}
}
func TestS3Layout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
restic.Handle
filename string
}{
{
restic.Handle{Type: restic.DataFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"),
},
}
l := &S3Layout{
Path: path,
Join: filepath.Join,
}
t.Run("Paths", func(t *testing.T) {
dirs := l.Paths()
want := []string{
filepath.Join(path, "data"),
filepath.Join(path, "snapshot"),
filepath.Join(path, "index"),
filepath.Join(path, "lock"),
filepath.Join(path, "key"),
}
sort.Sort(sort.StringSlice(want))
sort.Sort(sort.StringSlice(dirs))
if !reflect.DeepEqual(dirs, want) {
t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs)
}
})
for _, test := range tests {
t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
filename := l.Filename(test.Handle)
if filename != test.filename {
t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
}
})
}
}
func TestDetectLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
filename string
want string
}{
{"repo-layout-local.tar.gz", "*backend.DefaultLayout"},
{"repo-layout-cloud.tar.gz", "*backend.CloudLayout"},
{"repo-layout-s3-old.tar.gz", "*backend.S3Layout"},
}
var fs = &LocalFilesystem{}
for _, test := range tests {
for _, fs := range []Filesystem{fs, nil} {
t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename))
layout, err := DetectLayout(fs, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
if layout == nil {
t.Fatal("wanted some layout, but detect returned nil")
}
layoutName := fmt.Sprintf("%T", layout)
if layoutName != test.want {
t.Fatalf("want layout %v, got %v", test.want, layoutName)
}
RemoveAll(t, filepath.Join(path, "repo"))
})
}
}
}
func TestParseLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
layoutName string
defaultLayoutName string
want string
}{
{"default", "", "*backend.DefaultLayout"},
{"cloud", "", "*backend.CloudLayout"},
{"s3", "", "*backend.S3Layout"},
{"", "", "*backend.CloudLayout"},
}
SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-cloud.tar.gz"))
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {
layout, err := ParseLayout(&LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
if layout == nil {
t.Fatal("wanted some layout, but detect returned nil")
}
// test that the functions work (and don't panic)
_ = layout.Dirname(restic.Handle{Type: restic.DataFile})
_ = layout.Filename(restic.Handle{Type: restic.DataFile, Name: "1234"})
_ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout)
if layoutName != test.want {
t.Fatalf("want layout %v, got %v", test.want, layoutName)
}
})
}
}
func TestParseLayoutInvalid(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var invalidNames = []string{
"foo", "bar", "local",
}
for _, name := range invalidNames {
t.Run(name, func(t *testing.T) {
layout, err := ParseLayout(nil, name, "", path)
if err == nil {
t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout)
}
})
}
}
src/restic/backend/local/config.go
+7 -1
View File
@@ -4,11 +4,17 @@ import (
"strings"
"restic/errors"
"restic/options"
)
// Config holds all information needed to open a local repository.
type Config struct {
Path string
Path string
Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
}
func init() {
options.Register("local", Config{})
}
// ParseConfig parses a local backend config.
src/restic/backend/local/layout_test.go
+84
View File
@@ -0,0 +1,84 @@
package local
import (
"path/filepath"
"restic"
. "restic/test"
"testing"
)
func TestLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
filename string
layout string
failureExpected bool
datafiles map[string]bool
}{
{"repo-layout-local.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
}
for _, test := range tests {
t.Run(test.filename, func(t *testing.T) {
SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
repo := filepath.Join(path, "repo")
be, err := Open(Config{
Path: repo,
Layout: test.layout,
})
if err != nil {
t.Fatal(err)
}
if be == nil {
t.Fatalf("Open() returned nil but no error")
}
datafiles := make(map[string]bool)
for id := range be.List(restic.DataFile, nil) {
datafiles[id] = false
}
if len(datafiles) == 0 {
t.Errorf("List() returned zero data files")
}
for id := range test.datafiles {
if _, ok := datafiles[id]; !ok {
t.Errorf("datafile with id %v not found", id)
}
datafiles[id] = true
}
for id, v := range datafiles {
if !v {
t.Errorf("unexpected id %v found", id)
}
}
if err = be.Close(); err != nil {
t.Errorf("Close() returned error %v", err)
}
RemoveAll(t, filepath.Join(path, "repo"))
})
}
}
src/restic/backend/local/local.go
+62 -163
View File
@@ -2,7 +2,6 @@ package local
import (
"io"
"io/ioutil"
"os"
"path/filepath"
"restic"
@@ -17,53 +16,64 @@ import (
// Local is a backend in a local directory.
type Local struct {
Config
backend.Layout
}
// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}
func paths(dir string) []string {
return []string{
dir,
filepath.Join(dir, backend.Paths.Data),
filepath.Join(dir, backend.Paths.Snapshots),
filepath.Join(dir, backend.Paths.Index),
filepath.Join(dir, backend.Paths.Locks),
filepath.Join(dir, backend.Paths.Keys),
filepath.Join(dir, backend.Paths.Temp),
}
}
const defaultLayout = "default"
// Open opens the local backend as specified by config.
func Open(cfg Config) (*Local, error) {
debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
be := &Local{Config: cfg, Layout: l}
// test if all necessary dirs are there
for _, d := range paths(cfg.Path) {
for _, d := range be.Paths() {
if _, err := fs.Stat(d); err != nil {
return nil, errors.Wrap(err, "Open")
}
}
return &Local{Config: cfg}, nil
return be, nil
}
// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(cfg Config) (*Local, error) {
debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
be := &Local{
Config: cfg,
Layout: l,
}
// test if config file already exists
_, err := fs.Lstat(filepath.Join(cfg.Path, backend.Paths.Config))
_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
// create paths for data, refs and temp
for _, d := range paths(cfg.Path) {
for _, d := range be.Paths() {
err := fs.MkdirAll(d, backend.Modes.Dir)
if err != nil {
return nil, errors.Wrap(err, "MkdirAll")
}
}
// open backend
return Open(cfg)
return be, nil
}
// Location returns this backend's location (the directory name).
@@ -71,60 +81,6 @@ func (b *Local) Location() string {
return b.Path
}
// Construct path for given Type and name.
func filename(base string, t restic.FileType, name string) string {
if t == restic.ConfigFile {
return filepath.Join(base, "config")
}
return filepath.Join(dirname(base, t, name), name)
}
// Construct directory for given Type.
func dirname(base string, t restic.FileType, name string) string {
var n string
switch t {
case restic.DataFile:
n = backend.Paths.Data
if len(name) > 2 {
n = filepath.Join(n, name[:2])
}
case restic.SnapshotFile:
n = backend.Paths.Snapshots
case restic.IndexFile:
n = backend.Paths.Index
case restic.LockFile:
n = backend.Paths.Locks
case restic.KeyFile:
n = backend.Paths.Keys
}
return filepath.Join(base, n)
}
// copyToTempfile saves p into a tempfile in tempdir.
func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
tmpfile, err := ioutil.TempFile(tempdir, "temp-")
if err != nil {
return "", errors.Wrap(err, "TempFile")
}
_, err = io.Copy(tmpfile, rd)
if err != nil {
return "", errors.Wrap(err, "Write")
}
if err = tmpfile.Sync(); err != nil {
return "", errors.Wrap(err, "Syncn")
}
err = tmpfile.Close()
if err != nil {
return "", errors.Wrap(err, "Close")
}
return tmpfile.Name(), nil
}
// Save stores data in the backend at the handle.
func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
debug.Log("Save %v", h)
@@ -132,18 +88,7 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
return err
}
tmpfile, err := copyToTempfile(filepath.Join(b.Path, backend.Paths.Temp), rd)
debug.Log("saved %v to %v", h, tmpfile)
if err != nil {
return err
}
filename := filename(b.Path, h.Type, h.Name)
// test if new path already exists
if _, err := fs.Stat(filename); err == nil {
return errors.Errorf("Rename(): file %v already exists", filename)
}
filename := b.Filename(h)
// create directories if necessary, ignore errors
if h.Type == restic.DataFile {
@@ -153,12 +98,27 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
}
}
err = fs.Rename(tmpfile, filename)
debug.Log("save %v: rename %v -> %v: %v",
h, filepath.Base(tmpfile), filepath.Base(filename), err)
// create new file
f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
if err != nil {
return errors.Wrap(err, "Rename")
return errors.Wrap(err, "OpenFile")
}
// save data, then sync
_, err = io.Copy(f, rd)
if err != nil {
f.Close()
return errors.Wrap(err, "Write")
}
if err = f.Sync(); err != nil {
f.Close()
return errors.Wrap(err, "Sync")
}
err = f.Close()
if err != nil {
return errors.Wrap(err, "Close")
}
// set mode to read-only
@@ -183,7 +143,7 @@ func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser,
return nil, errors.New("offset is negative")
}
f, err := os.Open(filename(b.Path, h.Type, h.Name))
f, err := fs.Open(b.Filename(h))
if err != nil {
return nil, err
}
@@ -210,7 +170,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, err
}
fi, err := fs.Stat(filename(b.Path, h.Type, h.Name))
fi, err := fs.Stat(b.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
}
@@ -221,7 +181,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(h restic.Handle) (bool, error) {
debug.Log("Test %v", h)
_, err := fs.Stat(filename(b.Path, h.Type, h.Name))
_, err := fs.Stat(b.Filename(h))
if err != nil {
if os.IsNotExist(errors.Cause(err)) {
return false, nil
@@ -235,7 +195,7 @@ func (b *Local) Test(h restic.Handle) (bool, error) {
// Remove removes the blob with the given name and type.
func (b *Local) Remove(h restic.Handle) error {
debug.Log("Remove %v", h)
fn := filename(b.Path, h.Type, h.Name)
fn := b.Filename(h)
// reset read-only flag
err := fs.Chmod(fn, 0666)
@@ -250,91 +210,30 @@ func isFile(fi os.FileInfo) bool {
return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}
func readdir(d string) (fileInfos []os.FileInfo, err error) {
f, e := fs.Open(d)
if e != nil {
return nil, errors.Wrap(e, "Open")
}
defer func() {
e := f.Close()
if err == nil {
err = errors.Wrap(e, "Close")
}
}()
return f.Readdir(-1)
}
// listDir returns a list of all files in d.
func listDir(d string) (filenames []string, err error) {
fileInfos, err := readdir(d)
if err != nil {
return nil, err
}
for _, fi := range fileInfos {
if isFile(fi) {
filenames = append(filenames, fi.Name())
}
}
return filenames, nil
}
// listDirs returns a list of all files in directories within d.
func listDirs(dir string) (filenames []string, err error) {
fileInfos, err := readdir(dir)
if err != nil {
return nil, err
}
for _, fi := range fileInfos {
if !fi.IsDir() {
continue
}
files, err := listDir(filepath.Join(dir, fi.Name()))
if err != nil {
continue
}
filenames = append(filenames, files...)
}
return filenames, nil
}
// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string {
debug.Log("List %v", t)
lister := listDir
if t == restic.DataFile {
lister = listDirs
}
ch := make(chan string)
items, err := lister(filepath.Join(dirname(b.Path, t, "")))
if err != nil {
close(ch)
return ch
}
go func() {
defer close(ch)
for _, m := range items {
if m == "" {
continue
fs.Walk(b.Basedir(t), func(path string, fi os.FileInfo, err error) error {
if !isFile(fi) {
return err
}
select {
case ch <- m:
case ch <- filepath.Base(path):
case <-done:
return
return err
}
}
return err
})
}()
return ch
src/restic/backend/location/location_test.go
+4 -4
View File
@@ -79,7 +79,7 @@ var parseTests = []struct {
Config: sftp.Config{
User: "user",
Host: "host",
Dir: "/srv/repo",
Path: "/srv/repo",
},
},
},
@@ -89,7 +89,7 @@ var parseTests = []struct {
Config: sftp.Config{
User: "",
Host: "host",
Dir: "/srv/repo",
Path: "/srv/repo",
},
},
},
@@ -99,7 +99,7 @@ var parseTests = []struct {
Config: sftp.Config{
User: "user",
Host: "host",
Dir: "srv/repo",
Path: "srv/repo",
},
},
},
@@ -109,7 +109,7 @@ var parseTests = []struct {
Config: sftp.Config{
User: "user",
Host: "host",
Dir: "/srv/repo",
Path: "/srv/repo",
},
},
},
src/restic/backend/rest/rest.go
+20 -35
View File
@@ -22,39 +22,11 @@ const connLimit = 40
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &restBackend{}
// restPath returns the path to the given resource.
func restPath(url *url.URL, h restic.Handle) string {
u := *url
var dir string
switch h.Type {
case restic.ConfigFile:
dir = ""
h.Name = "config"
case restic.DataFile:
dir = backend.Paths.Data
case restic.SnapshotFile:
dir = backend.Paths.Snapshots
case restic.IndexFile:
dir = backend.Paths.Index
case restic.LockFile:
dir = backend.Paths.Locks
case restic.KeyFile:
dir = backend.Paths.Keys
default:
dir = string(h.Type)
}
u.Path = path.Join(url.Path, dir, h.Name)
return u.String()
}
type restBackend struct {
url *url.URL
connChan chan struct{}
client http.Client
backend.Layout
}
// Open opens the REST backend with the given config.
@@ -66,7 +38,20 @@ func Open(cfg Config) (restic.Backend, error) {
tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
client := http.Client{Transport: tr}
return &restBackend{url: cfg.URL, connChan: connChan, client: client}, nil
// use url without trailing slash for layout
url := cfg.URL.String()
if url[len(url)-1] == '/' {
url = url[:len(url)-1]
}
be := &restBackend{
url: cfg.URL,
connChan: connChan,
client: client,
Layout: &backend.CloudLayout{URL: url, Join: path.Join},
}
return be, nil
}
// Create creates a new REST on server configured in config.
@@ -124,7 +109,7 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
rd = backend.Closer{Reader: rd}
<-b.connChan
resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
resp, err := b.client.Post(b.Filename(h), "binary/octet-stream", rd)
b.connChan <- struct{}{}
if resp != nil {
@@ -167,7 +152,7 @@ func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCl
return nil, errors.Errorf("invalid length %d", length)
}
req, err := http.NewRequest("GET", restPath(b.url, h), nil)
req, err := http.NewRequest("GET", b.Filename(h), nil)
if err != nil {
return nil, errors.Wrap(err, "http.NewRequest")
}
@@ -207,7 +192,7 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
}
<-b.connChan
resp, err := b.client.Head(restPath(b.url, h))
resp, err := b.client.Head(b.Filename(h))
b.connChan <- struct{}{}
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.Head")
@@ -249,7 +234,7 @@ func (b *restBackend) Remove(h restic.Handle) error {
return err
}
req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
req, err := http.NewRequest("DELETE", b.Filename(h), nil)
if err != nil {
return errors.Wrap(err, "http.NewRequest")
}
@@ -275,7 +260,7 @@ func (b *restBackend) Remove(h restic.Handle) error {
func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan string {
ch := make(chan string)
url := restPath(b.url, restic.Handle{Type: t})
url := b.Dirname(restic.Handle{Type: t})
if !strings.HasSuffix(url, "/") {
url += "/"
}
src/restic/backend/rest/rest_path_test.go
-48
View File
@@ -1,48 +0,0 @@
package rest
import (
"net/url"
"restic"
"testing"
)
var restPathTests = []struct {
Handle restic.Handle
URL *url.URL
Result string
}{
{
URL: parseURL("https://hostname.foo"),
Handle: restic.Handle{
Type: restic.DataFile,
Name: "foobar",
},
Result: "https://hostname.foo/data/foobar",
},
{
URL: parseURL("https://hostname.foo:1234/prefix/repo"),
Handle: restic.Handle{
Type: restic.LockFile,
Name: "foobar",
},
Result: "https://hostname.foo:1234/prefix/repo/locks/foobar",
},
{
URL: parseURL("https://hostname.foo:1234/prefix/repo"),
Handle: restic.Handle{
Type: restic.ConfigFile,
Name: "foobar",
},
Result: "https://hostname.foo:1234/prefix/repo/config",
},
}
func TestRESTPaths(t *testing.T) {
for i, test := range restPathTests {
result := restPath(test.URL, test.Handle)
if result != test.Result {
t.Errorf("test %d: resulting URL does not match, want:\n %#v\ngot: \n %#v",
i, test.Result, result)
}
}
}
src/restic/backend/s3/s3.go
+8 -13
View File
@@ -27,6 +27,7 @@ type s3 struct {
prefix string
cacheMutex sync.RWMutex
cacheObjSize map[string]int64
backend.Layout
}
// Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -44,6 +45,7 @@ func Open(cfg Config) (restic.Backend, error) {
bucketname: cfg.Bucket,
prefix: cfg.Prefix,
cacheObjSize: make(map[string]int64),
Layout: &backend.S3Layout{URL: cfg.Endpoint, Join: path.Join},
}
tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
@@ -68,13 +70,6 @@ func Open(cfg Config) (restic.Backend, error) {
return be, nil
}
func (be *s3) s3path(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return path.Join(be.prefix, string(h.Type))
}
return path.Join(be.prefix, string(h.Type), h.Name)
}
func (be *s3) createConnections() {
be.connChan = make(chan struct{}, connLimit)
for i := 0; i < connLimit; i++ {
@@ -95,7 +90,7 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
debug.Log("Save %v", h)
objName := be.s3path(h)
objName := be.Filename(h)
// Check key does not already exist
_, err = be.client.StatObject(be.bucketname, objName)
@@ -149,7 +144,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
var obj *minio.Object
var size int64
objName := be.s3path(h)
objName := be.Filename(h)
// get token for connection
<-be.connChan
@@ -242,7 +237,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
debug.Log("%v", h)
objName := be.s3path(h)
objName := be.Filename(h)
var obj *minio.Object
obj, err = be.client.GetObject(be.bucketname, objName)
@@ -271,7 +266,7 @@ func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
// Test returns true if a blob of the given type and name exists in the backend.
func (be *s3) Test(h restic.Handle) (bool, error) {
found := false
objName := be.s3path(h)
objName := be.Filename(h)
_, err := be.client.StatObject(be.bucketname, objName)
if err == nil {
found = true
@@ -283,7 +278,7 @@ func (be *s3) Test(h restic.Handle) (bool, error) {
// Remove removes the blob with the given name and type.
func (be *s3) Remove(h restic.Handle) error {
objName := be.s3path(h)
objName := be.Filename(h)
err := be.client.RemoveObject(be.bucketname, objName)
debug.Log("Remove(%v) -> err %v", h, err)
return errors.Wrap(err, "client.RemoveObject")
@@ -296,7 +291,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
debug.Log("listing %v", t)
ch := make(chan string)
prefix := be.s3path(restic.Handle{Type: t}) + "/"
prefix := be.Dirname(restic.Handle{Type: t})
listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
src/restic/backend/sftp/config.go
+9 -2
View File
@@ -6,11 +6,18 @@ import (
"strings"
"restic/errors"
"restic/options"
)
// Config collects all information required to connect to an sftp server.
type Config struct {
User, Host, Dir string
User, Host, Path string
Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
Command string `option:"command" help:"specify command to create sftp connection"`
}
func init() {
options.Register("sftp", Config{})
}
// ParseConfig parses the string s and extracts the sftp config. The
@@ -60,6 +67,6 @@ func ParseConfig(s string) (interface{}, error) {
return Config{
User: user,
Host: host,
Dir: path.Clean(dir),
Path: path.Clean(dir),
}, nil
}
src/restic/backend/sftp/config_test.go
+12 -12
View File
@@ -9,53 +9,53 @@ var configTests = []struct {
// first form, user specified sftp://user@host/dir
{
"sftp://user@host/dir/subdir",
Config{User: "user", Host: "host", Dir: "dir/subdir"},
Config{User: "user", Host: "host", Path: "dir/subdir"},
},
{
"sftp://host/dir/subdir",
Config{Host: "host", Dir: "dir/subdir"},
Config{Host: "host", Path: "dir/subdir"},
},
{
"sftp://host//dir/subdir",
Config{Host: "host", Dir: "/dir/subdir"},
Config{Host: "host", Path: "/dir/subdir"},
},
{
"sftp://host:10022//dir/subdir",
Config{Host: "host:10022", Dir: "/dir/subdir"},
Config{Host: "host:10022", Path: "/dir/subdir"},
},
{
"sftp://user@host:10022//dir/subdir",
Config{User: "user", Host: "host:10022", Dir: "/dir/subdir"},
Config{User: "user", Host: "host:10022", Path: "/dir/subdir"},
},
{
"sftp://user@host/dir/subdir/../other",
Config{User: "user", Host: "host", Dir: "dir/other"},
Config{User: "user", Host: "host", Path: "dir/other"},
},
{
"sftp://user@host/dir///subdir",
Config{User: "user", Host: "host", Dir: "dir/subdir"},
Config{User: "user", Host: "host", Path: "dir/subdir"},
},
// second form, user specified sftp:user@host:/dir
{
"sftp:user@host:/dir/subdir",
Config{User: "user", Host: "host", Dir: "/dir/subdir"},
Config{User: "user", Host: "host", Path: "/dir/subdir"},
},
{
"sftp:host:../dir/subdir",
Config{Host: "host", Dir: "../dir/subdir"},
Config{Host: "host", Path: "../dir/subdir"},
},
{
"sftp:user@host:dir/subdir:suffix",
Config{User: "user", Host: "host", Dir: "dir/subdir:suffix"},
Config{User: "user", Host: "host", Path: "dir/subdir:suffix"},
},
{
"sftp:user@host:dir/subdir/../other",
Config{User: "user", Host: "host", Dir: "dir/other"},
Config{User: "user", Host: "host", Path: "dir/other"},
},
{
"sftp:user@host:dir///subdir",
Config{User: "user", Host: "host", Dir: "dir/subdir"},
Config{User: "user", Host: "host", Path: "dir/subdir"},
},
}
src/restic/backend/sftp/layout_test.go
+91
View File
@@ -0,0 +1,91 @@
package sftp_test
import (
"fmt"
"path/filepath"
"restic"
"restic/backend/sftp"
. "restic/test"
"testing"
)
func TestLayout(t *testing.T) {
if sftpserver == "" {
t.Skip("sftp server binary not available")
}
path, cleanup := TempDir(t)
defer cleanup()
var tests = []struct {
filename string
layout string
failureExpected bool
datafiles map[string]bool
}{
{"repo-layout-local.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
}
for _, test := range tests {
t.Run(test.filename, func(t *testing.T) {
SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
repo := filepath.Join(path, "repo")
be, err := sftp.Open(sftp.Config{
Command: fmt.Sprintf("%q -e", sftpserver),
Path: repo,
Layout: test.layout,
})
if err != nil {
t.Fatal(err)
}
if be == nil {
t.Fatalf("Open() returned nil but no error")
}
datafiles := make(map[string]bool)
for id := range be.List(restic.DataFile, nil) {
datafiles[id] = false
}
if len(datafiles) == 0 {
t.Errorf("List() returned zero data files")
}
for id := range test.datafiles {
if _, ok := datafiles[id]; !ok {
t.Errorf("datafile with id %v not found", id)
}
datafiles[id] = true
}
for id, v := range datafiles {
if !v {
t.Errorf("unexpected id %v found", id)
}
}
if err = be.Close(); err != nil {
t.Errorf("Close() returned error %v", err)
}
RemoveAll(t, filepath.Join(path, "repo"))
})
}
}
src/restic/backend/sftp/sftp.go
+124 -206
View File
@@ -2,13 +2,12 @@ package sftp
import (
"bufio"
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"os"
"os/exec"
"path"
"path/filepath"
"restic"
"strings"
"time"
@@ -21,10 +20,6 @@ import (
"github.com/pkg/sftp"
)
const (
tempfileRandomSuffixLength = 10
)
// SFTP is a backend in a directory accessed via SFTP.
type SFTP struct {
c *sftp.Client
@@ -32,11 +27,17 @@ type SFTP struct {
cmd *exec.Cmd
result <-chan error
backend.Layout
Config
}
var _ restic.Backend = &SFTP{}
const defaultLayout = "default"
func startClient(program string, args ...string) (*SFTP, error) {
debug.Log("start client %v %v", program, args)
// Connect to a remote host and request the sftp subsystem via the 'ssh'
// command. This assumes that passwordless login is correctly configured.
cmd := exec.Command(program, args...)
@@ -89,18 +90,6 @@ func startClient(program string, args ...string) (*SFTP, error) {
return &SFTP{c: client, cmd: cmd, result: ch}, nil
}
func paths(dir string) []string {
return []string{
dir,
Join(dir, backend.Paths.Data),
Join(dir, backend.Paths.Snapshots),
Join(dir, backend.Paths.Index),
Join(dir, backend.Paths.Locks),
Join(dir, backend.Paths.Keys),
Join(dir, backend.Paths.Temp),
}
}
// clientError returns an error if the client has exited. Otherwise, nil is
// returned immediately.
func (r *SFTP) clientError() error {
@@ -114,32 +103,70 @@ func (r *SFTP) clientError() error {
return nil
}
// Open opens an sftp backend. When the command is started via
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
// is expected at the given path. `dir` must be delimited by forward slashes
// ("/"), which is required by sftp.
func Open(dir string, program string, args ...string) (*SFTP, error) {
debug.Log("open backend with program %v, %v at %v", program, args, dir)
sftp, err := startClient(program, args...)
// Open opens an sftp backend as described by the config by running
// "ssh" with the appropriate arguments (or cfg.Command, if set).
func Open(cfg Config) (*SFTP, error) {
debug.Log("open backend with config %#v", cfg)
cmd, args, err := buildSSHCommand(cfg)
if err != nil {
return nil, err
}
sftp, err := startClient(cmd, args...)
if err != nil {
debug.Log("unable to start program: %v", err)
return nil, err
}
sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
// test if all necessary dirs and files are there
for _, d := range paths(dir) {
for _, d := range sftp.Paths() {
if _, err := sftp.c.Lstat(d); err != nil {
return nil, errors.Errorf("%s does not exist", d)
}
}
sftp.p = dir
debug.Log("layout: %v\n", sftp.Layout)
sftp.Config = cfg
sftp.p = cfg.Path
return sftp, nil
}
func buildSSHCommand(cfg Config) []string {
// Join combines path components with slashes (according to the sftp spec).
func (r *SFTP) Join(p ...string) string {
return path.Join(p...)
}
// ReadDir returns the entries for a directory.
func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) {
return r.c.ReadDir(dir)
}
// IsNotExist returns true if the error is caused by a not existing file.
func (r *SFTP) IsNotExist(err error) bool {
statusError, ok := err.(*sftp.StatusError)
if !ok {
return false
}
return statusError.Error() == `sftp: "No such file" (SSH_FX_NO_SUCH_FILE)`
}
func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
if cfg.Command != "" {
return SplitShellArgs(cfg.Command)
}
cmd = "ssh"
hostport := strings.Split(cfg.Host, ":")
args := []string{hostport[0]}
args = []string{hostport[0]}
if len(hostport) > 1 {
args = append(args, "-p", hostport[1])
}
@@ -149,35 +176,38 @@ func buildSSHCommand(cfg Config) []string {
}
args = append(args, "-s")
args = append(args, "sftp")
return args
return cmd, args, nil
}
// OpenWithConfig opens an sftp backend as described by the config by running
// "ssh" with the appropriate arguments.
func OpenWithConfig(cfg Config) (*SFTP, error) {
debug.Log("open with config %v", cfg)
return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}
// Create creates an sftp backend as described by the config by running
// "ssh" with the appropriate arguments (or cfg.Command, if set).
func Create(cfg Config) (*SFTP, error) {
cmd, args, err := buildSSHCommand(cfg)
if err != nil {
return nil, err
}
// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created. `dir` must
// be delimited by forward slashes ("/"), which is required by sftp.
func Create(dir string, program string, args ...string) (*SFTP, error) {
debug.Log("%v %v", program, args)
sftp, err := startClient(program, args...)
sftp, err := startClient(cmd, args...)
if err != nil {
debug.Log("unable to start program: %v", err)
return nil, err
}
sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
// test if config file already exists
_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
_, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config))
if err == nil {
return nil, errors.New("config file already exists")
}
// create paths for data, refs and temp blobs
for _, d := range paths(dir) {
for _, d := range sftp.Paths() {
err = sftp.mkdirAll(d, backend.Modes.Dir)
debug.Log("mkdirAll %v -> %v", d, err)
if err != nil {
return nil, err
}
@@ -189,14 +219,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) {
}
// open backend
return Open(dir, program, args...)
}
// CreateWithConfig creates an sftp backend as described by the config by running
// "ssh" with the appropriate arguments.
func CreateWithConfig(cfg Config) (*SFTP, error) {
debug.Log("config %v", cfg)
return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
return Open(cfg)
}
// Location returns this backend's location (the directory name).
@@ -204,28 +227,6 @@ func (r *SFTP) Location() string {
return r.p
}
// Return temp directory in correct directory for this backend.
func (r *SFTP) tempFile() (string, *sftp.File, error) {
// choose random suffix
buf := make([]byte, tempfileRandomSuffixLength)
_, err := io.ReadFull(rand.Reader, buf)
if err != nil {
return "", nil, errors.Errorf("unable to read %d random bytes for tempfile name: %v",
tempfileRandomSuffixLength, err)
}
// construct tempfile name
name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))
// create file in temp dir
f, err := r.c.Create(name)
if err != nil {
return "", nil, errors.Errorf("creating tempfile %q failed: %v", name, err)
}
return name, f, nil
}
func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
// check if directory already exists
fi, err := r.c.Lstat(dir)
@@ -258,9 +259,24 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
return r.c.Chmod(dir, mode)
}
// Rename temp file to final name according to type and name.
func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
filename := r.filename(h)
// Join joins the given paths and cleans them afterwards. This always uses
// forward slashes, which is required by sftp.
func Join(parts ...string) string {
return path.Clean(path.Join(parts...))
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
debug.Log("Save %v", h)
if err := r.clientError(); err != nil {
return err
}
if err := h.Valid(); err != nil {
return err
}
filename := r.Filename(h)
// create directories if necessary
if h.Type == restic.DataFile {
@@ -270,14 +286,22 @@ func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
}
}
// test if new file exists
if _, err := r.c.Lstat(filename); err == nil {
return errors.Errorf("Close(): file %v already exists", filename)
// create new file
f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
if err != nil {
return errors.Wrap(err, "OpenFile")
}
err := r.c.Rename(oldname, filename)
// save data
_, err = io.Copy(f, rd)
if err != nil {
return errors.Wrap(err, "Rename")
f.Close()
return errors.Wrap(err, "Write")
}
err = f.Close()
if err != nil {
return errors.Wrap(err, "Close")
}
// set mode to read-only
@@ -290,76 +314,6 @@ func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
return errors.Wrap(err, "Chmod")
}
// Join joins the given paths and cleans them afterwards. This always uses
// forward slashes, which is required by sftp.
func Join(parts ...string) string {
return path.Clean(path.Join(parts...))
}
// Construct path for given restic.Type and name.
func (r *SFTP) filename(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return Join(r.p, "config")
}
return Join(r.dirname(h), h.Name)
}
// Construct directory for given backend.Type.
func (r *SFTP) dirname(h restic.Handle) string {
var n string
switch h.Type {
case restic.DataFile:
n = backend.Paths.Data
if len(h.Name) > 2 {
n = Join(n, h.Name[:2])
}
case restic.SnapshotFile:
n = backend.Paths.Snapshots
case restic.IndexFile:
n = backend.Paths.Index
case restic.LockFile:
n = backend.Paths.Locks
case restic.KeyFile:
n = backend.Paths.Keys
}
return Join(r.p, n)
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
debug.Log("save to %v", h)
if err := r.clientError(); err != nil {
return err
}
if err := h.Valid(); err != nil {
return err
}
filename, tmpfile, err := r.tempFile()
if err != nil {
return err
}
n, err := io.Copy(tmpfile, rd)
if err != nil {
return errors.Wrap(err, "Write")
}
debug.Log("saved %v (%d bytes) to %v", h, n, filename)
err = tmpfile.Close()
if err != nil {
return errors.Wrap(err, "Close")
}
err = r.renameFile(filename, h)
debug.Log("save %v: rename %v: %v",
h, path.Base(filename), err)
return err
}
// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
@@ -373,7 +327,7 @@ func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, e
return nil, errors.New("offset is negative")
}
f, err := r.c.Open(r.filename(h))
f, err := r.c.Open(r.Filename(h))
if err != nil {
return nil, err
}
@@ -404,7 +358,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, err
}
fi, err := r.c.Lstat(r.filename(h))
fi, err := r.c.Lstat(r.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Lstat")
}
@@ -419,7 +373,7 @@ func (r *SFTP) Test(h restic.Handle) (bool, error) {
return false, err
}
_, err := r.c.Lstat(r.filename(h))
_, err := r.c.Lstat(r.Filename(h))
if os.IsNotExist(errors.Cause(err)) {
return false, nil
}
@@ -438,71 +392,35 @@ func (r *SFTP) Remove(h restic.Handle) error {
return err
}
return r.c.Remove(r.filename(h))
return r.c.Remove(r.Filename(h))
}
// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
debug.Log("list all %v", t)
debug.Log("List %v", t)
ch := make(chan string)
go func() {
defer close(ch)
if t == restic.DataFile {
// read first level
basedir := r.dirname(restic.Handle{Type: t})
walker := r.c.Walk(r.Basedir(t))
for walker.Step() {
if walker.Err() != nil {
continue
}
list1, err := r.c.ReadDir(basedir)
if err != nil {
if !walker.Stat().Mode().IsRegular() {
continue
}
select {
case ch <- filepath.Base(walker.Path()):
case <-done:
return
}
dirs := make([]string, 0, len(list1))
for _, d := range list1 {
dirs = append(dirs, d.Name())
}
// read files
for _, dir := range dirs {
entries, err := r.c.ReadDir(Join(basedir, dir))
if err != nil {
continue
}
items := make([]string, 0, len(entries))
for _, entry := range entries {
items = append(items, entry.Name())
}
for _, file := range items {
select {
case ch <- file:
case <-done:
return
}
}
}
} else {
entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t}))
if err != nil {
return
}
items := make([]string, 0, len(entries))
for _, entry := range entries {
items = append(items, entry.Name())
}
for _, file := range items {
select {
case ch <- file:
case <-done:
return
}
}
}
}()
src/restic/backend/sftp/sftp_backend_test.go
+19 -8
View File
@@ -1,6 +1,7 @@
package sftp_test
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -33,24 +34,29 @@ func createTempdir() error {
return nil
}
func init() {
sftpserver := ""
func findSFTPServerBinary() string {
for _, dir := range strings.Split(TestSFTPPath, ":") {
testpath := filepath.Join(dir, "sftp-server")
_, err := os.Stat(testpath)
if !os.IsNotExist(errors.Cause(err)) {
sftpserver = testpath
break
return testpath
}
}
return ""
}
var sftpserver = findSFTPServerBinary()
func init() {
if sftpserver == "" {
SkipMessage = "sftp server binary not found, skipping tests"
return
}
args := []string{"-e"}
cfg := sftp.Config{
Command: fmt.Sprintf("%q -e", sftpserver),
}
test.CreateFn = func() (restic.Backend, error) {
err := createTempdir()
@@ -58,7 +64,9 @@ func init() {
return nil, err
}
return sftp.Create(tempBackendDir, sftpserver, args...)
cfg.Path = tempBackendDir
return sftp.Create(cfg)
}
test.OpenFn = func() (restic.Backend, error) {
@@ -66,7 +74,10 @@ func init() {
if err != nil {
return nil, err
}
return sftp.Open(tempBackendDir, sftpserver, args...)
cfg.Path = tempBackendDir
return sftp.Open(cfg)
}
test.CleanupFn = func() error {
src/restic/backend/sftp/split.go
+77
View File
@@ -0,0 +1,77 @@
package sftp
import (
"restic/errors"
"unicode"
)
// shellSplitter splits a command string into separater arguments. It supports
// single and double quoted strings.
type shellSplitter struct {
quote rune
lastChar rune
}
func (s *shellSplitter) isSplitChar(c rune) bool {
// only test for quotes if the last char was not a backslash
if s.lastChar != '\\' {
// quote ended
if s.quote != 0 && c == s.quote {
s.quote = 0
return true
}
// quote starts
if s.quote == 0 && (c == '"' || c == '\'') {
s.quote = c
return true
}
}
s.lastChar = c
// within quote
if s.quote != 0 {
return false
}
// outside quote
return c == '\\' || unicode.IsSpace(c)
}
// SplitShellArgs returns the list of arguments from a shell command string.
func SplitShellArgs(data string) (cmd string, args []string, err error) {
s := &shellSplitter{}
// derived from strings.SplitFunc
fieldStart := -1 // Set to -1 when looking for start of field.
for i, rune := range data {
if s.isSplitChar(rune) {
if fieldStart >= 0 {
args = append(args, data[fieldStart:i])
fieldStart = -1
}
} else if fieldStart == -1 {
fieldStart = i
}
}
if fieldStart >= 0 { // Last field might end at EOF.
args = append(args, data[fieldStart:])
}
switch s.quote {
case '\'':
return "", nil, errors.New("single-quoted string not terminated")
case '"':
return "", nil, errors.New("double-quoted string not terminated")
}
if len(args) == 0 {
return "", nil, errors.New("command string is empty")
}
cmd, args = args[0], args[1:]
return cmd, args, nil
}
src/restic/backend/sftp/split_test.go
+115
View File
@@ -0,0 +1,115 @@
package sftp
import (
"reflect"
"testing"
)
func TestShellSplitter(t *testing.T) {
var tests = []struct {
data string
cmd string
args []string
}{
{
`foo`,
"foo", []string{},
},
{
`'foo'`,
"foo", []string{},
},
{
`foo bar baz`,
"foo", []string{"bar", "baz"},
},
{
`foo 'bar' baz`,
"foo", []string{"bar", "baz"},
},
{
`'bar box' baz`,
"bar box", []string{"baz"},
},
{
`"bar 'box'" baz`,
"bar 'box'", []string{"baz"},
},
{
`'bar "box"' baz`,
`bar "box"`, []string{"baz"},
},
{
`\"bar box baz`,
`"bar`, []string{"box", "baz"},
},
{
`"bar/foo/x" "box baz"`,
"bar/foo/x", []string{"box baz"},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
cmd, args, err := SplitShellArgs(test.data)
if err != nil {
t.Fatal(err)
}
if cmd != test.cmd {
t.Fatalf("wrong cmd returned, want:\n %#v\ngot:\n %#v",
test.cmd, cmd)
}
if !reflect.DeepEqual(args, test.args) {
t.Fatalf("wrong args returned, want:\n %#v\ngot:\n %#v",
test.args, args)
}
})
}
}
func TestShellSplitterInvalid(t *testing.T) {
var tests = []struct {
data string
err string
}{
{
"foo'",
"single-quoted string not terminated",
},
{
`foo"`,
"double-quoted string not terminated",
},
{
"foo 'bar",
"single-quoted string not terminated",
},
{
`foo "bar`,
"double-quoted string not terminated",
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
cmd, args, err := SplitShellArgs(test.data)
if err == nil {
t.Fatalf("expected error not found: %v", test.err)
}
if err.Error() != test.err {
t.Fatalf("expected error not found, want:\n %q\ngot:\n %q", test.err, err.Error())
}
if cmd != "" {
t.Fatalf("splitter returned cmd from invalid data: %v", cmd)
}
if len(args) > 0 {
t.Fatalf("splitter returned fields from invalid data: %v", args)
}
})
}
}
src/restic/backend/sftp/sshcmd_test.go
+29 -23
View File
@@ -1,46 +1,52 @@
package sftp
import "testing"
import (
"reflect"
"testing"
)
var sshcmdTests = []struct {
cfg Config
s []string
cfg Config
cmd string
args []string
}{
{
Config{User: "user", Host: "host", Dir: "dir/subdir"},
Config{User: "user", Host: "host", Path: "dir/subdir"},
"ssh",
[]string{"host", "-l", "user", "-s", "sftp"},
},
{
Config{Host: "host", Dir: "dir/subdir"},
Config{Host: "host", Path: "dir/subdir"},
"ssh",
[]string{"host", "-s", "sftp"},
},
{
Config{Host: "host:10022", Dir: "/dir/subdir"},
Config{Host: "host:10022", Path: "/dir/subdir"},
"ssh",
[]string{"host", "-p", "10022", "-s", "sftp"},
},
{
Config{User: "user", Host: "host:10022", Dir: "/dir/subdir"},
Config{User: "user", Host: "host:10022", Path: "/dir/subdir"},
"ssh",
[]string{"host", "-p", "10022", "-l", "user", "-s", "sftp"},
},
}
func TestBuildSSHCommand(t *testing.T) {
for i, test := range sshcmdTests {
cmd := buildSSHCommand(test.cfg)
failed := false
if len(cmd) != len(test.s) {
failed = true
} else {
for l := range test.s {
if test.s[l] != cmd[l] {
failed = true
break
}
for _, test := range sshcmdTests {
t.Run("", func(t *testing.T) {
cmd, args, err := buildSSHCommand(test.cfg)
if err != nil {
t.Fatal(err)
}
}
if failed {
t.Errorf("test %d: wrong cmd, want:\n %v\ngot:\n %v",
i, test.s, cmd)
}
if cmd != test.cmd {
t.Fatalf("cmd: want %v, got %v", test.cmd, cmd)
}
if !reflect.DeepEqual(test.args, args) {
t.Fatalf("wrong args, want:\n %v\ngot:\n %v", test.args, args)
}
})
}
}
src/restic/backend/test/tests.go
+23 -22
View File
@@ -43,7 +43,7 @@ func open(t testing.TB) restic.Backend {
if !butInitialized {
be, err := CreateFn()
if err != nil {
t.Fatalf("Create returned unexpected error: %v", err)
t.Fatalf("Create returned unexpected error: %+v", err)
}
but = be
@@ -54,7 +54,7 @@ func open(t testing.TB) restic.Backend {
var err error
but, err = OpenFn()
if err != nil {
t.Fatalf("Open returned unexpected error: %v", err)
t.Fatalf("Open returned unexpected error: %+v", err)
}
}
@@ -68,7 +68,7 @@ func close(t testing.TB) {
err := but.Close()
if err != nil {
t.Fatalf("Close returned unexpected error: %v", err)
t.Fatalf("Close returned unexpected error: %+v", err)
}
but = nil
@@ -82,14 +82,14 @@ func TestCreate(t testing.TB) {
be, err := CreateFn()
if err != nil {
t.Fatalf("Create returned error: %v", err)
t.Fatalf("Create returned error: %+v", err)
}
butInitialized = true
err = be.Close()
if err != nil {
t.Fatalf("Close returned error: %v", err)
t.Fatalf("Close returned error: %+v", err)
}
}
@@ -101,12 +101,12 @@ func TestOpen(t testing.TB) {
be, err := OpenFn()
if err != nil {
t.Fatalf("Open returned error: %v", err)
t.Fatalf("Open returned error: %+v", err)
}
err = be.Close()
if err != nil {
t.Fatalf("Close returned error: %v", err)
t.Fatalf("Close returned error: %+v", err)
}
}
@@ -132,7 +132,7 @@ func TestCreateWithConfig(t testing.TB) {
// remove config
err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""})
if err != nil {
t.Fatalf("unexpected error removing config: %v", err)
t.Fatalf("unexpected error removing config: %+v", err)
}
}
@@ -162,7 +162,7 @@ func TestConfig(t testing.TB) {
err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
if err != nil {
t.Fatalf("Save() error: %v", err)
t.Fatalf("Save() error: %+v", err)
}
// try accessing the config with different names, should all return the
@@ -171,7 +171,7 @@ func TestConfig(t testing.TB) {
h := restic.Handle{Type: restic.ConfigFile, Name: name}
buf, err := backend.LoadAll(b, h)
if err != nil {
t.Fatalf("unable to read config with name %q: %v", name, err)
t.Fatalf("unable to read config with name %q: %+v", name, err)
}
if string(buf) != testString {
@@ -203,7 +203,7 @@ func TestLoad(t testing.TB) {
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
err = b.Save(handle, bytes.NewReader(data))
if err != nil {
t.Fatalf("Save() error: %v", err)
t.Fatalf("Save() error: %+v", err)
}
rd, err := b.Load(handle, 100, -1)
@@ -238,13 +238,13 @@ func TestLoad(t testing.TB) {
rd, err := b.Load(handle, getlen, int64(o))
if err != nil {
t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err)
t.Errorf("Load(%d, %d) returned unexpected error: %+v", l, o, err)
continue
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %+v", l, o, err)
rd.Close()
continue
}
@@ -269,7 +269,7 @@ func TestLoad(t testing.TB) {
err = rd.Close()
if err != nil {
t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %+v", l, o, err)
continue
}
}
@@ -325,7 +325,7 @@ func TestSave(t testing.TB) {
err = b.Remove(h)
if err != nil {
t.Fatalf("error removing item: %v", err)
t.Fatalf("error removing item: %+v", err)
}
}
@@ -366,7 +366,7 @@ func TestSave(t testing.TB) {
err = b.Remove(h)
if err != nil {
t.Fatalf("error removing item: %v", err)
t.Fatalf("error removing item: %+v", err)
}
}
@@ -391,13 +391,13 @@ func TestSaveFilenames(t testing.TB) {
h := restic.Handle{Name: test.name, Type: restic.DataFile}
err := b.Save(h, strings.NewReader(test.data))
if err != nil {
t.Errorf("test %d failed: Save() returned %v", i, err)
t.Errorf("test %d failed: Save() returned %+v", i, err)
continue
}
buf, err := backend.LoadAll(b, h)
if err != nil {
t.Errorf("test %d failed: Load() returned %v", i, err)
t.Errorf("test %d failed: Load() returned %+v", i, err)
continue
}
@@ -407,7 +407,7 @@ func TestSaveFilenames(t testing.TB) {
err = b.Remove(h)
if err != nil {
t.Errorf("test %d failed: Remove() returned %v", i, err)
t.Errorf("test %d failed: Remove() returned %+v", i, err)
continue
}
}
@@ -511,7 +511,7 @@ func TestBackend(t testing.TB) {
// test that the blob is gone
ok, err := b.Test(h)
test.OK(t, err)
test.Assert(t, ok == false, "removed blob still present")
test.Assert(t, !ok, "removed blob still present")
// create blob
err = b.Save(h, strings.NewReader(ts.data))
@@ -553,6 +553,7 @@ func TestBackend(t testing.TB) {
found, err := b.Test(h)
test.OK(t, err)
test.Assert(t, found, fmt.Sprintf("id %q not found", id))
test.OK(t, b.Remove(h))
@@ -576,7 +577,7 @@ func TestDelete(t testing.TB) {
err := be.Delete()
if err != nil {
t.Fatalf("error deleting backend: %v", err)
t.Fatalf("error deleting backend: %+v", err)
}
}
@@ -594,6 +595,6 @@ func TestCleanup(t testing.TB) {
err := CleanupFn()
if err != nil {
t.Fatalf("Cleanup returned error: %v", err)
t.Fatalf("Cleanup returned error: %+v", err)
}
}
src/restic/backend/testdata/repo-layout-cloud.tar.gz
Vendored
BIN
View File
Binary file not shown.
src/restic/backend/testdata/repo-layout-local.tar.gz
Vendored
BIN
View File
Binary file not shown.
src/restic/backend/testdata/repo-layout-s3-old.tar.gz
Vendored
BIN
View File
Binary file not shown.
src/restic/backend/utils_test.go
+2 -4
View File
@@ -49,8 +49,7 @@ func TestLoadSmallBuffer(t *testing.T) {
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
OK(t, err)
buf := make([]byte, len(data)-23)
buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
OK(t, err)
if len(buf) != len(data) {
@@ -75,8 +74,7 @@ func TestLoadLargeBuffer(t *testing.T) {
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
OK(t, err)
buf := make([]byte, len(data)+100)
buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
OK(t, err)
if len(buf) != len(data) {
src/restic/options/options.go
+80
View File
@@ -3,6 +3,7 @@ package options
import (
"reflect"
"restic/errors"
"sort"
"strconv"
"strings"
"time"
@@ -11,6 +12,85 @@ import (
// Options holds options in the form key=value.
type Options map[string]string
var opts []Help
// Register allows registering options so that they can be listed with List.
func Register(ns string, cfg interface{}) {
opts = appendAllOptions(opts, ns, cfg)
}
// List returns a list of all registered options (using Register()).
func List() (list []Help) {
list = make([]Help, 0, len(opts))
for _, opt := range opts {
list = append(list, opt)
}
return list
}
// appendAllOptions appends all options in cfg to opts, sorted by namespace.
func appendAllOptions(opts []Help, ns string, cfg interface{}) []Help {
for _, opt := range listOptions(cfg) {
opt.Namespace = ns
opts = append(opts, opt)
}
sort.Sort(helpList(opts))
return opts
}
// listOptions returns a list of options of cfg.
func listOptions(cfg interface{}) (opts []Help) {
// resolve indirection if cfg is a pointer
v := reflect.Indirect(reflect.ValueOf(cfg))
for i := 0; i < v.NumField(); i++ {
f := v.Type().Field(i)
h := Help{
Name: f.Tag.Get("option"),
Text: f.Tag.Get("help"),
}
if h.Name == "" {
continue
}
opts = append(opts, h)
}
return opts
}
// Help contains information about an option.
type Help struct {
Namespace string
Name string
Text string
}
type helpList []Help
// Len is the number of elements in the collection.
func (h helpList) Len() int {
return len(h)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (h helpList) Less(i, j int) bool {
if h[i].Namespace == h[j].Namespace {
return h[i].Name < h[j].Name
}
return h[i].Namespace < h[j].Namespace
}
// Swap swaps the elements with indexes i and j.
func (h helpList) Swap(i, j int) {
h[i], h[j] = h[j], h[i]
}
// splitKeyValue splits at the first equals (=) sign.
func splitKeyValue(s string) (key string, value string) {
data := strings.SplitN(s, "=", 2)
src/restic/options/options_test.go
+92
View File
@@ -218,3 +218,95 @@ func TestOptionsApplyInvalid(t *testing.T) {
})
}
}
func TestListOptions(t *testing.T) {
var teststruct = struct {
Foo string `option:"foo" help:"bar text help"`
}{}
var tests = []struct {
cfg interface{}
opts []Help
}{
{
struct {
Foo string `option:"foo" help:"bar text help"`
}{},
[]Help{
Help{Name: "foo", Text: "bar text help"},
},
},
{
struct {
Foo string `option:"foo" help:"bar text help"`
Bar string `option:"bar" help:"bar text help"`
}{},
[]Help{
Help{Name: "foo", Text: "bar text help"},
Help{Name: "bar", Text: "bar text help"},
},
},
{
struct {
Bar string `option:"bar" help:"bar text help"`
Foo string `option:"foo" help:"bar text help"`
}{},
[]Help{
Help{Name: "bar", Text: "bar text help"},
Help{Name: "foo", Text: "bar text help"},
},
},
{
&teststruct,
[]Help{
Help{Name: "foo", Text: "bar text help"},
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
opts := listOptions(test.cfg)
if !reflect.DeepEqual(opts, test.opts) {
t.Fatalf("wrong opts, want:\n %v\ngot:\n %v", test.opts, opts)
}
})
}
}
func TestAppendAllOptions(t *testing.T) {
var tests = []struct {
cfgs map[string]interface{}
opts []Help
}{
{
map[string]interface{}{
"local": struct {
Foo string `option:"foo" help:"bar text help"`
}{},
"sftp": struct {
Foo string `option:"foo" help:"bar text help2"`
Bar string `option:"bar" help:"bar text help"`
}{},
},
[]Help{
Help{Namespace: "local", Name: "foo", Text: "bar text help"},
Help{Namespace: "sftp", Name: "bar", Text: "bar text help"},
Help{Namespace: "sftp", Name: "foo", Text: "bar text help2"},
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
var opts []Help
for ns, cfg := range test.cfgs {
opts = appendAllOptions(opts, ns, cfg)
}
if !reflect.DeepEqual(opts, test.opts) {
t.Fatalf("wrong list, want:\n %v\ngot:\n %v", test.opts, opts)
}
})
}
}
src/restic/repository/testing.go
+5 -1
View File
@@ -19,8 +19,12 @@ var testKDFParams = crypto.KDFParams{
P: 1,
}
type logger interface {
Logf(format string, args ...interface{})
}
// TestUseLowSecurityKDFParameters configures low-security KDF parameters for testing.
func TestUseLowSecurityKDFParameters(t testing.TB) {
func TestUseLowSecurityKDFParameters(t logger) {
t.Logf("using low-security KDF parameters for test")
KDFParams = &testKDFParams
}
src/restic/test/helpers.go
+18
View File
@@ -170,3 +170,21 @@ func RemoveAll(t testing.TB, path string) {
ResetReadOnly(t, path)
OK(t, os.RemoveAll(path))
}
// TempDir returns a temporary directory that is removed when cleanup is
// called, except if TestCleanupTempDirs is set to false.
func TempDir(t testing.TB) (path string, cleanup func()) {
tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
if err != nil {
t.Fatal(err)
}
return tempdir, func() {
if !TestCleanupTempDirs {
t.Logf("leaving temporary directory %v used for test", tempdir)
return
}
RemoveAll(t, tempdir)
}
}
src/restic/test/vars.go
+1 -1
View File
@@ -11,7 +11,7 @@ var (
TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "")
RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true)
RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true)
TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh")
TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec")
TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".")
BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".")
TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "")