Merge pull request #966 from restic/unify-repo-layout

WIP: Unify repository layout
This commit is contained in:
Alexander Neumann
2017-05-22 21:15:38 +02:00
17 changed files with 200 additions and 201 deletions
+2
View File
@@ -1,6 +1,7 @@
package main
import (
"io/ioutil"
"testing"
)
@@ -13,6 +14,7 @@ func TestFlags(t *testing.T) {
for _, cmd := range cmdRoot.Commands() {
t.Run(cmd.Name(), func(t *testing.T) {
cmd.Flags().SetOutput(ioutil.Discard)
err := cmd.ParseFlags([]string{"--help"})
if err.Error() == "pflag: help requested" {
err = nil
+4 -6
View File
@@ -12,12 +12,10 @@ func TestRestoreLocalLayout(t *testing.T) {
filename string
layout string
}{
{"repo-layout-cloud.tar.gz", ""},
{"repo-layout-local.tar.gz", ""},
{"repo-layout-s3-old.tar.gz", ""},
{"repo-layout-cloud.tar.gz", "cloud"},
{"repo-layout-local.tar.gz", "default"},
{"repo-layout-s3-old.tar.gz", "s3"},
{"repo-layout-default.tar.gz", ""},
{"repo-layout-s3legacy.tar.gz", ""},
{"repo-layout-default.tar.gz", "default"},
{"repo-layout-s3legacy.tar.gz", "s3legacy"},
}
for _, test := range tests {
+9 -64
View File
@@ -85,36 +85,6 @@ func hasBackendFile(fs Filesystem, dir string) (bool, error) {
return false, nil
}
var dataSubdirName = regexp.MustCompile("^[a-fA-F0-9]{2}$")
func hasSubdirBackendFile(fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(dir)
if err != nil && fs.IsNotExist(errors.Cause(err)) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "ReadDir")
}
for _, subdir := range entries {
if !dataSubdirName.MatchString(subdir.Name()) {
continue
}
present, err := hasBackendFile(fs, fs.Join(dir, subdir.Name()))
if err != nil {
return false, err
}
if present {
return true, nil
}
}
return false, nil
}
// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
// cannot be detected automatically.
var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
@@ -128,39 +98,19 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
repo = &LocalFilesystem{}
}
// key file in the "keys" dir (DefaultLayout or CloudLayout)
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3Layout)
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// data file in "data" directory (S3Layout or CloudLayout)
foundDataFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
if err != nil {
return nil, err
}
// data file in subdir of "data" directory (DefaultLayout)
foundDataSubdirFile, err := hasSubdirBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
if err != nil {
return nil, err
}
if foundKeysFile && foundDataFile && !foundKeyFile && !foundDataSubdirFile {
debug.Log("found cloud layout at %v", dir)
return &CloudLayout{
Path: dir,
Join: repo.Join,
}, nil
}
if foundKeysFile && foundDataSubdirFile && !foundKeyFile && !foundDataFile {
if foundKeysFile && !foundKeyFile {
debug.Log("found default layout at %v", dir)
return &DefaultLayout{
Path: dir,
@@ -168,9 +118,9 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
}, nil
}
if foundKeyFile && foundDataFile && !foundKeysFile && !foundDataSubdirFile {
if foundKeyFile && !foundKeysFile {
debug.Log("found s3 layout at %v", dir)
return &S3Layout{
return &S3LegacyLayout{
Path: dir,
Join: repo.Join,
}, nil
@@ -190,13 +140,8 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
Path: path,
Join: repo.Join,
}
case "cloud":
l = &CloudLayout{
Path: path,
Join: repo.Join,
}
case "s3":
l = &S3Layout{
case "s3legacy":
l = &S3LegacyLayout{
Path: path,
Join: repo.Join,
}
@@ -205,7 +150,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
// use the default layout if auto detection failed
if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
debug.Log("error: %v, use default layout %v", defaultLayout)
debug.Log("error: %v, use default layout %v", err, defaultLayout)
return ParseLayout(repo, defaultLayout, "", path)
}
@@ -214,7 +159,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
}
debug.Log("layout detected: %v", l)
default:
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, cloud, s3", layout)
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout)
}
return l, nil
-46
View File
@@ -1,46 +0,0 @@
package backend
import "restic"
// CloudLayout implements the default layout for cloud storage backends, as
// described in the Design document.
type CloudLayout struct {
URL string
Path string
Join func(...string) string
}
var cloudLayoutPaths = defaultLayoutPaths
// Dirname returns the directory path for a given file type and name.
func (l *CloudLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *CloudLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *CloudLayout) Paths() (dirs []string) {
for _, p := range cloudLayoutPaths {
dirs = append(dirs, l.URL+l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for files of type t.
func (l *CloudLayout) Basedir(t restic.FileType) string {
return l.URL + l.Join(l.Path, cloudLayoutPaths[t])
}
+2 -2
View File
@@ -24,10 +24,10 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == restic.DataFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2])
p = l.Join(p, h.Name[:2]) + "/"
}
return l.Join(l.Path, p)
return l.Join(l.Path, p) + "/"
}
// Filename returns a path to a file, including its name.
+45
View File
@@ -0,0 +1,45 @@
package backend
import "restic"
// RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct {
URL string
Path string
Join func(...string) string
}
var restLayoutPaths = defaultLayoutPaths
// Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *RESTLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *RESTLayout) Paths() (dirs []string) {
for _, p := range restLayoutPaths {
dirs = append(dirs, l.URL+l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t restic.FileType) string {
return l.URL + l.Join(l.Path, restLayoutPaths[t])
}
@@ -2,9 +2,9 @@ package backend
import "restic"
// S3Layout implements the old layout used for s3 cloud storage backends, as
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
type S3Layout struct {
type S3LegacyLayout struct {
URL string
Path string
Join func(...string) string
@@ -19,7 +19,7 @@ var s3LayoutPaths = map[restic.FileType]string{
}
// join calls Join with the first empty elements removed.
func (l *S3Layout) join(url string, items ...string) string {
func (l *S3LegacyLayout) join(url string, items ...string) string {
for len(items) > 0 && items[0] == "" {
items = items[1:]
}
@@ -35,7 +35,7 @@ func (l *S3Layout) join(url string, items ...string) string {
}
// Dirname returns the directory path for a given file type and name.
func (l *S3Layout) Dirname(h restic.Handle) string {
func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@@ -44,7 +44,7 @@ func (l *S3Layout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *S3Layout) Filename(h restic.Handle) string {
func (l *S3LegacyLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
@@ -55,7 +55,7 @@ func (l *S3Layout) Filename(h restic.Handle) string {
}
// Paths returns all directory names
func (l *S3Layout) Paths() (dirs []string) {
func (l *S3LegacyLayout) Paths() (dirs []string) {
for _, p := range s3LayoutPaths {
dirs = append(dirs, l.Join(l.Path, p))
}
@@ -63,6 +63,6 @@ func (l *S3Layout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for type t.
func (l *S3Layout) Basedir(t restic.FileType) string {
func (l *S3LegacyLayout) Basedir(t restic.FileType) string {
return l.Join(l.Path, s3LayoutPaths[t])
}
+19 -21
View File
@@ -79,7 +79,7 @@ func TestDefaultLayout(t *testing.T) {
}
}
func TestCloudLayout(t *testing.T) {
func TestRESTLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
@@ -113,7 +113,7 @@ func TestCloudLayout(t *testing.T) {
},
}
l := &CloudLayout{
l := &RESTLayout{
Path: path,
Join: filepath.Join,
}
@@ -147,7 +147,7 @@ func TestCloudLayout(t *testing.T) {
}
}
func TestCloudLayoutURLs(t *testing.T) {
func TestRESTLayoutURLs(t *testing.T) {
var tests = []struct {
l Layout
h restic.Handle
@@ -155,55 +155,55 @@ func TestCloudLayoutURLs(t *testing.T) {
dir string
}{
{
&CloudLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3Layout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3Layout{URL: "", Path: "", Join: path.Join},
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3Layout{URL: "", Path: "", Join: path.Join},
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3Layout{URL: "", Path: "/", Join: path.Join},
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"/config",
"/",
@@ -225,7 +225,7 @@ func TestCloudLayoutURLs(t *testing.T) {
}
}
func TestS3Layout(t *testing.T) {
func TestS3LegacyLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
@@ -259,7 +259,7 @@ func TestS3Layout(t *testing.T) {
},
}
l := &S3Layout{
l := &S3LegacyLayout{
Path: path,
Join: filepath.Join,
}
@@ -301,9 +301,8 @@ func TestDetectLayout(t *testing.T) {
filename string
want string
}{
{"repo-layout-local.tar.gz", "*backend.DefaultLayout"},
{"repo-layout-cloud.tar.gz", "*backend.CloudLayout"},
{"repo-layout-s3-old.tar.gz", "*backend.S3Layout"},
{"repo-layout-default.tar.gz", "*backend.DefaultLayout"},
{"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"},
}
var fs = &LocalFilesystem{}
@@ -342,12 +341,11 @@ func TestParseLayout(t *testing.T) {
want string
}{
{"default", "", "*backend.DefaultLayout"},
{"cloud", "", "*backend.CloudLayout"},
{"s3", "", "*backend.S3Layout"},
{"", "", "*backend.CloudLayout"},
{"s3legacy", "", "*backend.S3LegacyLayout"},
{"", "", "*backend.DefaultLayout"},
}
SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-cloud.tar.gz"))
SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz"))
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {
+2 -7
View File
@@ -17,17 +17,12 @@ func TestLayout(t *testing.T) {
failureExpected bool
datafiles map[string]bool
}{
{"repo-layout-local.tar.gz", "", false, map[string]bool{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
+1 -1
View File
@@ -48,7 +48,7 @@ func Open(cfg Config) (restic.Backend, error) {
url: cfg.URL,
connChan: connChan,
client: client,
Layout: &backend.CloudLayout{URL: url, Join: path.Join},
Layout: &backend.RESTLayout{URL: url, Join: path.Join},
}
return be, nil
+6
View File
@@ -6,6 +6,7 @@ import (
"strings"
"restic/errors"
"restic/options"
)
// Config contains all configuration necessary to connect to an s3 compatible
@@ -16,6 +17,11 @@ type Config struct {
KeyID, Secret string
Bucket string
Prefix string
Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"`
}
func init() {
options.Register("s3", Config{})
}
const defaultPrefix = "restic"
+86 -1
View File
@@ -8,6 +8,7 @@ import (
"restic"
"strings"
"sync"
"time"
"restic/backend"
"restic/errors"
@@ -30,6 +31,8 @@ type s3 struct {
backend.Layout
}
const defaultLayout = "s3legacy"
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
@@ -45,11 +48,17 @@ func Open(cfg Config) (restic.Backend, error) {
bucketname: cfg.Bucket,
prefix: cfg.Prefix,
cacheObjSize: make(map[string]int64),
Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
}
client.SetCustomTransport(backend.Transport())
l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil {
return nil, err
}
be.Layout = l
be.createConnections()
found, err := client.BucketExists(cfg.Bucket)
@@ -76,6 +85,77 @@ func (be *s3) createConnections() {
}
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *s3) IsNotExist(err error) bool {
debug.Log("IsNotExist(%T, %#v)", err, err)
if os.IsNotExist(err) {
return true
}
return false
}
// Join combines path components with slashes.
func (be *s3) Join(p ...string) string {
return path.Join(p...)
}
type fileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
func (fi fileInfo) Name() string { return fi.name } // base name of the file
func (fi fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others
func (fi fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir()
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
// ReadDir returns the entries for a directory.
func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) {
debug.Log("ReadDir(%v)", dir)
// make sure dir ends with a slash
if dir[len(dir)-1] != '/' {
dir += "/"
}
done := make(chan struct{})
defer close(done)
for obj := range be.client.ListObjects(be.bucketname, dir, false, done) {
if obj.Key == "" {
continue
}
name := strings.TrimPrefix(obj.Key, dir)
if name == "" {
return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir)
}
entry := fileInfo{
name: name,
size: obj.Size,
modTime: obj.LastModified,
}
if name[len(name)-1] == '/' {
entry.isDir = true
entry.mode = os.ModeDir | 0755
entry.name = name[:len(name)-1]
} else {
entry.mode = 0644
}
list = append(list, entry)
}
return list, nil
}
// Location returns this backend's location (the bucket name).
func (be *s3) Location() string {
return be.bucketname
@@ -290,6 +370,11 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
prefix := be.Dirname(restic.Handle{Type: t})
// make sure prefix ends with a slash
if prefix[len(prefix)-1] != '/' {
prefix += "/"
}
listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
go func() {
+2 -7
View File
@@ -23,17 +23,12 @@ func TestLayout(t *testing.T) {
failureExpected bool
datafiles map[string]bool
}{
{"repo-layout-local.tar.gz", "", false, map[string]bool{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
Binary file not shown.