mirror of
https://github.com/restic/restic.git
synced 2026-08-25 15:03:17 +00:00
Move things around for gb
This moves all restic source files to src/, and all vendored dependencies to vendor/src.
This commit is contained in:
Alexander Neumann
@@ -1,5 +0,0 @@
|
||||
// Package backend provides local and remote storage for restic repositories.
|
||||
// All backends need to implement the Backend interface. There is a
|
||||
// MockBackend, which can be used for mocking in tests, and a MemBackend, which
|
||||
// stores all data in a hash internally.
|
||||
package backend
|
||||
@@ -1,70 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrNoIDPrefixFound is returned by Find() when no ID for the given prefix
|
||||
// could be found.
|
||||
var ErrNoIDPrefixFound = errors.New("no ID found")
|
||||
|
||||
// ErrMultipleIDMatches is returned by Find() when multiple IDs with the given
|
||||
// prefix are found.
|
||||
var ErrMultipleIDMatches = errors.New("multiple IDs with prefix found")
|
||||
|
||||
// Find loads the list of all blobs of type t and searches for names which
|
||||
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
|
||||
// If more than one is found, nil and ErrMultipleIDMatches is returned.
|
||||
func Find(be Lister, t Type, prefix string) (string, error) {
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
|
||||
match := ""
|
||||
|
||||
// TODO: optimize by sorting list etc.
|
||||
for name := range be.List(t, done) {
|
||||
if prefix == name[:len(prefix)] {
|
||||
if match == "" {
|
||||
match = name
|
||||
} else {
|
||||
return "", ErrMultipleIDMatches
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if match != "" {
|
||||
return match, nil
|
||||
}
|
||||
|
||||
return "", ErrNoIDPrefixFound
|
||||
}
|
||||
|
||||
const minPrefixLength = 8
|
||||
|
||||
// PrefixLength returns the number of bytes required so that all prefixes of
|
||||
// all names of type t are unique.
|
||||
func PrefixLength(be Lister, t Type) (int, error) {
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
|
||||
// load all IDs of the given type
|
||||
list := make([]string, 0, 100)
|
||||
for name := range be.List(t, done) {
|
||||
list = append(list, name)
|
||||
}
|
||||
|
||||
// select prefixes of length l, test if the last one is the same as the current one
|
||||
outer:
|
||||
for l := minPrefixLength; l < IDSize; l++ {
|
||||
var last string
|
||||
|
||||
for _, name := range list {
|
||||
if last == name[:l] {
|
||||
continue outer
|
||||
}
|
||||
last = name[:l]
|
||||
}
|
||||
|
||||
return l, nil
|
||||
}
|
||||
|
||||
return IDSize, nil
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
type mockBackend struct {
|
||||
list func(backend.Type, <-chan struct{}) <-chan string
|
||||
}
|
||||
|
||||
func (m mockBackend) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
return m.list(t, done)
|
||||
}
|
||||
|
||||
var samples = backend.IDs{
|
||||
ParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
|
||||
ParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
|
||||
ParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
|
||||
ParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
|
||||
ParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
|
||||
ParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
|
||||
ParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
|
||||
ParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
|
||||
}
|
||||
|
||||
func TestPrefixLength(t *testing.T) {
|
||||
list := samples
|
||||
|
||||
m := mockBackend{}
|
||||
m.list = func(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, id := range list {
|
||||
select {
|
||||
case ch <- id.String():
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
l, err := backend.PrefixLength(m, backend.Snapshot)
|
||||
OK(t, err)
|
||||
Equals(t, 19, l)
|
||||
|
||||
list = samples[:3]
|
||||
l, err = backend.PrefixLength(m, backend.Snapshot)
|
||||
OK(t, err)
|
||||
Equals(t, 19, l)
|
||||
|
||||
list = samples[3:]
|
||||
l, err = backend.PrefixLength(m, backend.Snapshot)
|
||||
OK(t, err)
|
||||
Equals(t, 8, l)
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Handle is used to store and access data in a backend.
|
||||
type Handle struct {
|
||||
Type Type
|
||||
Name string
|
||||
}
|
||||
|
||||
func (h Handle) String() string {
|
||||
name := h.Name
|
||||
if len(name) > 10 {
|
||||
name = name[:10]
|
||||
}
|
||||
return fmt.Sprintf("<%s/%s>", h.Type, name)
|
||||
}
|
||||
|
||||
// Valid returns an error if h is not valid.
|
||||
func (h Handle) Valid() error {
|
||||
if h.Type == "" {
|
||||
return errors.New("type is empty")
|
||||
}
|
||||
|
||||
switch h.Type {
|
||||
case Data:
|
||||
case Key:
|
||||
case Lock:
|
||||
case Snapshot:
|
||||
case Index:
|
||||
case Config:
|
||||
default:
|
||||
return fmt.Errorf("invalid Type %q", h.Type)
|
||||
}
|
||||
|
||||
if h.Type == Config {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.Name == "" {
|
||||
return errors.New("invalid Name")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "testing"
|
||||
|
||||
var handleTests = []struct {
|
||||
h Handle
|
||||
valid bool
|
||||
}{
|
||||
{Handle{Name: "foo"}, false},
|
||||
{Handle{Type: "foobar"}, false},
|
||||
{Handle{Type: Config, Name: ""}, true},
|
||||
{Handle{Type: Data, Name: ""}, false},
|
||||
{Handle{Type: "", Name: "x"}, false},
|
||||
{Handle{Type: Lock, Name: "010203040506"}, true},
|
||||
}
|
||||
|
||||
func TestHandleValid(t *testing.T) {
|
||||
for i, test := range handleTests {
|
||||
err := test.h.Valid()
|
||||
if err != nil && test.valid {
|
||||
t.Errorf("test %v failed: error returned for valid handle: %v", i, err)
|
||||
}
|
||||
|
||||
if !test.valid && err == nil {
|
||||
t.Errorf("test %v failed: expected error for invalid handle not found", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
-108
@@ -1,108 +0,0 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// Hash returns the ID for data.
|
||||
func Hash(data []byte) ID {
|
||||
return sha256.Sum256(data)
|
||||
}
|
||||
|
||||
// IDSize contains the size of an ID, in bytes.
|
||||
const IDSize = sha256.Size
|
||||
|
||||
// ID references content within a repository.
|
||||
type ID [IDSize]byte
|
||||
|
||||
// ParseID converts the given string to an ID.
|
||||
func ParseID(s string) (ID, error) {
|
||||
b, err := hex.DecodeString(s)
|
||||
|
||||
if err != nil {
|
||||
return ID{}, err
|
||||
}
|
||||
|
||||
if len(b) != IDSize {
|
||||
return ID{}, errors.New("invalid length for hash")
|
||||
}
|
||||
|
||||
id := ID{}
|
||||
copy(id[:], b)
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (id ID) String() string {
|
||||
return hex.EncodeToString(id[:])
|
||||
}
|
||||
|
||||
const shortStr = 4
|
||||
|
||||
// Str returns the shortened string version of id.
|
||||
func (id *ID) Str() string {
|
||||
if id == nil {
|
||||
return "[nil]"
|
||||
}
|
||||
|
||||
if id.IsNull() {
|
||||
return "[null]"
|
||||
}
|
||||
|
||||
return hex.EncodeToString(id[:shortStr])
|
||||
}
|
||||
|
||||
// IsNull returns true iff id only consists of null bytes.
|
||||
func (id ID) IsNull() bool {
|
||||
var nullID ID
|
||||
|
||||
return id == nullID
|
||||
}
|
||||
|
||||
// Equal compares an ID to another other.
|
||||
func (id ID) Equal(other ID) bool {
|
||||
return id == other
|
||||
}
|
||||
|
||||
// EqualString compares this ID to another one, given as a string.
|
||||
func (id ID) EqualString(other string) (bool, error) {
|
||||
s, err := hex.DecodeString(other)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
id2 := ID{}
|
||||
copy(id2[:], s)
|
||||
|
||||
return id == id2, nil
|
||||
}
|
||||
|
||||
// Compare compares this ID to another one, returning -1, 0, or 1.
|
||||
func (id ID) Compare(other ID) int {
|
||||
return bytes.Compare(other[:], id[:])
|
||||
}
|
||||
|
||||
// MarshalJSON returns the JSON encoding of id.
|
||||
func (id ID) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(id.String())
|
||||
}
|
||||
|
||||
// UnmarshalJSON parses the JSON-encoded data and stores the result in id.
|
||||
func (id *ID) UnmarshalJSON(b []byte) error {
|
||||
var s string
|
||||
err := json.Unmarshal(b, &s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = hex.Decode(id[:], []byte(s))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIDMethods(t *testing.T) {
|
||||
var id ID
|
||||
|
||||
if id.Str() != "[null]" {
|
||||
t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[null]", id.Str())
|
||||
}
|
||||
|
||||
var pid *ID
|
||||
if pid.Str() != "[nil]" {
|
||||
t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[nil]", pid.Str())
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
var TestStrings = []struct {
|
||||
id string
|
||||
data string
|
||||
}{
|
||||
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
|
||||
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
|
||||
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
|
||||
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
|
||||
}
|
||||
|
||||
func TestID(t *testing.T) {
|
||||
for _, test := range TestStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
OK(t, err)
|
||||
|
||||
id2, err := backend.ParseID(test.id)
|
||||
OK(t, err)
|
||||
Assert(t, id.Equal(id2), "ID.Equal() does not work as expected")
|
||||
|
||||
ret, err := id.EqualString(test.id)
|
||||
OK(t, err)
|
||||
Assert(t, ret, "ID.EqualString() returned wrong value")
|
||||
|
||||
// test json marshalling
|
||||
buf, err := id.MarshalJSON()
|
||||
OK(t, err)
|
||||
Equals(t, "\""+test.id+"\"", string(buf))
|
||||
|
||||
var id3 backend.ID
|
||||
err = id3.UnmarshalJSON(buf)
|
||||
OK(t, err)
|
||||
Equals(t, id, id3)
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// IDs is an ordered list of IDs that implements sort.Interface.
|
||||
type IDs []ID
|
||||
|
||||
func (ids IDs) Len() int {
|
||||
return len(ids)
|
||||
}
|
||||
|
||||
func (ids IDs) Less(i, j int) bool {
|
||||
if len(ids[i]) < len(ids[j]) {
|
||||
return true
|
||||
}
|
||||
|
||||
for k, b := range ids[i] {
|
||||
if b == ids[j][k] {
|
||||
continue
|
||||
}
|
||||
|
||||
if b < ids[j][k] {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (ids IDs) Swap(i, j int) {
|
||||
ids[i], ids[j] = ids[j], ids[i]
|
||||
}
|
||||
|
||||
// Uniq returns list without duplicate IDs. The returned list retains the order
|
||||
// of the original list so that the order of the first occurrence of each ID
|
||||
// stays the same.
|
||||
func (ids IDs) Uniq() (list IDs) {
|
||||
seen := NewIDSet()
|
||||
|
||||
for _, id := range ids {
|
||||
if seen.Has(id) {
|
||||
continue
|
||||
}
|
||||
|
||||
list = append(list, id)
|
||||
seen.Insert(id)
|
||||
}
|
||||
|
||||
return list
|
||||
}
|
||||
|
||||
type shortID ID
|
||||
|
||||
func (id shortID) String() string {
|
||||
return hex.EncodeToString(id[:shortStr])
|
||||
}
|
||||
|
||||
func (ids IDs) String() string {
|
||||
elements := make([]shortID, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
elements = append(elements, shortID(id))
|
||||
}
|
||||
return fmt.Sprintf("%v", elements)
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
var uniqTests = []struct {
|
||||
before, after backend.IDs
|
||||
}{
|
||||
{
|
||||
backend.IDs{
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
},
|
||||
backend.IDs{
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
|
||||
},
|
||||
},
|
||||
{
|
||||
backend.IDs{
|
||||
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
},
|
||||
backend.IDs{
|
||||
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
},
|
||||
},
|
||||
{
|
||||
backend.IDs{
|
||||
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
|
||||
ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
},
|
||||
backend.IDs{
|
||||
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
|
||||
ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
|
||||
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestUniqIDs(t *testing.T) {
|
||||
for i, test := range uniqTests {
|
||||
uniq := test.before.Uniq()
|
||||
if !reflect.DeepEqual(uniq, test.after) {
|
||||
t.Errorf("uniqIDs() test %v failed\n wanted: %v\n got: %v", i, test.after, uniq)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "sort"
|
||||
|
||||
// IDSet is a set of IDs.
|
||||
type IDSet map[ID]struct{}
|
||||
|
||||
// NewIDSet returns a new IDSet, populated with ids.
|
||||
func NewIDSet(ids ...ID) IDSet {
|
||||
m := make(IDSet)
|
||||
for _, id := range ids {
|
||||
m[id] = struct{}{}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// Has returns true iff id is contained in the set.
|
||||
func (s IDSet) Has(id ID) bool {
|
||||
_, ok := s[id]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Insert adds id to the set.
|
||||
func (s IDSet) Insert(id ID) {
|
||||
s[id] = struct{}{}
|
||||
}
|
||||
|
||||
// Delete removes id from the set.
|
||||
func (s IDSet) Delete(id ID) {
|
||||
delete(s, id)
|
||||
}
|
||||
|
||||
// List returns a slice of all IDs in the set.
|
||||
func (s IDSet) List() IDs {
|
||||
list := make(IDs, 0, len(s))
|
||||
for id := range s {
|
||||
list = append(list, id)
|
||||
}
|
||||
|
||||
sort.Sort(list)
|
||||
|
||||
return list
|
||||
}
|
||||
|
||||
// Equals returns true iff s equals other.
|
||||
func (s IDSet) Equals(other IDSet) bool {
|
||||
if len(s) != len(other) {
|
||||
return false
|
||||
}
|
||||
|
||||
for id := range s {
|
||||
if _, ok := other[id]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
for id := range other {
|
||||
if _, ok := s[id]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (s IDSet) String() string {
|
||||
str := s.List().String()
|
||||
if len(str) < 2 {
|
||||
return "{}"
|
||||
}
|
||||
|
||||
return "{" + str[1:len(str)-1] + "}"
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
var idsetTests = []struct {
|
||||
id backend.ID
|
||||
seen bool
|
||||
}{
|
||||
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), false},
|
||||
{ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), false},
|
||||
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
|
||||
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
|
||||
{ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true},
|
||||
{ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), false},
|
||||
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
|
||||
{ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true},
|
||||
{ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), true},
|
||||
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
|
||||
}
|
||||
|
||||
func TestIDSet(t *testing.T) {
|
||||
set := backend.NewIDSet()
|
||||
for i, test := range idsetTests {
|
||||
seen := set.Has(test.id)
|
||||
if seen != test.seen {
|
||||
t.Errorf("IDSet test %v failed: wanted %v, got %v", i, test.seen, seen)
|
||||
}
|
||||
set.Insert(test.id)
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package backend
|
||||
|
||||
// Type is the type of a Blob.
|
||||
type Type string
|
||||
|
||||
// These are the different data types a backend can store.
|
||||
const (
|
||||
Data Type = "data"
|
||||
Key = "key"
|
||||
Lock = "lock"
|
||||
Snapshot = "snapshot"
|
||||
Index = "index"
|
||||
Config = "config"
|
||||
)
|
||||
|
||||
// Backend is used to store and access data.
|
||||
type Backend interface {
|
||||
// Location returns a string that describes the type and location of the
|
||||
// repository.
|
||||
Location() string
|
||||
|
||||
// Test a boolean value whether a Blob with the name and type exists.
|
||||
Test(t Type, name string) (bool, error)
|
||||
|
||||
// Remove removes a Blob with type t and name.
|
||||
Remove(t Type, name string) error
|
||||
|
||||
// Close the backend
|
||||
Close() error
|
||||
|
||||
Lister
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
Load(h Handle, p []byte, off int64) (int, error)
|
||||
|
||||
// Save stores the data in the backend under the given handle.
|
||||
Save(h Handle, p []byte) error
|
||||
|
||||
// Stat returns information about the blob identified by h.
|
||||
Stat(h Handle) (BlobInfo, error)
|
||||
}
|
||||
|
||||
// Lister implements listing data items stored in a backend.
|
||||
type Lister interface {
|
||||
// List returns a channel that yields all names of blobs of type t in an
|
||||
// arbitrary order. A goroutine is started for this. If the channel done is
|
||||
// closed, sending stops.
|
||||
List(t Type, done <-chan struct{}) <-chan string
|
||||
}
|
||||
|
||||
// Deleter are backends that allow to self-delete all content stored in them.
|
||||
type Deleter interface {
|
||||
// Delete the complete repository.
|
||||
Delete() error
|
||||
}
|
||||
|
||||
// BlobInfo is returned by Stat() and contains information about a stored blob.
|
||||
type BlobInfo struct {
|
||||
Size int64
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package local_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
func TestLocalBackendCreate(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreate(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendOpen(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestOpen(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendCreateWithConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreateWithConfig(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendLocation(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLocation(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestConfig(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendLoad(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSave(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendSaveFilenames(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSaveFilenames(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendBackend(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestBackend(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendDelete(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestDelete(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendCleanup(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCleanup(t)
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParseConfig parses a local backend config.
|
||||
func ParseConfig(cfg string) (interface{}, error) {
|
||||
if !strings.HasPrefix(cfg, "local:") {
|
||||
return nil, errors.New(`invalid format, prefix "local" not found`)
|
||||
}
|
||||
|
||||
return cfg[6:], nil
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
// Package local implements repository storage in a local directory.
|
||||
package local
|
||||
@@ -1,288 +0,0 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/debug"
|
||||
)
|
||||
|
||||
// Local is a backend in a local directory.
|
||||
type Local struct {
|
||||
p string
|
||||
}
|
||||
|
||||
func paths(dir string) []string {
|
||||
return []string{
|
||||
dir,
|
||||
filepath.Join(dir, backend.Paths.Data),
|
||||
filepath.Join(dir, backend.Paths.Snapshots),
|
||||
filepath.Join(dir, backend.Paths.Index),
|
||||
filepath.Join(dir, backend.Paths.Locks),
|
||||
filepath.Join(dir, backend.Paths.Keys),
|
||||
filepath.Join(dir, backend.Paths.Temp),
|
||||
}
|
||||
}
|
||||
|
||||
// Open opens the local backend as specified by config.
|
||||
func Open(dir string) (*Local, error) {
|
||||
// test if all necessary dirs are there
|
||||
for _, d := range paths(dir) {
|
||||
if _, err := os.Stat(d); err != nil {
|
||||
return nil, fmt.Errorf("%s does not exist", d)
|
||||
}
|
||||
}
|
||||
|
||||
return &Local{p: dir}, nil
|
||||
}
|
||||
|
||||
// Create creates all the necessary files and directories for a new local
|
||||
// backend at dir. Afterwards a new config blob should be created.
|
||||
func Create(dir string) (*Local, error) {
|
||||
// test if config file already exists
|
||||
_, err := os.Lstat(filepath.Join(dir, backend.Paths.Config))
|
||||
if err == nil {
|
||||
return nil, errors.New("config file already exists")
|
||||
}
|
||||
|
||||
// create paths for data, refs and temp
|
||||
for _, d := range paths(dir) {
|
||||
err := os.MkdirAll(d, backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// open backend
|
||||
return Open(dir)
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the directory name).
|
||||
func (b *Local) Location() string {
|
||||
return b.p
|
||||
}
|
||||
|
||||
// Construct path for given Type and name.
|
||||
func filename(base string, t backend.Type, name string) string {
|
||||
if t == backend.Config {
|
||||
return filepath.Join(base, "config")
|
||||
}
|
||||
|
||||
return filepath.Join(dirname(base, t, name), name)
|
||||
}
|
||||
|
||||
// Construct directory for given Type.
|
||||
func dirname(base string, t backend.Type, name string) string {
|
||||
var n string
|
||||
switch t {
|
||||
case backend.Data:
|
||||
n = backend.Paths.Data
|
||||
if len(name) > 2 {
|
||||
n = filepath.Join(n, name[:2])
|
||||
}
|
||||
case backend.Snapshot:
|
||||
n = backend.Paths.Snapshots
|
||||
case backend.Index:
|
||||
n = backend.Paths.Index
|
||||
case backend.Lock:
|
||||
n = backend.Paths.Locks
|
||||
case backend.Key:
|
||||
n = backend.Paths.Keys
|
||||
}
|
||||
return filepath.Join(base, n)
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f, err := os.Open(filename(b.p, h.Type, h.Name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
e := f.Close()
|
||||
if err == nil && e != nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
|
||||
if off > 0 {
|
||||
_, err = f.Seek(off, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return io.ReadFull(f, p)
|
||||
}
|
||||
|
||||
// writeToTempfile saves p into a tempfile in tempdir.
|
||||
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
|
||||
tmpfile, err := ioutil.TempFile(tempdir, "temp-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
n, err := tmpfile.Write(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if n != len(p) {
|
||||
return "", errors.New("not all bytes writen")
|
||||
}
|
||||
|
||||
if err = tmpfile.Sync(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = tmpfile.Close()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tmpfile.Name(), nil
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (b *Local) Save(h backend.Handle, p []byte) (err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
|
||||
debug.Log("local.Save", "saved %v (%d bytes) to %v", h, len(p), tmpfile)
|
||||
|
||||
filename := filename(b.p, h.Type, h.Name)
|
||||
|
||||
// test if new path already exists
|
||||
if _, err := os.Stat(filename); err == nil {
|
||||
return fmt.Errorf("Rename(): file %v already exists", filename)
|
||||
}
|
||||
|
||||
// create directories if necessary, ignore errors
|
||||
if h.Type == backend.Data {
|
||||
err = os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = os.Rename(tmpfile, filename)
|
||||
debug.Log("local.Save", "save %v: rename %v -> %v: %v",
|
||||
h, filepath.Base(tmpfile), filepath.Base(filename), err)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set mode to read-only
|
||||
fi, err := os.Stat(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return setNewFileMode(filename, fi)
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
fi, err := os.Stat(filename(b.p, h.Type, h.Name))
|
||||
if err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
return backend.BlobInfo{Size: fi.Size()}, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (b *Local) Test(t backend.Type, name string) (bool, error) {
|
||||
_, err := os.Stat(filename(b.p, t, name))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (b *Local) Remove(t backend.Type, name string) error {
|
||||
fn := filename(b.p, t, name)
|
||||
|
||||
// reset read-only flag
|
||||
err := os.Chmod(fn, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Remove(fn)
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine is started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
var pattern string
|
||||
if t == backend.Data {
|
||||
pattern = filepath.Join(dirname(b.p, t, ""), "*", "*")
|
||||
} else {
|
||||
pattern = filepath.Join(dirname(b.p, t, ""), "*")
|
||||
}
|
||||
|
||||
ch := make(chan string)
|
||||
matches, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
for i := range matches {
|
||||
matches[i] = filepath.Base(matches[i])
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, m := range matches {
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- m:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Delete removes the repository and all files.
|
||||
func (b *Local) Delete() error {
|
||||
return os.RemoveAll(b.p)
|
||||
}
|
||||
|
||||
// Close closes all open files.
|
||||
func (b *Local) Close() error {
|
||||
// this does not need to do anything, all open files are closed within the
|
||||
// same function.
|
||||
return nil
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package local_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/local"
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var tempBackendDir string
|
||||
|
||||
//go:generate go run ../test/generate_backend_tests.go
|
||||
|
||||
func createTempdir() error {
|
||||
if tempBackendDir != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
tempdir, err := ioutil.TempDir("", "restic-local-test-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("created new test backend at %v\n", tempdir)
|
||||
tempBackendDir = tempdir
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
test.CreateFn = func() (backend.Backend, error) {
|
||||
err := createTempdir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return local.Create(tempBackendDir)
|
||||
}
|
||||
|
||||
test.OpenFn = func() (backend.Backend, error) {
|
||||
err := createTempdir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return local.Open(tempBackendDir)
|
||||
}
|
||||
|
||||
test.CleanupFn = func() error {
|
||||
if tempBackendDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("removing test backend at %v\n", tempBackendDir)
|
||||
err := os.RemoveAll(tempBackendDir)
|
||||
tempBackendDir = ""
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
// +build !windows
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// set file to readonly
|
||||
func setNewFileMode(f string, fi os.FileInfo) error {
|
||||
return os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// We don't modify read-only on windows,
|
||||
// since it will make us unable to delete the file,
|
||||
// and this isn't common practice on this platform.
|
||||
func setNewFileMode(f string, fi os.FileInfo) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package mem_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
func TestMemBackendCreate(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreate(t)
|
||||
}
|
||||
|
||||
func TestMemBackendOpen(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestOpen(t)
|
||||
}
|
||||
|
||||
func TestMemBackendCreateWithConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreateWithConfig(t)
|
||||
}
|
||||
|
||||
func TestMemBackendLocation(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLocation(t)
|
||||
}
|
||||
|
||||
func TestMemBackendConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestConfig(t)
|
||||
}
|
||||
|
||||
func TestMemBackendLoad(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestMemBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSave(t)
|
||||
}
|
||||
|
||||
func TestMemBackendSaveFilenames(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSaveFilenames(t)
|
||||
}
|
||||
|
||||
func TestMemBackendBackend(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestBackend(t)
|
||||
}
|
||||
|
||||
func TestMemBackendDelete(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestDelete(t)
|
||||
}
|
||||
|
||||
func TestMemBackendCleanup(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCleanup(t)
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package mem
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/debug"
|
||||
)
|
||||
|
||||
type entry struct {
|
||||
Type backend.Type
|
||||
Name string
|
||||
}
|
||||
|
||||
type memMap map[entry][]byte
|
||||
|
||||
// MemoryBackend is a mock backend that uses a map for storing all data in
|
||||
// memory. This should only be used for tests.
|
||||
type MemoryBackend struct {
|
||||
data memMap
|
||||
m sync.Mutex
|
||||
|
||||
backend.MockBackend
|
||||
}
|
||||
|
||||
// New returns a new backend that saves all data in a map in memory.
|
||||
func New() *MemoryBackend {
|
||||
be := &MemoryBackend{
|
||||
data: make(memMap),
|
||||
}
|
||||
|
||||
be.MockBackend.TestFn = func(t backend.Type, name string) (bool, error) {
|
||||
return memTest(be, t, name)
|
||||
}
|
||||
|
||||
be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
|
||||
return memLoad(be, h, p, off)
|
||||
}
|
||||
|
||||
be.MockBackend.SaveFn = func(h backend.Handle, p []byte) error {
|
||||
return memSave(be, h, p)
|
||||
}
|
||||
|
||||
be.MockBackend.StatFn = func(h backend.Handle) (backend.BlobInfo, error) {
|
||||
return memStat(be, h)
|
||||
}
|
||||
|
||||
be.MockBackend.RemoveFn = func(t backend.Type, name string) error {
|
||||
return memRemove(be, t, name)
|
||||
}
|
||||
|
||||
be.MockBackend.ListFn = func(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
return memList(be, t, done)
|
||||
}
|
||||
|
||||
be.MockBackend.DeleteFn = func() error {
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
be.data = make(memMap)
|
||||
return nil
|
||||
}
|
||||
|
||||
be.MockBackend.LocationFn = func() string {
|
||||
return "Memory Backend"
|
||||
}
|
||||
|
||||
debug.Log("MemoryBackend.New", "created new memory backend")
|
||||
|
||||
return be
|
||||
}
|
||||
|
||||
func (be *MemoryBackend) insert(t backend.Type, name string, data []byte) error {
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
if _, ok := be.data[entry{t, name}]; ok {
|
||||
return errors.New("already present")
|
||||
}
|
||||
|
||||
be.data[entry{t, name}] = data
|
||||
return nil
|
||||
}
|
||||
|
||||
func memTest(be *MemoryBackend, t backend.Type, name string) (bool, error) {
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
debug.Log("MemoryBackend.Test", "test %v %v", t, name)
|
||||
|
||||
if _, ok := be.data[entry{t, name}]; ok {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
if h.Type == backend.Config {
|
||||
h.Name = ""
|
||||
}
|
||||
|
||||
debug.Log("MemoryBackend.Load", "get %v offset %v len %v", h, off, len(p))
|
||||
|
||||
if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
|
||||
return 0, errors.New("no such data")
|
||||
}
|
||||
|
||||
buf := be.data[entry{h.Type, h.Name}]
|
||||
if off > int64(len(buf)) {
|
||||
return 0, errors.New("offset beyond end of file")
|
||||
}
|
||||
|
||||
buf = buf[off:]
|
||||
|
||||
n := copy(p, buf)
|
||||
|
||||
if len(p) > len(buf) {
|
||||
return n, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func memSave(be *MemoryBackend, h backend.Handle, p []byte) error {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
if h.Type == backend.Config {
|
||||
h.Name = ""
|
||||
}
|
||||
|
||||
if _, ok := be.data[entry{h.Type, h.Name}]; ok {
|
||||
return errors.New("file already exists")
|
||||
}
|
||||
|
||||
debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h)
|
||||
buf := make([]byte, len(p))
|
||||
copy(buf, p)
|
||||
be.data[entry{h.Type, h.Name}] = buf
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func memStat(be *MemoryBackend, h backend.Handle) (backend.BlobInfo, error) {
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
if err := h.Valid(); err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
if h.Type == backend.Config {
|
||||
h.Name = ""
|
||||
}
|
||||
|
||||
debug.Log("MemoryBackend.Stat", "stat %v", h)
|
||||
|
||||
e, ok := be.data[entry{h.Type, h.Name}]
|
||||
if !ok {
|
||||
return backend.BlobInfo{}, errors.New("no such data")
|
||||
}
|
||||
|
||||
return backend.BlobInfo{Size: int64(len(e))}, nil
|
||||
}
|
||||
|
||||
func memRemove(be *MemoryBackend, t backend.Type, name string) error {
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
debug.Log("MemoryBackend.Remove", "get %v %v", t, name)
|
||||
|
||||
if _, ok := be.data[entry{t, name}]; !ok {
|
||||
return errors.New("no such data")
|
||||
}
|
||||
|
||||
delete(be.data, entry{t, name})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func memList(be *MemoryBackend, t backend.Type, done <-chan struct{}) <-chan string {
|
||||
be.m.Lock()
|
||||
defer be.m.Unlock()
|
||||
|
||||
ch := make(chan string)
|
||||
|
||||
var ids []string
|
||||
for entry := range be.data {
|
||||
if entry.Type != t {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, entry.Name)
|
||||
}
|
||||
|
||||
debug.Log("MemoryBackend.List", "list %v: %v", t, ids)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, id := range ids {
|
||||
select {
|
||||
case ch <- id:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package mem_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/mem"
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var be backend.Backend
|
||||
|
||||
//go:generate go run ../test/generate_backend_tests.go
|
||||
|
||||
func init() {
|
||||
test.CreateFn = func() (backend.Backend, error) {
|
||||
if be != nil {
|
||||
return nil, errors.New("temporary memory backend dir already exists")
|
||||
}
|
||||
|
||||
be = mem.New()
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
test.OpenFn = func() (backend.Backend, error) {
|
||||
if be == nil {
|
||||
return nil, errors.New("repository not initialized")
|
||||
}
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
test.CleanupFn = func() error {
|
||||
be = nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "errors"
|
||||
|
||||
// MockBackend implements a backend whose functions can be specified. This
|
||||
// should only be used for tests.
|
||||
type MockBackend struct {
|
||||
CloseFn func() error
|
||||
LoadFn func(h Handle, p []byte, off int64) (int, error)
|
||||
SaveFn func(h Handle, p []byte) error
|
||||
StatFn func(h Handle) (BlobInfo, error)
|
||||
ListFn func(Type, <-chan struct{}) <-chan string
|
||||
RemoveFn func(Type, string) error
|
||||
TestFn func(Type, string) (bool, error)
|
||||
DeleteFn func() error
|
||||
LocationFn func() string
|
||||
}
|
||||
|
||||
// Close the backend.
|
||||
func (m *MockBackend) Close() error {
|
||||
if m.CloseFn == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return m.CloseFn()
|
||||
}
|
||||
|
||||
// Location returns a location string.
|
||||
func (m *MockBackend) Location() string {
|
||||
if m.LocationFn == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return m.LocationFn()
|
||||
}
|
||||
|
||||
// Load loads data from the backend.
|
||||
func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) {
|
||||
if m.LoadFn == nil {
|
||||
return 0, errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.LoadFn(h, p, off)
|
||||
}
|
||||
|
||||
// Save data in the backend.
|
||||
func (m *MockBackend) Save(h Handle, p []byte) error {
|
||||
if m.SaveFn == nil {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.SaveFn(h, p)
|
||||
}
|
||||
|
||||
// Stat an object in the backend.
|
||||
func (m *MockBackend) Stat(h Handle) (BlobInfo, error) {
|
||||
if m.StatFn == nil {
|
||||
return BlobInfo{}, errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.StatFn(h)
|
||||
}
|
||||
|
||||
// List items of type t.
|
||||
func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string {
|
||||
if m.ListFn == nil {
|
||||
ch := make(chan string)
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
return m.ListFn(t, done)
|
||||
}
|
||||
|
||||
// Remove data from the backend.
|
||||
func (m *MockBackend) Remove(t Type, name string) error {
|
||||
if m.RemoveFn == nil {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.RemoveFn(t, name)
|
||||
}
|
||||
|
||||
// Test for the existence of a specific item.
|
||||
func (m *MockBackend) Test(t Type, name string) (bool, error) {
|
||||
if m.TestFn == nil {
|
||||
return false, errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.TestFn(t, name)
|
||||
}
|
||||
|
||||
// Delete all data.
|
||||
func (m *MockBackend) Delete() error {
|
||||
if m.DeleteFn == nil {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.DeleteFn()
|
||||
}
|
||||
|
||||
// Make sure that MockBackend implements the backend interface.
|
||||
var _ Backend = &MockBackend{}
|
||||
@@ -1,26 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "os"
|
||||
|
||||
// Paths contains the default paths for file-based backends (e.g. local).
|
||||
var Paths = struct {
|
||||
Data string
|
||||
Snapshots string
|
||||
Index string
|
||||
Locks string
|
||||
Keys string
|
||||
Temp string
|
||||
Config string
|
||||
}{
|
||||
"data",
|
||||
"snapshots",
|
||||
"index",
|
||||
"locks",
|
||||
"keys",
|
||||
"tmp",
|
||||
"config",
|
||||
}
|
||||
|
||||
// Modes holds the default modes for directories and files for file-based
|
||||
// backends.
|
||||
var Modes = struct{ Dir, File os.FileMode }{0700, 0600}
|
||||
@@ -1,87 +0,0 @@
|
||||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
func TestS3BackendCreate(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreate(t)
|
||||
}
|
||||
|
||||
func TestS3BackendOpen(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestOpen(t)
|
||||
}
|
||||
|
||||
func TestS3BackendCreateWithConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreateWithConfig(t)
|
||||
}
|
||||
|
||||
func TestS3BackendLocation(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLocation(t)
|
||||
}
|
||||
|
||||
func TestS3BackendConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestConfig(t)
|
||||
}
|
||||
|
||||
func TestS3BackendLoad(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestS3BackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSave(t)
|
||||
}
|
||||
|
||||
func TestS3BackendSaveFilenames(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSaveFilenames(t)
|
||||
}
|
||||
|
||||
func TestS3BackendBackend(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestBackend(t)
|
||||
}
|
||||
|
||||
func TestS3BackendDelete(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestDelete(t)
|
||||
}
|
||||
|
||||
func TestS3BackendCleanup(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCleanup(t)
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to an s3 compatible
|
||||
// server.
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
UseHTTP bool
|
||||
KeyID, Secret string
|
||||
Bucket string
|
||||
Prefix string
|
||||
}
|
||||
|
||||
const defaultPrefix = "restic"
|
||||
|
||||
// ParseConfig parses the string s and extracts the s3 config. The two
|
||||
// supported configuration formats are s3://host/bucketname/prefix and
|
||||
// s3:host:bucketname/prefix. The host can also be a valid s3 region
|
||||
// name. If no prefix is given the prefix "restic" will be used.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
switch {
|
||||
case strings.HasPrefix(s, "s3:http"):
|
||||
// assume that a URL has been specified, parse it and
|
||||
// use the host as the endpoint and the path as the
|
||||
// bucket name and prefix
|
||||
url, err := url.Parse(s[3:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if url.Path == "" {
|
||||
return nil, errors.New("s3: bucket name not found")
|
||||
}
|
||||
|
||||
path := strings.SplitN(url.Path[1:], "/", 2)
|
||||
return createConfig(url.Host, path, url.Scheme == "http")
|
||||
case strings.HasPrefix(s, "s3://"):
|
||||
s = s[5:]
|
||||
case strings.HasPrefix(s, "s3:"):
|
||||
s = s[3:]
|
||||
default:
|
||||
return nil, errors.New("s3: invalid format")
|
||||
}
|
||||
// use the first entry of the path as the endpoint and the
|
||||
// remainder as bucket name and prefix
|
||||
path := strings.SplitN(s, "/", 3)
|
||||
return createConfig(path[0], path[1:], false)
|
||||
}
|
||||
|
||||
func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
|
||||
var prefix string
|
||||
switch {
|
||||
case len(p) < 1:
|
||||
return nil, errors.New("s3: invalid format, host/region or bucket name not found")
|
||||
case len(p) == 1 || p[1] == "":
|
||||
prefix = defaultPrefix
|
||||
default:
|
||||
prefix = path.Clean(p[1])
|
||||
}
|
||||
return Config{
|
||||
Endpoint: endpoint,
|
||||
UseHTTP: useHTTP,
|
||||
Bucket: p[0],
|
||||
Prefix: prefix,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package s3
|
||||
|
||||
import "testing"
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"s3://eu-central-1/bucketname", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "restic",
|
||||
}},
|
||||
{"s3://eu-central-1/bucketname/", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "restic",
|
||||
}},
|
||||
{"s3://eu-central-1/bucketname/prefix/directory", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
}},
|
||||
{"s3://eu-central-1/bucketname/prefix/directory/", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
}},
|
||||
{"s3:eu-central-1/foobar", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "restic",
|
||||
}},
|
||||
{"s3:eu-central-1/foobar/", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "restic",
|
||||
}},
|
||||
{"s3:eu-central-1/foobar/prefix/directory", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "prefix/directory",
|
||||
}},
|
||||
{"s3:eu-central-1/foobar/prefix/directory/", Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "prefix/directory",
|
||||
}},
|
||||
{"s3:https://hostname:9999/foobar", Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "restic",
|
||||
}},
|
||||
{"s3:https://hostname:9999/foobar/", Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "restic",
|
||||
}},
|
||||
{"s3:http://hostname:9999/foobar", Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "restic",
|
||||
UseHTTP: true,
|
||||
}},
|
||||
{"s3:http://hostname:9999/foobar/", Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "restic",
|
||||
UseHTTP: true,
|
||||
}},
|
||||
{"s3:http://hostname:9999/bucket/prefix/directory", Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "bucket",
|
||||
Prefix: "prefix/directory",
|
||||
UseHTTP: true,
|
||||
}},
|
||||
{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "bucket",
|
||||
Prefix: "prefix/directory",
|
||||
UseHTTP: true,
|
||||
}},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/debug"
|
||||
)
|
||||
|
||||
const connLimit = 10
|
||||
|
||||
// s3 is a backend which stores the data on an S3 endpoint.
|
||||
type s3 struct {
|
||||
client minio.CloudStorageClient
|
||||
connChan chan struct{}
|
||||
bucketname string
|
||||
prefix string
|
||||
}
|
||||
|
||||
// Open opens the S3 backend at bucket and region. The bucket is created if it
|
||||
// does not exist yet.
|
||||
func Open(cfg Config) (backend.Backend, error) {
|
||||
debug.Log("s3.Open", "open, config %#v", cfg)
|
||||
|
||||
client, err := minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, cfg.UseHTTP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
|
||||
be.createConnections()
|
||||
|
||||
if err := client.BucketExists(cfg.Bucket); err != nil {
|
||||
debug.Log("s3.Open", "BucketExists(%v) returned err %v, trying to create the bucket", cfg.Bucket, err)
|
||||
|
||||
// create new bucket with default ACL in default region
|
||||
err = client.MakeBucket(cfg.Bucket, "", "")
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
func (be *s3) s3path(t backend.Type, name string) string {
|
||||
var path string
|
||||
|
||||
if be.prefix != "" {
|
||||
path = be.prefix + "/"
|
||||
}
|
||||
path += string(t)
|
||||
|
||||
if t == backend.Config {
|
||||
return path
|
||||
}
|
||||
return path + "/" + name
|
||||
}
|
||||
|
||||
func (be *s3) createConnections() {
|
||||
be.connChan = make(chan struct{}, connLimit)
|
||||
for i := 0; i < connLimit; i++ {
|
||||
be.connChan <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the bucket name).
|
||||
func (be *s3) Location() string {
|
||||
return be.bucketname
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) {
|
||||
debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p))
|
||||
path := be.s3path(h.Type, h.Name)
|
||||
obj, err := be.client.GetObject(be.bucketname, path)
|
||||
if err != nil {
|
||||
debug.Log("s3.GetReader", " err %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if off > 0 {
|
||||
_, err = obj.Seek(off, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
<-be.connChan
|
||||
defer func() {
|
||||
be.connChan <- struct{}{}
|
||||
}()
|
||||
return io.ReadFull(obj, p)
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (be s3) Save(h backend.Handle, p []byte) (err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
debug.Log("s3.Save", "%v bytes at %d", len(p), h)
|
||||
|
||||
path := be.s3path(h.Type, h.Name)
|
||||
|
||||
// Check key does not already exist
|
||||
_, err = be.client.StatObject(be.bucketname, path)
|
||||
if err == nil {
|
||||
debug.Log("s3.blob.Finalize()", "%v already exists", h)
|
||||
return errors.New("key already exists")
|
||||
}
|
||||
|
||||
<-be.connChan
|
||||
defer func() {
|
||||
be.connChan <- struct{}{}
|
||||
}()
|
||||
|
||||
debug.Log("s3.Save", "PutObject(%v, %v, %v, %v)",
|
||||
be.bucketname, path, int64(len(p)), "binary/octet-stream")
|
||||
n, err := be.client.PutObject(be.bucketname, path, bytes.NewReader(p), "binary/octet-stream")
|
||||
debug.Log("s3.Save", "%v -> %v bytes, err %#v", path, n, err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
|
||||
debug.Log("s3.Stat", "%v")
|
||||
path := be.s3path(h.Type, h.Name)
|
||||
obj, err := be.client.GetObject(be.bucketname, path)
|
||||
if err != nil {
|
||||
debug.Log("s3.Stat", "GetObject() err %v", err)
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
fi, err := obj.Stat()
|
||||
if err != nil {
|
||||
debug.Log("s3.Stat", "Stat() err %v", err)
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
return backend.BlobInfo{Size: fi.Size}, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (be *s3) Test(t backend.Type, name string) (bool, error) {
|
||||
found := false
|
||||
path := be.s3path(t, name)
|
||||
_, err := be.client.StatObject(be.bucketname, path)
|
||||
if err == nil {
|
||||
found = true
|
||||
}
|
||||
|
||||
// If error, then not found
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (be *s3) Remove(t backend.Type, name string) error {
|
||||
path := be.s3path(t, name)
|
||||
err := be.client.RemoveObject(be.bucketname, path)
|
||||
debug.Log("s3.Remove", "%v %v -> err %v", t, name, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine is started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
debug.Log("s3.List", "listing %v", t)
|
||||
ch := make(chan string)
|
||||
|
||||
prefix := be.s3path(t, "")
|
||||
|
||||
listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for obj := range listresp {
|
||||
m := strings.TrimPrefix(obj.Key, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- m:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Remove keys for a specified backend type.
|
||||
func (be *s3) removeKeys(t backend.Type) error {
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
for key := range be.List(backend.Data, done) {
|
||||
err := be.Remove(backend.Data, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
|
||||
func (be *s3) Delete() error {
|
||||
alltypes := []backend.Type{
|
||||
backend.Data,
|
||||
backend.Key,
|
||||
backend.Lock,
|
||||
backend.Snapshot,
|
||||
backend.Index}
|
||||
|
||||
for _, t := range alltypes {
|
||||
err := be.removeKeys(t)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return be.Remove(backend.Config, "")
|
||||
}
|
||||
|
||||
// Close does nothing
|
||||
func (be *s3) Close() error { return nil }
|
||||
@@ -1,72 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/s3"
|
||||
"github.com/restic/restic/backend/test"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
//go:generate go run ../test/generate_backend_tests.go
|
||||
|
||||
func init() {
|
||||
if TestS3Server == "" {
|
||||
SkipMessage = "s3 test server not available"
|
||||
return
|
||||
}
|
||||
|
||||
url, err := url.Parse(TestS3Server)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s3.Config{
|
||||
Endpoint: url.Host,
|
||||
Bucket: "restictestbucket",
|
||||
KeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
|
||||
Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
}
|
||||
|
||||
if url.Scheme == "http" {
|
||||
cfg.UseHTTP = true
|
||||
}
|
||||
|
||||
test.CreateFn = func() (backend.Backend, error) {
|
||||
be, err := s3.Open(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
exists, err := be.Test(backend.Config, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if exists {
|
||||
return nil, errors.New("config already exists")
|
||||
}
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
test.OpenFn = func() (backend.Backend, error) {
|
||||
return s3.Open(cfg)
|
||||
}
|
||||
|
||||
// test.CleanupFn = func() error {
|
||||
// if tempBackendDir == "" {
|
||||
// return nil
|
||||
// }
|
||||
|
||||
// fmt.Printf("removing test backend at %v\n", tempBackendDir)
|
||||
// err := os.RemoveAll(tempBackendDir)
|
||||
// tempBackendDir = ""
|
||||
// return err
|
||||
// }
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package sftp_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
func TestSftpBackendCreate(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreate(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendOpen(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestOpen(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendCreateWithConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreateWithConfig(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendLocation(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLocation(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestConfig(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendLoad(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSave(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendSaveFilenames(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSaveFilenames(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendBackend(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestBackend(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendDelete(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestDelete(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendCleanup(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCleanup(t)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Config collects all information required to connect to an sftp server.
|
||||
type Config struct {
|
||||
User, Host, Dir string
|
||||
}
|
||||
|
||||
// ParseConfig extracts all information for the sftp connection from the string s.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
if strings.HasPrefix(s, "sftp://") {
|
||||
return parseFormat1(s)
|
||||
}
|
||||
|
||||
// otherwise parse in the sftp:user@host:path format, which means we'll get
|
||||
// "user@host:path" in s
|
||||
return parseFormat2(s)
|
||||
}
|
||||
|
||||
// parseFormat1 parses the first format, starting with a slash, so the user
|
||||
// either specified "sftp://host/path", so we'll get everything after the first
|
||||
// colon character
|
||||
func parseFormat1(s string) (Config, error) {
|
||||
url, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
cfg := Config{
|
||||
Host: url.Host,
|
||||
Dir: url.Path[1:],
|
||||
}
|
||||
if url.User != nil {
|
||||
cfg.User = url.User.Username()
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// parseFormat2 parses the second format, sftp:user@host:path
|
||||
func parseFormat2(s string) (cfg Config, err error) {
|
||||
// split user/host and path at the second colon
|
||||
data := strings.SplitN(s, ":", 3)
|
||||
if len(data) < 3 {
|
||||
return Config{}, errors.New("sftp: invalid format, hostname or path not found")
|
||||
}
|
||||
|
||||
if data[0] != "sftp" {
|
||||
return Config{}, errors.New(`invalid format, does not start with "sftp:"`)
|
||||
}
|
||||
|
||||
userhost := data[1]
|
||||
cfg.Dir = data[2]
|
||||
|
||||
data = strings.SplitN(userhost, "@", 2)
|
||||
if len(data) == 2 {
|
||||
cfg.User = data[0]
|
||||
cfg.Host = data[1]
|
||||
} else {
|
||||
cfg.Host = userhost
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package sftp
|
||||
|
||||
import "testing"
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
// first form, user specified sftp://user@host/dir
|
||||
{
|
||||
"sftp://user@host/dir/subdir",
|
||||
Config{User: "user", Host: "host", Dir: "dir/subdir"},
|
||||
},
|
||||
{
|
||||
"sftp://host/dir/subdir",
|
||||
Config{Host: "host", Dir: "dir/subdir"},
|
||||
},
|
||||
{
|
||||
"sftp://host//dir/subdir",
|
||||
Config{Host: "host", Dir: "/dir/subdir"},
|
||||
},
|
||||
{
|
||||
"sftp://host:10022//dir/subdir",
|
||||
Config{Host: "host:10022", Dir: "/dir/subdir"},
|
||||
},
|
||||
{
|
||||
"sftp://user@host:10022//dir/subdir",
|
||||
Config{User: "user", Host: "host:10022", Dir: "/dir/subdir"},
|
||||
},
|
||||
|
||||
// second form, user specified sftp:user@host:/dir
|
||||
{
|
||||
"sftp:foo@bar:/baz/quux",
|
||||
Config{User: "foo", Host: "bar", Dir: "/baz/quux"},
|
||||
},
|
||||
{
|
||||
"sftp:bar:../baz/quux",
|
||||
Config{Host: "bar", Dir: "../baz/quux"},
|
||||
},
|
||||
{
|
||||
"sftp:fux@bar:baz/qu:ux",
|
||||
Config{User: "fux", Host: "bar", Dir: "baz/qu:ux"},
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d failed: %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d: wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
// Package sftp implements repository storage in a directory on a remote server
|
||||
// via the sftp protocol.
|
||||
package sftp
|
||||
@@ -1,465 +0,0 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/juju/errors"
|
||||
"github.com/pkg/sftp"
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/debug"
|
||||
)
|
||||
|
||||
const (
|
||||
tempfileRandomSuffixLength = 10
|
||||
)
|
||||
|
||||
// SFTP is a backend in a directory accessed via SFTP.
|
||||
type SFTP struct {
|
||||
c *sftp.Client
|
||||
p string
|
||||
|
||||
cmd *exec.Cmd
|
||||
}
|
||||
|
||||
func startClient(program string, args ...string) (*SFTP, error) {
|
||||
// Connect to a remote host and request the sftp subsystem via the 'ssh'
|
||||
// command. This assumes that passwordless login is correctly configured.
|
||||
cmd := exec.Command(program, args...)
|
||||
|
||||
// send errors from ssh to stderr
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
// ignore signals sent to the parent (e.g. SIGINT)
|
||||
cmd.SysProcAttr = ignoreSigIntProcAttr()
|
||||
|
||||
// get stdin and stdout
|
||||
wr, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
rd, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// start the process
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// open the SFTP session
|
||||
client, err := sftp.NewClientPipe(rd, wr)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
return &SFTP{c: client, cmd: cmd}, nil
|
||||
}
|
||||
|
||||
func paths(dir string) []string {
|
||||
return []string{
|
||||
dir,
|
||||
Join(dir, backend.Paths.Data),
|
||||
Join(dir, backend.Paths.Snapshots),
|
||||
Join(dir, backend.Paths.Index),
|
||||
Join(dir, backend.Paths.Locks),
|
||||
Join(dir, backend.Paths.Keys),
|
||||
Join(dir, backend.Paths.Temp),
|
||||
}
|
||||
}
|
||||
|
||||
// Open opens an sftp backend. When the command is started via
|
||||
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
|
||||
// is expected at the given path.
|
||||
func Open(dir string, program string, args ...string) (*SFTP, error) {
|
||||
sftp, err := startClient(program, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// test if all necessary dirs and files are there
|
||||
for _, d := range paths(dir) {
|
||||
if _, err := sftp.c.Lstat(d); err != nil {
|
||||
return nil, fmt.Errorf("%s does not exist", d)
|
||||
}
|
||||
}
|
||||
|
||||
sftp.p = dir
|
||||
return sftp, nil
|
||||
}
|
||||
|
||||
func buildSSHCommand(cfg Config) []string {
|
||||
hostport := strings.Split(cfg.Host, ":")
|
||||
args := []string{hostport[0]}
|
||||
if len(hostport) > 1 {
|
||||
args = append(args, "-p", hostport[1])
|
||||
}
|
||||
if cfg.User != "" {
|
||||
args = append(args, "-l")
|
||||
args = append(args, cfg.User)
|
||||
}
|
||||
args = append(args, "-s")
|
||||
args = append(args, "sftp")
|
||||
return args
|
||||
}
|
||||
|
||||
// OpenWithConfig opens an sftp backend as described by the config by running
|
||||
// "ssh" with the appropiate arguments.
|
||||
func OpenWithConfig(cfg Config) (*SFTP, error) {
|
||||
return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
|
||||
}
|
||||
|
||||
// Create creates all the necessary files and directories for a new sftp
|
||||
// backend at dir. Afterwards a new config blob should be created.
|
||||
func Create(dir string, program string, args ...string) (*SFTP, error) {
|
||||
sftp, err := startClient(program, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// test if config file already exists
|
||||
_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
|
||||
if err == nil {
|
||||
return nil, errors.New("config file already exists")
|
||||
}
|
||||
|
||||
// create paths for data, refs and temp blobs
|
||||
for _, d := range paths(dir) {
|
||||
err = sftp.mkdirAll(d, backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = sftp.c.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = sftp.cmd.Wait()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open backend
|
||||
return Open(dir, program, args...)
|
||||
}
|
||||
|
||||
// CreateWithConfig creates an sftp backend as described by the config by running
|
||||
// "ssh" with the appropiate arguments.
|
||||
func CreateWithConfig(cfg Config) (*SFTP, error) {
|
||||
return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the directory name).
|
||||
func (r *SFTP) Location() string {
|
||||
return r.p
|
||||
}
|
||||
|
||||
// Return temp directory in correct directory for this backend.
|
||||
func (r *SFTP) tempFile() (string, *sftp.File, error) {
|
||||
// choose random suffix
|
||||
buf := make([]byte, tempfileRandomSuffixLength)
|
||||
_, err := io.ReadFull(rand.Reader, buf)
|
||||
if err != nil {
|
||||
return "", nil, errors.Annotatef(err,
|
||||
"unable to read %d random bytes for tempfile name",
|
||||
tempfileRandomSuffixLength)
|
||||
}
|
||||
|
||||
// construct tempfile name
|
||||
name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))
|
||||
|
||||
// create file in temp dir
|
||||
f, err := r.c.Create(name)
|
||||
if err != nil {
|
||||
return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
|
||||
}
|
||||
|
||||
return name, f, nil
|
||||
}
|
||||
|
||||
func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
|
||||
// check if directory already exists
|
||||
fi, err := r.c.Lstat(dir)
|
||||
if err == nil {
|
||||
if fi.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
|
||||
}
|
||||
|
||||
// create parent directories
|
||||
errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)
|
||||
|
||||
// create directory
|
||||
errMkdir := r.c.Mkdir(dir)
|
||||
|
||||
// test if directory was created successfully
|
||||
fi, err = r.c.Lstat(dir)
|
||||
if err != nil {
|
||||
// return previous errors
|
||||
return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
|
||||
}
|
||||
|
||||
if !fi.IsDir() {
|
||||
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
|
||||
}
|
||||
|
||||
// set mode
|
||||
return r.c.Chmod(dir, mode)
|
||||
}
|
||||
|
||||
// Rename temp file to final name according to type and name.
|
||||
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
|
||||
filename := r.filename(t, name)
|
||||
|
||||
// create directories if necessary
|
||||
if t == backend.Data {
|
||||
err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// test if new file exists
|
||||
if _, err := r.c.Lstat(filename); err == nil {
|
||||
return fmt.Errorf("Close(): file %v already exists", filename)
|
||||
}
|
||||
|
||||
err := r.c.Rename(oldname, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set mode to read-only
|
||||
fi, err := r.c.Lstat(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
|
||||
}
|
||||
|
||||
// Join joins the given paths and cleans them afterwards.
|
||||
func Join(parts ...string) string {
|
||||
return filepath.Clean(strings.Join(parts, "/"))
|
||||
}
|
||||
|
||||
// Construct path for given backend.Type and name.
|
||||
func (r *SFTP) filename(t backend.Type, name string) string {
|
||||
if t == backend.Config {
|
||||
return Join(r.p, "config")
|
||||
}
|
||||
|
||||
return Join(r.dirname(t, name), name)
|
||||
}
|
||||
|
||||
// Construct directory for given backend.Type.
|
||||
func (r *SFTP) dirname(t backend.Type, name string) string {
|
||||
var n string
|
||||
switch t {
|
||||
case backend.Data:
|
||||
n = backend.Paths.Data
|
||||
if len(name) > 2 {
|
||||
n = Join(n, name[:2])
|
||||
}
|
||||
case backend.Snapshot:
|
||||
n = backend.Paths.Snapshots
|
||||
case backend.Index:
|
||||
n = backend.Paths.Index
|
||||
case backend.Lock:
|
||||
n = backend.Paths.Locks
|
||||
case backend.Key:
|
||||
n = backend.Paths.Keys
|
||||
}
|
||||
return Join(r.p, n)
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f, err := r.c.Open(r.filename(h.Type, h.Name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
e := f.Close()
|
||||
if err == nil && e != nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
|
||||
if off > 0 {
|
||||
_, err = f.Seek(off, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return io.ReadFull(f, p)
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (r *SFTP) Save(h backend.Handle, p []byte) (err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
filename, tmpfile, err := r.tempFile()
|
||||
debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename)
|
||||
|
||||
n, err := tmpfile.Write(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != len(p) {
|
||||
return errors.New("not all bytes writen")
|
||||
}
|
||||
|
||||
err = tmpfile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = r.renameFile(filename, h.Type, h.Name)
|
||||
debug.Log("sftp.Save", "save %v: rename %v: %v",
|
||||
h, filepath.Base(filename), err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: renameFile: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
|
||||
if err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
return backend.BlobInfo{Size: fi.Size()}, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
|
||||
_, err := r.c.Lstat(r.filename(t, name))
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Remove removes the content stored at name.
|
||||
func (r *SFTP) Remove(t backend.Type, name string) error {
|
||||
return r.c.Remove(r.filename(t, name))
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine is started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
ch := make(chan string)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
if t == backend.Data {
|
||||
// read first level
|
||||
basedir := r.dirname(t, "")
|
||||
|
||||
list1, err := r.c.ReadDir(basedir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
dirs := make([]string, 0, len(list1))
|
||||
for _, d := range list1 {
|
||||
dirs = append(dirs, d.Name())
|
||||
}
|
||||
|
||||
// read files
|
||||
for _, dir := range dirs {
|
||||
entries, err := r.c.ReadDir(Join(basedir, dir))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
items := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
items = append(items, entry.Name())
|
||||
}
|
||||
|
||||
for _, file := range items {
|
||||
select {
|
||||
case ch <- file:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
entries, err := r.c.ReadDir(r.dirname(t, ""))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
items := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
items = append(items, entry.Name())
|
||||
}
|
||||
|
||||
for _, file := range items {
|
||||
select {
|
||||
case ch <- file:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
|
||||
}
|
||||
|
||||
// Close closes the sftp connection and terminates the underlying command.
|
||||
func (r *SFTP) Close() error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := r.c.Close()
|
||||
debug.Log("sftp.Close", "Close returned error %v", err)
|
||||
|
||||
if err := r.cmd.Process.Kill(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return r.cmd.Wait()
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package sftp_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/sftp"
|
||||
"github.com/restic/restic/backend/test"
|
||||
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
var tempBackendDir string
|
||||
|
||||
//go:generate go run ../test/generate_backend_tests.go
|
||||
|
||||
func createTempdir() error {
|
||||
if tempBackendDir != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
tempdir, err := ioutil.TempDir("", "restic-local-test-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("created new test backend at %v\n", tempdir)
|
||||
tempBackendDir = tempdir
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
sftpserver := ""
|
||||
|
||||
for _, dir := range strings.Split(TestSFTPPath, ":") {
|
||||
testpath := filepath.Join(dir, "sftp-server")
|
||||
_, err := os.Stat(testpath)
|
||||
if !os.IsNotExist(err) {
|
||||
sftpserver = testpath
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if sftpserver == "" {
|
||||
SkipMessage = "sftp server binary not found, skipping tests"
|
||||
return
|
||||
}
|
||||
|
||||
test.CreateFn = func() (backend.Backend, error) {
|
||||
err := createTempdir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sftp.Create(tempBackendDir, sftpserver)
|
||||
}
|
||||
|
||||
test.OpenFn = func() (backend.Backend, error) {
|
||||
err := createTempdir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sftp.Open(tempBackendDir, sftpserver)
|
||||
}
|
||||
|
||||
test.CleanupFn = func() error {
|
||||
if tempBackendDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("removing test backend at %v\n", tempBackendDir)
|
||||
err := os.RemoveAll(tempBackendDir)
|
||||
tempBackendDir = ""
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
// +build !windows
|
||||
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// ignoreSigIntProcAttr returns a syscall.SysProcAttr that
|
||||
// disables SIGINT on parent.
|
||||
func ignoreSigIntProcAttr() *syscall.SysProcAttr {
|
||||
return &syscall.SysProcAttr{Setsid: true}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// ignoreSigIntProcAttr returns a default syscall.SysProcAttr
|
||||
// on Windows.
|
||||
func ignoreSigIntProcAttr() *syscall.SysProcAttr {
|
||||
return &syscall.SysProcAttr{}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package test_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
func TestTestBackendCreate(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreate(t)
|
||||
}
|
||||
|
||||
func TestTestBackendOpen(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestOpen(t)
|
||||
}
|
||||
|
||||
func TestTestBackendCreateWithConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreateWithConfig(t)
|
||||
}
|
||||
|
||||
func TestTestBackendLocation(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLocation(t)
|
||||
}
|
||||
|
||||
func TestTestBackendConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestConfig(t)
|
||||
}
|
||||
|
||||
func TestTestBackendLoad(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestTestBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSave(t)
|
||||
}
|
||||
|
||||
func TestTestBackendSaveFilenames(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSaveFilenames(t)
|
||||
}
|
||||
|
||||
func TestTestBackendBackend(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestBackend(t)
|
||||
}
|
||||
|
||||
func TestTestBackendDelete(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestDelete(t)
|
||||
}
|
||||
|
||||
func TestTestBackendCleanup(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCleanup(t)
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"text/template"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var data struct {
|
||||
Package string
|
||||
PackagePrefix string
|
||||
Funcs []string
|
||||
}
|
||||
|
||||
var testTemplate = `
|
||||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package {{ .Package }}
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
{{ $prefix := .PackagePrefix }}
|
||||
{{ range $f := .Funcs }}
|
||||
func Test{{ $prefix }}{{ $f }}(t *testing.T){
|
||||
if SkipMessage != "" { t.Skip(SkipMessage) }
|
||||
test.Test{{ $f }}(t)
|
||||
}
|
||||
|
||||
{{ end }}
|
||||
`
|
||||
|
||||
var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in")
|
||||
var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to")
|
||||
var packageName = flag.String("package", "", "the package name to use")
|
||||
var prefix = flag.String("prefix", "", "test function prefix")
|
||||
var quiet = flag.Bool("quiet", false, "be quiet")
|
||||
|
||||
func errx(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var funcRegex = regexp.MustCompile(`^func\s+Test(.+)\s*\(`)
|
||||
|
||||
func findTestFunctions() (funcs []string) {
|
||||
f, err := os.Open(*testFile)
|
||||
errx(err)
|
||||
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
match := funcRegex.FindStringSubmatch(sc.Text())
|
||||
if len(match) > 0 {
|
||||
funcs = append(funcs, match[1])
|
||||
}
|
||||
}
|
||||
|
||||
if err := sc.Err(); err != nil {
|
||||
log.Fatalf("Error scanning file: %v", err)
|
||||
}
|
||||
|
||||
errx(f.Close())
|
||||
return funcs
|
||||
}
|
||||
|
||||
func generateOutput(wr io.Writer, data interface{}) {
|
||||
t := template.Must(template.New("backendtest").Parse(testTemplate))
|
||||
|
||||
cmd := exec.Command("gofmt")
|
||||
cmd.Stdout = wr
|
||||
in, err := cmd.StdinPipe()
|
||||
errx(err)
|
||||
errx(cmd.Start())
|
||||
errx(t.Execute(in, data))
|
||||
errx(in.Close())
|
||||
errx(cmd.Wait())
|
||||
}
|
||||
|
||||
func packageTestFunctionPrefix(pkg string) string {
|
||||
if pkg == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
r, n := utf8.DecodeRuneInString(pkg)
|
||||
return string(unicode.ToUpper(r)) + pkg[n:]
|
||||
}
|
||||
|
||||
func init() {
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
func main() {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Getwd() %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
pkg := *packageName
|
||||
if pkg == "" {
|
||||
pkg = filepath.Base(dir)
|
||||
}
|
||||
|
||||
f, err := os.Create(*outputFile)
|
||||
errx(err)
|
||||
|
||||
data.Package = pkg + "_test"
|
||||
if *prefix != "" {
|
||||
data.PackagePrefix = *prefix
|
||||
} else {
|
||||
data.PackagePrefix = packageTestFunctionPrefix(pkg) + "Backend"
|
||||
}
|
||||
data.Funcs = findTestFunctions()
|
||||
generateOutput(f, data)
|
||||
|
||||
errx(f.Close())
|
||||
|
||||
if !*quiet {
|
||||
fmt.Printf("wrote backend tests for package %v to %v\n", data.Package, *outputFile)
|
||||
}
|
||||
}
|
||||
@@ -1,525 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
var CreateFn func() (backend.Backend, error)
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
var OpenFn func() (backend.Backend, error)
|
||||
|
||||
// CleanupFn removes temporary files and directories created during the tests.
|
||||
var CleanupFn func() error
|
||||
|
||||
var but backend.Backend // backendUnderTest
|
||||
var butInitialized bool
|
||||
|
||||
func open(t testing.TB) backend.Backend {
|
||||
if OpenFn == nil {
|
||||
t.Fatal("OpenFn not set")
|
||||
}
|
||||
|
||||
if CreateFn == nil {
|
||||
t.Fatalf("CreateFn not set")
|
||||
}
|
||||
|
||||
if !butInitialized {
|
||||
be, err := CreateFn()
|
||||
if err != nil {
|
||||
t.Fatalf("Create returned unexpected error: %v", err)
|
||||
}
|
||||
|
||||
but = be
|
||||
butInitialized = true
|
||||
}
|
||||
|
||||
if but == nil {
|
||||
var err error
|
||||
but, err = OpenFn()
|
||||
if err != nil {
|
||||
t.Fatalf("Open returned unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return but
|
||||
}
|
||||
|
||||
func close(t testing.TB) {
|
||||
if but == nil {
|
||||
t.Fatalf("trying to close non-existing backend")
|
||||
}
|
||||
|
||||
err := but.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Close returned unexpected error: %v", err)
|
||||
}
|
||||
|
||||
but = nil
|
||||
}
|
||||
|
||||
// TestCreate creates a backend.
|
||||
func TestCreate(t testing.TB) {
|
||||
if CreateFn == nil {
|
||||
t.Fatalf("CreateFn not set!")
|
||||
}
|
||||
|
||||
be, err := CreateFn()
|
||||
if err != nil {
|
||||
fmt.Printf("foo\n")
|
||||
t.Fatalf("Create returned error: %v", err)
|
||||
}
|
||||
|
||||
butInitialized = true
|
||||
|
||||
err = be.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Close returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpen opens a previously created backend.
|
||||
func TestOpen(t testing.TB) {
|
||||
if OpenFn == nil {
|
||||
t.Fatalf("OpenFn not set!")
|
||||
}
|
||||
|
||||
be, err := OpenFn()
|
||||
if err != nil {
|
||||
t.Fatalf("Open returned error: %v", err)
|
||||
}
|
||||
|
||||
err = be.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Close returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCreateWithConfig tests that creating a backend in a location which already
|
||||
// has a config file fails.
|
||||
func TestCreateWithConfig(t testing.TB) {
|
||||
if CreateFn == nil {
|
||||
t.Fatalf("CreateFn not set")
|
||||
}
|
||||
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
// save a config
|
||||
store(t, b, backend.Config, []byte("test config"))
|
||||
|
||||
// now create the backend again, this must fail
|
||||
_, err := CreateFn()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error not found for creating a backend with an existing config file")
|
||||
}
|
||||
|
||||
// remove config
|
||||
err = b.Remove(backend.Config, "")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error removing config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocation tests that a location string is returned.
|
||||
func TestLocation(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
l := b.Location()
|
||||
if l == "" {
|
||||
t.Fatalf("invalid location string %q", l)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfig saves and loads a config from the backend.
|
||||
func TestConfig(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
var testString = "Config"
|
||||
|
||||
// create config and read it back
|
||||
_, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("did not get expected error for non-existing config")
|
||||
}
|
||||
|
||||
err = b.Save(backend.Handle{Type: backend.Config}, []byte(testString))
|
||||
if err != nil {
|
||||
t.Fatalf("Save() error: %v", err)
|
||||
}
|
||||
|
||||
// try accessing the config with different names, should all return the
|
||||
// same config
|
||||
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
|
||||
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to read config with name %q: %v", name, err)
|
||||
}
|
||||
|
||||
if string(buf) != testString {
|
||||
t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoad tests the backend's Load function.
|
||||
func TestLoad(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
_, err := b.Load(backend.Handle{}, nil, 0)
|
||||
if err == nil {
|
||||
t.Fatalf("Load() did not return an error for invalid handle")
|
||||
}
|
||||
|
||||
_, err = b.Load(backend.Handle{Type: backend.Data, Name: "foobar"}, nil, 0)
|
||||
if err == nil {
|
||||
t.Fatalf("Load() did not return an error for non-existing blob")
|
||||
}
|
||||
|
||||
length := rand.Intn(1<<24) + 2000
|
||||
|
||||
data := Random(23, length)
|
||||
id := backend.Hash(data)
|
||||
|
||||
handle := backend.Handle{Type: backend.Data, Name: id.String()}
|
||||
err = b.Save(handle, data)
|
||||
if err != nil {
|
||||
t.Fatalf("Save() error: %v", err)
|
||||
}
|
||||
|
||||
for i := 0; i < 50; i++ {
|
||||
l := rand.Intn(length + 2000)
|
||||
o := rand.Intn(length + 2000)
|
||||
|
||||
d := data
|
||||
if o < len(d) {
|
||||
d = d[o:]
|
||||
} else {
|
||||
o = len(d)
|
||||
d = d[:0]
|
||||
}
|
||||
|
||||
if l > 0 && l < len(d) {
|
||||
d = d[:l]
|
||||
}
|
||||
|
||||
buf := make([]byte, l)
|
||||
n, err := b.Load(handle, buf, int64(o))
|
||||
|
||||
// if we requested data beyond the end of the file, ignore
|
||||
// ErrUnexpectedEOF error
|
||||
if l > len(d) && err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
buf = buf[:len(d)]
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
|
||||
continue
|
||||
}
|
||||
|
||||
if n != len(buf) {
|
||||
t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
|
||||
len(buf), int64(o), len(buf), n)
|
||||
continue
|
||||
}
|
||||
|
||||
buf = buf[:n]
|
||||
if !bytes.Equal(buf, d) {
|
||||
t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// load with a too-large buffer, this should return io.ErrUnexpectedEOF
|
||||
buf := make([]byte, length+100)
|
||||
n, err := b.Load(handle, buf, 0)
|
||||
if n != length {
|
||||
t.Errorf("wrong length for larger buffer returned, want %d, got %d", length, n)
|
||||
}
|
||||
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
t.Errorf("wrong error returned for larger buffer: want io.ErrUnexpectedEOF, got %#v", err)
|
||||
}
|
||||
|
||||
OK(t, b.Remove(backend.Data, id.String()))
|
||||
}
|
||||
|
||||
// TestSave tests saving data in the backend.
|
||||
func TestSave(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
var id backend.ID
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
length := rand.Intn(1<<23) + 200000
|
||||
data := Random(23, length)
|
||||
// use the first 32 byte as the ID
|
||||
copy(id[:], data)
|
||||
|
||||
h := backend.Handle{
|
||||
Type: backend.Data,
|
||||
Name: fmt.Sprintf("%s-%d", id, i),
|
||||
}
|
||||
err := b.Save(h, data)
|
||||
OK(t, err)
|
||||
|
||||
buf, err := backend.LoadAll(b, h, nil)
|
||||
OK(t, err)
|
||||
if len(buf) != len(data) {
|
||||
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Fatalf("data not equal")
|
||||
}
|
||||
|
||||
fi, err := b.Stat(h)
|
||||
OK(t, err)
|
||||
|
||||
if fi.Size != int64(len(data)) {
|
||||
t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
|
||||
}
|
||||
|
||||
err = b.Remove(h.Type, h.Name)
|
||||
if err != nil {
|
||||
t.Fatalf("error removing item: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var filenameTests = []struct {
|
||||
name string
|
||||
data string
|
||||
}{
|
||||
{"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"},
|
||||
{"foobar", "foobar"},
|
||||
{
|
||||
"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5",
|
||||
"foobar content of data blob",
|
||||
},
|
||||
}
|
||||
|
||||
// TestSaveFilenames tests saving data with various file names in the backend.
|
||||
func TestSaveFilenames(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
for i, test := range filenameTests {
|
||||
h := backend.Handle{Name: test.name, Type: backend.Data}
|
||||
err := b.Save(h, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("test %d failed: Save() returned %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
buf, err := backend.LoadAll(b, h, nil)
|
||||
if err != nil {
|
||||
t.Errorf("test %d failed: Load() returned %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, []byte(test.data)) {
|
||||
t.Errorf("test %d: returned wrong bytes", i)
|
||||
}
|
||||
|
||||
err = b.Remove(h.Type, h.Name)
|
||||
if err != nil {
|
||||
t.Errorf("test %d failed: Remove() returned %v", i, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var testStrings = []struct {
|
||||
id string
|
||||
data string
|
||||
}{
|
||||
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
|
||||
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
|
||||
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
|
||||
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
|
||||
}
|
||||
|
||||
func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) {
|
||||
id := backend.Hash(data)
|
||||
err := b.Save(backend.Handle{Name: id.String(), Type: tpe}, data)
|
||||
OK(t, err)
|
||||
}
|
||||
|
||||
func read(t testing.TB, rd io.Reader, expectedData []byte) {
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
OK(t, err)
|
||||
if expectedData != nil {
|
||||
Equals(t, expectedData, buf)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBackend tests all functions of the backend.
|
||||
func TestBackend(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
for _, tpe := range []backend.Type{
|
||||
backend.Data, backend.Key, backend.Lock,
|
||||
backend.Snapshot, backend.Index,
|
||||
} {
|
||||
// detect non-existing files
|
||||
for _, test := range testStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
OK(t, err)
|
||||
|
||||
// test if blob is already in repository
|
||||
ret, err := b.Test(tpe, id.String())
|
||||
OK(t, err)
|
||||
Assert(t, !ret, "blob was found to exist before creating")
|
||||
|
||||
// try to stat a not existing blob
|
||||
h := backend.Handle{Type: tpe, Name: id.String()}
|
||||
_, err = b.Stat(h)
|
||||
Assert(t, err != nil, "blob data could be extracted before creation")
|
||||
|
||||
// try to read not existing blob
|
||||
_, err = b.Load(h, nil, 0)
|
||||
Assert(t, err != nil, "blob reader could be obtained before creation")
|
||||
|
||||
// try to get string out, should fail
|
||||
ret, err = b.Test(tpe, id.String())
|
||||
OK(t, err)
|
||||
Assert(t, !ret, "id %q was found (but should not have)", test.id)
|
||||
}
|
||||
|
||||
// add files
|
||||
for _, test := range testStrings {
|
||||
store(t, b, tpe, []byte(test.data))
|
||||
|
||||
// test Load()
|
||||
h := backend.Handle{Type: tpe, Name: test.id}
|
||||
buf, err := backend.LoadAll(b, h, nil)
|
||||
OK(t, err)
|
||||
Equals(t, test.data, string(buf))
|
||||
|
||||
// try to read it out with an offset and a length
|
||||
start := 1
|
||||
end := len(test.data) - 2
|
||||
length := end - start
|
||||
|
||||
buf2 := make([]byte, length)
|
||||
n, err := b.Load(h, buf2, int64(start))
|
||||
OK(t, err)
|
||||
Equals(t, length, n)
|
||||
Equals(t, test.data[start:end], string(buf2))
|
||||
}
|
||||
|
||||
// test adding the first file again
|
||||
test := testStrings[0]
|
||||
|
||||
// create blob
|
||||
err := b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data))
|
||||
Assert(t, err != nil, "expected error, got %v", err)
|
||||
|
||||
// remove and recreate
|
||||
err = b.Remove(tpe, test.id)
|
||||
OK(t, err)
|
||||
|
||||
// test that the blob is gone
|
||||
ok, err := b.Test(tpe, test.id)
|
||||
OK(t, err)
|
||||
Assert(t, ok == false, "removed blob still present")
|
||||
|
||||
// create blob
|
||||
err = b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data))
|
||||
OK(t, err)
|
||||
|
||||
// list items
|
||||
IDs := backend.IDs{}
|
||||
|
||||
for _, test := range testStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
OK(t, err)
|
||||
IDs = append(IDs, id)
|
||||
}
|
||||
|
||||
list := backend.IDs{}
|
||||
|
||||
for s := range b.List(tpe, nil) {
|
||||
list = append(list, ParseID(s))
|
||||
}
|
||||
|
||||
if len(IDs) != len(list) {
|
||||
t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
|
||||
}
|
||||
|
||||
sort.Sort(IDs)
|
||||
sort.Sort(list)
|
||||
|
||||
if !reflect.DeepEqual(IDs, list) {
|
||||
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
|
||||
}
|
||||
|
||||
// remove content if requested
|
||||
if TestCleanupTempDirs {
|
||||
for _, test := range testStrings {
|
||||
id, err := backend.ParseID(test.id)
|
||||
OK(t, err)
|
||||
|
||||
found, err := b.Test(tpe, id.String())
|
||||
OK(t, err)
|
||||
|
||||
OK(t, b.Remove(tpe, id.String()))
|
||||
|
||||
found, err = b.Test(tpe, id.String())
|
||||
OK(t, err)
|
||||
Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete tests the Delete function.
|
||||
func TestDelete(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
be, ok := b.(backend.Deleter)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
err := be.Delete()
|
||||
if err != nil {
|
||||
t.Fatalf("error deleting backend: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCleanup runs the cleanup function after all tests are run.
|
||||
func TestCleanup(t testing.TB) {
|
||||
if CleanupFn == nil {
|
||||
t.Log("CleanupFn function not set")
|
||||
return
|
||||
}
|
||||
|
||||
if !TestCleanupTempDirs {
|
||||
t.Logf("not cleaning up backend")
|
||||
return
|
||||
}
|
||||
|
||||
err := CleanupFn()
|
||||
if err != nil {
|
||||
t.Fatalf("Cleanup returned error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package test_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/mem"
|
||||
"github.com/restic/restic/backend/test"
|
||||
)
|
||||
|
||||
var be backend.Backend
|
||||
|
||||
//go:generate go run ../test/generate_backend_tests.go
|
||||
|
||||
func init() {
|
||||
test.CreateFn = func() (backend.Backend, error) {
|
||||
if be != nil {
|
||||
return nil, errors.New("temporary memory backend dir already exists")
|
||||
}
|
||||
|
||||
be = mem.New()
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
test.OpenFn = func() (backend.Backend, error) {
|
||||
if be == nil {
|
||||
return nil, errors.New("repository not initialized")
|
||||
}
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
test.CleanupFn = func() error {
|
||||
be = nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package backend
|
||||
|
||||
import "io"
|
||||
|
||||
// LoadAll reads all data stored in the backend for the handle. The buffer buf
|
||||
// is resized to accomodate all data in the blob. Errors returned by be.Load()
|
||||
// are passed on, except io.ErrUnexpectedEOF is silenced and nil returned
|
||||
// instead, since it means this function is working properly.
|
||||
func LoadAll(be Backend, h Handle, buf []byte) ([]byte, error) {
|
||||
fi, err := be.Stat(h)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fi.Size > int64(len(buf)) {
|
||||
buf = make([]byte, int(fi.Size))
|
||||
}
|
||||
|
||||
n, err := be.Load(h, buf, 0)
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
buf = buf[:n]
|
||||
return buf, err
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
"github.com/restic/restic/backend/mem"
|
||||
. "github.com/restic/restic/test"
|
||||
)
|
||||
|
||||
const KiB = 1 << 10
|
||||
const MiB = 1 << 20
|
||||
|
||||
func TestLoadAll(t *testing.T) {
|
||||
b := mem.New()
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
data := Random(23+i, rand.Intn(MiB)+500*KiB)
|
||||
|
||||
id := backend.Hash(data)
|
||||
err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data)
|
||||
OK(t, err)
|
||||
|
||||
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, nil)
|
||||
OK(t, err)
|
||||
|
||||
if len(buf) != len(data) {
|
||||
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
|
||||
continue
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Errorf("wrong data returned")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSmallBuffer(t *testing.T) {
|
||||
b := mem.New()
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
data := Random(23+i, rand.Intn(MiB)+500*KiB)
|
||||
|
||||
id := backend.Hash(data)
|
||||
err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data)
|
||||
OK(t, err)
|
||||
|
||||
buf := make([]byte, len(data)-23)
|
||||
buf, err = backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, buf)
|
||||
OK(t, err)
|
||||
|
||||
if len(buf) != len(data) {
|
||||
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
|
||||
continue
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Errorf("wrong data returned")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadLargeBuffer(t *testing.T) {
|
||||
b := mem.New()
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
data := Random(23+i, rand.Intn(MiB)+500*KiB)
|
||||
|
||||
id := backend.Hash(data)
|
||||
err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data)
|
||||
OK(t, err)
|
||||
|
||||
buf := make([]byte, len(data)+100)
|
||||
buf, err = backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, buf)
|
||||
OK(t, err)
|
||||
|
||||
if len(buf) != len(data) {
|
||||
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
|
||||
continue
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Errorf("wrong data returned")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user