add VERSION file for 0.1.0

WIP: cleanup readme

.travis.yml

@@ -4,13 +4,12 @@ sudo: false
 go:
   - 1.3.3
   - 1.4.2
+  - 1.5
 
 os:
   - linux
   - osx
 
-env: GOX_OS="linux darwin openbsd freebsd" GOX_ARCH="386 amd64 arm"
-
 notifications:
   irc:
     channels:
@@ -25,22 +24,9 @@ install:
   - export PATH="$PATH:$GOBIN"
   - export GOPATH="$GOPATH:${TRAVIS_BUILD_DIR}/Godeps/_workspace"
   - go env
-  - go get github.com/mattn/goveralls
-  - go get github.com/mitchellh/gox
-  - go version | grep -q "go1\.3" && export GOX_ARCH="386 amd64" || true
-  - go version | grep -q "darwin" && export GOX_OS="darwin" || true
-  - uname -s | grep -qi darwin && brew install caskroom/cask/brew-cask || true
-  - uname -s | grep -qi darwin && brew cask install osxfuse || true
-  - uname -s | grep -vqi darwin && export RESTIC_TEST_FUSE="0" || true
-  - echo "cross-compile for \"$GOX_OS\" on \"$GOX_ARCH\""
-  - gox -build-toolchain -os "$GOX_OS" -arch "$GOX_ARCH"
 
 script:
-  - gox -verbose -os "$GOX_OS" -arch "$GOX_ARCH" -tags "release" ./cmd/restic
-  - gox -verbose -os "$GOX_OS" -arch "$GOX_ARCH" -tags "debug" ./cmd/restic
-  - go run build.go
-  - go run run_tests.go all.cov
-  - GOARCH=386 RESTIC_TEST_INTEGRATION=0 go test ./...
+  - go run run_integration_tests.go
+
+after_success:
   - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true
-  - gofmt -l *.go */*.go */*/*.go
-  - test -z "$(gofmt -l *.go */*.go */*/*.go)"

Makefile

@@ -1,22 +1,12 @@
 .PHONY: all clean test
 
-SOURCE=$(wildcard *.go) $(wildcard */*.go) $(wildcard */*/*.go)
-
-export GOPATH GOX_OS
-
 all: restic
 
 restic: $(SOURCE)
 	go run build.go
 
-restic.debug: $(SOURCE)
-	go run build.go -tags debug
-
 clean:
-	rm -rf restic restic.debug
+	rm -rf restic
 
 test: $(SOURCE)
 	go run run_tests.go /dev/null
-
-all.cov: $(SOURCE)
-	go run run_tests.go all.cov

README.md

@@ -1,26 +1,22 @@
 [![Stories in Ready](https://badge.waffle.io/restic/restic.png?label=ready&title=Ready)](https://waffle.io/restic/restic)
 [![Build Status](https://travis-ci.org/restic/restic.svg?branch=master)](https://travis-ci.org/restic/restic)
+[![Build status](https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true)](https://ci.appveyor.com/project/fd0/restic/branch/master)
 [![sourcegraph status](https://sourcegraph.com/api/repos/github.com/restic/restic/.badges/status.png)](https://sourcegraph.com/github.com/restic/restic)
 [![Coverage Status](https://coveralls.io/repos/restic/restic/badge.svg)](https://coveralls.io/r/restic/restic)
 
-WARNING
-=======
+Restic Design Principles
+========================
 
-WARNING: At the moment, consider restic as alpha quality software, it is not
-yet finished. Do not use it for real data!
+Restic is a program that does backups right and was designed with the following
+principles in mind:
 
-Restic
-======
-
-Restic is a program that does backups right. The design goals are:
-
- * Easy: Doing backups should be a frictionless process, otherwise you are
-   tempted to skip it.  Restic should be easy to configure and use, so that in
-   the unlikely event of a data loss you can just restore it. Likewise,
+ * Easy: Doing backups should be a frictionless process, otherwise you might be
+   tempted to skip it.  Restic should be easy to configure and use, so that, in
+   the event of a data loss, you can just restore it. Likewise,
    restoring data should not be complicated.
 
  * Fast: Backing up your data with restic should only be limited by your
-   network or harddisk bandwidth so that you can backup your files every day.
+   network or hard disk bandwidth so that you can backup your files every day.
    Nobody does backups if it takes too much time. Restoring backups should only
    transfer data that is needed for the files that are to be restored, so that
    this process is also fast.
@@ -36,12 +32,12 @@ Restic is a program that does backups right. The design goals are:
 
  * Efficient: With the growth of data, additional snapshots should only take
    the storage of the actual increment. Even more, duplicate data should be
-   de-duplicated before it is actually written to the storage backend to save
+   de-duplicated before it is actually written to the storage back end to save
    precious backup space.
 
 
-Building
-========
+Build restic
+============
 
 Install Go/Golang (at least version 1.3), then run `go run build.go`,
 afterwards you'll find the binary in the current directory:
@@ -76,20 +72,27 @@ afterwards you'll find the binary in the current directory:
 A short demo recording can be found here:
 [![asciicast](https://asciinema.org/a/23554.png)](https://asciinema.org/a/23554)
 
+Compatibility
+=============
+
+Backward compatibility for backups is important so that our users are always
+able to restore saved data. Therefore restic follows [Semantic
+Versioning](http://semver.org) to clearly define which versions are compatible.
+The repository and data structures contained therein are considered the "Public
+API" in the sense of Semantic Versioning.
+
+We guarantee backward compatibility of all repositories within one major version;
+as long as we do not increment the major version, data can be read and restored.
+We strive to be fully backward compatible to all prior versions.
+
 Contribute and Documentation
 ============================
 
 Contributions are welcome! More information can be found in
 [`CONTRIBUTING.md`](CONTRIBUTING.md). A document describing the design of
-restic and the data structures stored on disc is contained in
+restic and the data structures stored on the backend is contained in
 [`doc/Design.md`](doc/Design.md).
-
-Development
-===========
-
-For development, please have a look at [`CONTRIBUTING.md`](CONTRIBUTING.md),
-especially the section "Development Environment". If you have any questions,
-please get in touch!
+The development environment is described in [`CONTRIBUTING.md`](CONTRIBUTING.md).
 
 Contact
 =======

VERSION

@@ -1 +1 @@
-0.0.1alpha
+0.1.0

appveyor.yml

@@ -0,0 +1,14 @@
+clone_folder: c:\gopath\src\github.com\restic\restic
+
+environment:
+  GOPATH: c:\gopath;c:\gopath\src\github.com\restic\restic\Godeps\_workspace
+
+install:
+  - go version
+  - go env
+  - appveyor DownloadFile http://downloads.sourceforge.net/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
+  - 7z x tar.zip bin/tar.exe
+  - set PATH=bin/;%PATH%
+
+build_script:
+  - go run run_integration_tests.go

backend/local/local.go

@@ -267,6 +267,12 @@ func (b *Local) Remove(t backend.Type, name string) error {
 	b.open[fn] = nil
 	b.mu.Unlock()
 
+	// reset read-only flag
+	err := os.Chmod(fn, 0666)
+	if err != nil {
+		return err
+	}
+
 	return os.Remove(fn)
 }
 

build.go

@@ -18,6 +18,7 @@ import (
 var (
 	verbose    bool
 	keepGopath bool
+	runTests   bool
 )
 
 const timeFormat = "2006-01-02 15:04:05"
@@ -32,6 +33,21 @@ func specialDir(name string) bool {
 	return base[0] == '_' || base[0] == '.'
 }
 
+// excludePath returns true if the file should not be copied to the new GOPATH.
+func excludePath(name string) bool {
+	ext := path.Ext(name)
+	if ext == ".go" || ext == ".s" {
+		return false
+	}
+
+	parentDir := filepath.Base(filepath.Dir(name))
+	if parentDir == "testdata" {
+		return false
+	}
+
+	return true
+}
+
 // updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
 // to dst/prefix/, so calling
 //
@@ -60,8 +76,7 @@ func updateGopath(dst, src, prefix string) error {
 			return nil
 		}
 
-		ext := path.Ext(name)
-		if ext != ".go" && ext != ".s" {
+		if excludePath(name) {
 			return nil
 		}
 
@@ -133,7 +148,10 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
 	fmt.Fprintf(output, "\n")
 	fmt.Fprintf(output, "OPTIONS:\n")
-	fmt.Fprintf(output, "  -v     --verbose   output more messages\n")
+	fmt.Fprintf(output, "  -v     --verbose     output more messages\n")
+	fmt.Fprintf(output, "  -t     --tags        specify additional build tags\n")
+	fmt.Fprintf(output, "  -k     --keep-gopath do not remove the GOPATH after build\n")
+	fmt.Fprintf(output, "  -T     --test        run tests\n")
 }
 
 func verbosePrintf(message string, args ...interface{}) {
@@ -170,6 +188,18 @@ func build(gopath string, args ...string) error {
 	return cmd.Run()
 }
 
+// test runs "go test args..." with GOPATH set to gopath.
+func test(gopath string, args ...string) error {
+	args = append([]string{"test"}, args...)
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	verbosePrintf("go %s\n", args)
+
+	return cmd.Run()
+}
+
 // getVersion returns a version string, either from the file VERSION in the
 // current directory or from git.
 func getVersion() string {
@@ -218,6 +248,8 @@ func main() {
 		case "-t", "-tags", "--tags":
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-T", "--test":
+			runTests = true
 		case "-h":
 			showUsage(os.Stdout)
 		default:
@@ -258,6 +290,17 @@ func main() {
 		die("copying files from %v to %v failed: %v\n", root, gopath, err)
 	}
 
+	defer func() {
+		if !keepGopath {
+			verbosePrintf("remove %v\n", gopath)
+			if err = os.RemoveAll(gopath); err != nil {
+				die("remove GOPATH at %s failed: %v\n", err)
+			}
+		} else {
+			fmt.Printf("leaving temporary GOPATH at %v\n", gopath)
+		}
+	}()
+
 	output := "restic"
 	if runtime.GOOS == "windows" {
 		output = "restic.exe"
@@ -276,15 +319,15 @@ func main() {
 
 	err = build(gopath, args...)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "build failed: %v\n", err)
+		die("build failed: %v\n", err)
 	}
 
-	if !keepGopath {
-		verbosePrintf("remove %v\n", gopath)
-		if err = os.RemoveAll(gopath); err != nil {
-			die("remove GOPATH at %s failed: %v\n", err)
+	if runTests {
+		verbosePrintf("running tests\n")
+
+		err = test(gopath, "github.com/restic/restic/...")
+		if err != nil {
+			die("running tests failed: %v\n", err)
 		}
-	} else {
-		fmt.Printf("leaving temporary GOPATH at %v\n", gopath)
 	}
 }

cache.go

@@ -235,6 +235,8 @@ func getWindowsCacheDir() (string, error) {
 		if err != nil {
 			return "", err
 		}
+
+		return cachedir, nil
 	}
 
 	if err != nil {

doc/Design.md

@@ -6,15 +6,15 @@ Terminology
 
 This section introduces terminology used in this document.
 
-*Repository*: All data produced during a backup is sent to and stored at a
-repository in structured form, for example in a file system hierarchy of with
-several subdirectories. A repository implementation must be able to fulfil a
+*Repository*: All data produced during a backup is sent to and stored in a
+repository in a structured form, for example in a file system hierarchy with
+several subdirectories. A repository implementation must be able to fulfill a
 number of operations, e.g. list the contents.
 
 *Blob*: A Blob combines a number of data bytes with identifying information
 like the SHA256 hash of the data and its length.
 
-*Pack*: A Pack combines one or more Blobs together, e.g. in a single file.
+*Pack*: A Pack combines one or more Blobs, e.g. in a single file.
 
 *Snapshot*: A Snapshot stands for the state of a file or directory that has
 been backed up at some point in time. The state here means the content and meta
@@ -22,7 +22,7 @@ data like the name and modification time for the file or the directory and its
 contents.
 
 *Storage ID*: A storage ID is the SHA-256 hash of the content stored in the
-repository. This ID is needed in order to load the file from the repository.
+repository. This ID is required in order to load the file from the repository.
 
 Repository Format
 =================
@@ -36,19 +36,20 @@ parallel. Only the delete operation removes data from the repository.
 
 At the time of writing, the only implemented repository type is based on
 directories and files. Such repositories can be accessed locally on the same
-system or via the integrated SFTP client. The directory layout is the same for
-both access methods. This repository type is described in the following.
+system or via the integrated SFTP client (or any other storage back end).
+The directory layout is the same for both access methods.
+This repository type is described in the following section.
 
 Repositories consist of several directories and a file called `config`. For
 all other files stored in the repository, the name for the file is the lower
 case hexadecimal representation of the storage ID, which is the SHA-256 hash of
-the file's contents. This allows easily checking all files for accidental
-modifications like disk read errors by simply running the program `sha256sum`
+the file's contents. This allows for easy verification of files for accidental
+modifications, like disk read errors, by simply running the program `sha256sum`
 and comparing its output to the file name. If the prefix of a filename is
 unique amongst all the other files in the same directory, the prefix may be
 used instead of the complete filename.
 
-Apart from the files stored below the `keys` directory, all files are encrypted
+Apart from the files stored within the `keys` directory, all files are encrypted
 with AES-256 in counter mode (CTR). The integrity of the encrypted data is
 secured by a Poly1305-AES message authentication code (sometimes also referred
 to as a "signature").
@@ -398,7 +399,7 @@ required to create a lock on the repository before doing anything.
 
 Locks come in two types: Exclusive and non-exclusive locks. At most one
 process can have an exclusive lock on the repository, and during that time
-there mustn't be any other locks (exclusive and non-exclusive). There may be
+there must not be any other locks (exclusive and non-exclusive). There may be
 multiple non-exclusive locks in parallel.
 
 A lock is a file in the subdir `locks` whose filename is the storage ID of

lock.go

@@ -207,17 +207,10 @@ func (l *Lock) Stale() bool {
 		return false
 	}
 
-	proc, err := os.FindProcess(l.PID)
-	defer proc.Release()
-	if err != nil {
-		debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err)
-		return true
-	}
-
-	debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID)
-	err = proc.Signal(syscall.SIGHUP)
-	if err != nil {
-		debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err)
+	// check if we can reach the process retaining the lock
+	exists := l.processExists()
+	if !exists {
+		debug.Log("Lock.Stale", "could not reach process, %d, lock is probably stale\n", l.PID)
 		return true
 	}
 

lock_test.go

@@ -134,7 +134,7 @@ var staleLockTests = []struct {
 		timestamp:        time.Now(),
 		stale:            true,
 		staleOnOtherHost: false,
-		pid:              os.Getpid() + 500,
+		pid:              os.Getpid() + 500000,
 	},
 }
 
@@ -204,7 +204,7 @@ func TestRemoveAllLocks(t *testing.T) {
 	id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
 	OK(t, err)
 
-	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500)
+	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
 	OK(t, err)
 
 	OK(t, restic.RemoveAllLocks(repo))

run_integration_tests.go

@@ -0,0 +1,189 @@
+// +build ignore
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type CIEnvironment interface {
+	Prepare()
+	RunTests()
+}
+
+type TravisEnvironment struct {
+	goxArch []string
+	goxOS   []string
+}
+
+func (env *TravisEnvironment) Prepare() {
+	msg("preparing environment for Travis CI\n")
+
+	run("go", "get", "github.com/mattn/goveralls")
+	run("go", "get", "github.com/mitchellh/gox")
+
+	if runtime.GOOS == "darwin" {
+		// install the libraries necessary for fuse
+		run("brew", "install", "caskroom/cask/brew-cask")
+		run("brew", "cask", "install", "osxfuse")
+	}
+
+	// only test cross compilation on linux with Travis
+	if runtime.GOOS == "linux" {
+		env.goxArch = []string{"386", "amd64"}
+		if !strings.HasPrefix(runtime.Version(), "go1.3") {
+			env.goxArch = append(env.goxArch, "arm")
+		}
+
+		env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"}
+	} else {
+		env.goxArch = []string{runtime.GOARCH}
+		env.goxOS = []string{runtime.GOOS}
+	}
+
+	msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch)
+
+	if !strings.HasPrefix(runtime.Version(), "go1.5") {
+		run("gox", "-build-toolchain",
+			"-os", strings.Join(env.goxOS, " "),
+			"-arch", strings.Join(env.goxArch, " "))
+	}
+}
+
+func (env *TravisEnvironment) RunTests() {
+	// run fuse tests on darwin
+	if runtime.GOOS != "darwin" {
+		msg("skip fuse integration tests on %v\n", runtime.GOOS)
+		os.Setenv("RESTIC_TEST_FUSE", "0")
+	}
+
+	// compile for all target architectures with tags
+	for _, tags := range []string{"release", "debug"} {
+		run("gox", "-verbose",
+			"-os", strings.Join(env.goxOS, " "),
+			"-arch", strings.Join(env.goxArch, " "),
+			"-tags", tags,
+			"./cmd/restic")
+	}
+
+	// run the build script
+	run("go", "run", "build.go")
+
+	// gather coverage information
+	run("go", "run", "run_tests.go", "all.cov")
+
+	runGofmt()
+}
+
+type AppveyorEnvironment struct{}
+
+func (env *AppveyorEnvironment) Prepare() {
+	msg("preparing environment for Appveyor CI\n")
+}
+
+func (env *AppveyorEnvironment) RunTests() {
+	run("go", "run", "build.go", "-v", "-T")
+}
+
+// findGoFiles returns a list of go source code file names below dir.
+func findGoFiles(dir string) (list []string, err error) {
+	err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
+		if filepath.Base(name) == "Godeps" {
+			return filepath.SkipDir
+		}
+
+		if filepath.Ext(name) == ".go" {
+			relpath, err := filepath.Rel(dir, name)
+			if err != nil {
+				return err
+			}
+
+			list = append(list, relpath)
+		}
+
+		return err
+	})
+
+	return list, err
+}
+
+func msg(format string, args ...interface{}) {
+	fmt.Printf("CI: "+format, args...)
+}
+
+func runGofmt() {
+	dir, err := os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Getwd(): %v\n", err)
+		os.Exit(5)
+	}
+
+	files, err := findGoFiles(dir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error finding Go files: %v\n", err)
+		os.Exit(4)
+	}
+
+	msg("runGofmt() with %d files\n", len(files))
+	args := append([]string{"-l"}, files...)
+	cmd := exec.Command("gofmt", args...)
+	cmd.Stderr = os.Stderr
+
+	buf, err := cmd.Output()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error running gofmt: %v", err)
+		fmt.Fprintf(os.Stderr, "output:\n%s\n", buf)
+		os.Exit(3)
+	}
+
+	if len(buf) > 0 {
+		fmt.Fprintf(os.Stderr, "not formatted with `gofmt`:\n")
+		fmt.Fprintln(os.Stderr, string(buf))
+		os.Exit(6)
+	}
+}
+
+func run(command string, args ...string) {
+	msg("run %v %v\n", command, strings.Join(args, " "))
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error running %v %v: %v",
+			command, strings.Join(args, " "), err)
+		os.Exit(3)
+	}
+}
+
+func isTravis() bool {
+	return os.Getenv("TRAVIS_BUILD_DIR") != ""
+}
+
+func isAppveyor() bool {
+	return runtime.GOOS == "windows"
+}
+
+func main() {
+	var env CIEnvironment
+
+	switch {
+	case isTravis():
+		env = &TravisEnvironment{}
+	case isAppveyor():
+		env = &AppveyorEnvironment{}
+	default:
+		fmt.Fprintln(os.Stderr, "unknown CI environment")
+		os.Exit(1)
+	}
+
+	for _, f := range []func(){env.Prepare, env.RunTests} {
+		f()
+	}
+}

test/helpers.go

@@ -2,7 +2,10 @@ package test_helper
 
 import (
 	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -89,14 +92,28 @@ func RandomReader(seed, size int) *bytes.Reader {
 
 // SetupTarTestFixture extracts the tarFile to outputDir.
 func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
-	f, err := os.Open(tarFile)
-	defer f.Close()
+	input, err := os.Open(tarFile)
+	defer input.Close()
 	OK(t, err)
 
-	cmd := exec.Command("tar", "xzf", "-")
+	var rd io.Reader
+	switch filepath.Ext(tarFile) {
+	case ".gz":
+		r, err := gzip.NewReader(input)
+		OK(t, err)
+
+		defer r.Close()
+		rd = r
+	case ".bzip2":
+		rd = bzip2.NewReader(input)
+	default:
+		rd = input
+	}
+
+	cmd := exec.Command("tar", "xf", "-")
 	cmd.Dir = outputDir
 
-	cmd.Stdin = f
+	cmd.Stdin = rd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr