amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-08-14 17:43:17 +00:00
Code Issues Projects Releases Wiki Activity

Move build.go and run_integration_tests.go to root

Browse Source
This commit is contained in:
Alexander Neumann
2016-02-20 17:31:21 +01:00
parent 7b6629802b
commit 841326d713
2 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/restic/build.go
-377
View File
@@ -1,377 +0,0 @@
// +build ignore
package main
import (
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"
"time"
)
var (
verbose bool
keepGopath bool
runTests bool
)
const timeFormat = "2006-01-02 15:04:05"
// specialDir returns true if the file begins with a special character ('.' or '_').
func specialDir(name string) bool {
if name == "." {
return false
}
base := filepath.Base(name)
return base[0] == '_' || base[0] == '.'
}
// excludePath returns true if the file should not be copied to the new GOPATH.
func excludePath(name string) bool {
ext := path.Ext(name)
if ext == ".go" || ext == ".s" {
return false
}
parentDir := filepath.Base(filepath.Dir(name))
if parentDir == "testdata" {
return false
}
return true
}
// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
// to dst/prefix/, so calling
//
// updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
//
// with "/home/u/restic" containing the file "foo.go" yields the following tree
// at "/tmp/gopath":
//
// /tmp/gopath
// └── src
// └── github.com
// └── restic
// └── restic
// └── foo.go
func updateGopath(dst, src, prefix string) error {
return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
if specialDir(name) {
if fi.IsDir() {
return filepath.SkipDir
}
return nil
}
if fi.IsDir() {
return nil
}
if excludePath(name) {
return nil
}
intermediatePath, err := filepath.Rel(src, name)
if err != nil {
return err
}
fileSrc := filepath.Join(src, intermediatePath)
fileDst := filepath.Join(dst, "src", prefix, intermediatePath)
return copyFile(fileDst, fileSrc)
})
}
// copyFile creates dst from src, preserving file attributes and timestamps.
func copyFile(dst, src string) error {
fi, err := os.Stat(src)
if err != nil {
return err
}
fsrc, err := os.Open(src)
if err != nil {
return err
}
if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
return err
}
fdst, err := os.Create(dst)
if err != nil {
return err
}
if _, err = io.Copy(fdst, fsrc); err != nil {
return err
}
if err == nil {
err = fsrc.Close()
}
if err == nil {
err = fdst.Close()
}
if err == nil {
err = os.Chmod(dst, fi.Mode())
}
if err == nil {
err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
}
return nil
}
// die prints the message with fmt.Fprintf() to stderr and exits with an error
// code.
func die(message string, args ...interface{}) {
fmt.Fprintf(os.Stderr, message, args...)
os.Exit(1)
}
func showUsage(output io.Writer) {
fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
fmt.Fprintf(output, "\n")
fmt.Fprintf(output, "OPTIONS:\n")
fmt.Fprintf(output, " -v --verbose output more messages\n")
fmt.Fprintf(output, " -t --tags specify additional build tags\n")
fmt.Fprintf(output, " -k --keep-gopath do not remove the GOPATH after build\n")
fmt.Fprintf(output, " -T --test run tests\n")
}
func verbosePrintf(message string, args ...interface{}) {
if !verbose {
return
}
fmt.Printf("build: "+message, args...)
}
// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
// present).
func cleanEnv() (env []string) {
for _, v := range os.Environ() {
if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
continue
}
env = append(env, v)
}
return env
}
// build runs "go build args..." with GOPATH set to gopath.
func build(gopath string, args ...string) error {
args = append([]string{"build"}, args...)
cmd := exec.Command("go", args...)
cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
verbosePrintf("go %s\n", args)
return cmd.Run()
}
// test runs "go test args..." with GOPATH set to gopath.
func test(gopath string, args ...string) error {
args = append([]string{"test"}, args...)
cmd := exec.Command("go", args...)
cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
verbosePrintf("go %s\n", args)
return cmd.Run()
}
// getVersion returns the version string from the file VERSION in the current
// directory.
func getVersionFromFile() string {
buf, err := ioutil.ReadFile("VERSION")
if err != nil {
verbosePrintf("error reading file VERSION: %v\n", err)
return ""
}
return strings.TrimSpace(string(buf))
}
// getVersion returns a version string which is a combination of the contents
// of the file VERSION in the current directory and the version from git (if
// available).
func getVersion() string {
versionFile := getVersionFromFile()
versionGit := getVersionFromGit()
verbosePrintf("version from file 'VERSION' is %q, version from git %q\n",
versionFile, versionGit)
switch {
case versionFile == "":
return versionGit
case versionGit == "":
return versionFile
}
return fmt.Sprintf("%s (%s)", versionFile, versionGit)
}
// getVersionFromGit returns a version string that identifies the currently
// checked out git commit.
func getVersionFromGit() string {
cmd := exec.Command("git", "describe",
"--long", "--tags", "--dirty", "--always")
out, err := cmd.Output()
if err != nil {
verbosePrintf("git describe returned error: %v\n", err)
return ""
}
version := strings.TrimSpace(string(out))
verbosePrintf("git version is %s\n", version)
return version
}
// Constants represents a set of constants that are set in the final binary to
// the given value via compiler flags.
type Constants map[string]string
// LDFlags returns the string that can be passed to go build's `-ldflags`.
func (cs Constants) LDFlags() string {
l := make([]string, 0, len(cs))
v := runtime.Version()
if strings.HasPrefix(v, "go1.5") || strings.HasPrefix(v, "go1.6") {
for k, v := range cs {
l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
}
} else {
for k, v := range cs {
l = append(l, fmt.Sprintf(`-X %q %q`, k, v))
}
}
return strings.Join(l, " ")
}
func main() {
buildTags := []string{}
skipNext := false
params := os.Args[1:]
for i, arg := range params {
if skipNext {
skipNext = false
continue
}
switch arg {
case "-v", "--verbose":
verbose = true
case "-k", "--keep-gopath":
keepGopath = true
case "-t", "-tags", "--tags":
skipNext = true
buildTags = strings.Split(params[i+1], " ")
case "-T", "--test":
runTests = true
case "-h":
showUsage(os.Stdout)
default:
fmt.Fprintf(os.Stderr, "Error: unknown option %q\n\n", arg)
showUsage(os.Stderr)
os.Exit(1)
}
}
if len(buildTags) == 0 {
verbosePrintf("adding build-tag release\n")
buildTags = []string{"release"}
}
for i := range buildTags {
buildTags[i] = strings.TrimSpace(buildTags[i])
}
verbosePrintf("build tags: %s\n", buildTags)
root, err := os.Getwd()
if err != nil {
die("Getwd(): %v\n", err)
}
gopath, err := ioutil.TempDir("", "restic-build-")
if err != nil {
die("TempDir(): %v\n", err)
}
verbosePrintf("create GOPATH at %v\n", gopath)
if err = updateGopath(gopath, root, "github.com/restic/restic"); err != nil {
die("copying files from %v to %v failed: %v\n", root, gopath, err)
}
vendor := filepath.Join(root, "Godeps", "_workspace", "src")
if err = updateGopath(gopath, vendor, ""); err != nil {
die("copying files from %v to %v failed: %v\n", root, gopath, err)
}
defer func() {
if !keepGopath {
verbosePrintf("remove %v\n", gopath)
if err = os.RemoveAll(gopath); err != nil {
die("remove GOPATH at %s failed: %v\n", err)
}
} else {
verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
}
}()
output := "restic"
if runtime.GOOS == "windows" {
output = "restic.exe"
}
version := getVersion()
compileTime := time.Now().Format(timeFormat)
constants := Constants{`main.compiledAt`: compileTime}
if version != "" {
constants["main.version"] = version
}
ldflags := "-s " + constants.LDFlags()
verbosePrintf("ldflags: %s\n", ldflags)
args := []string{
"-tags", strings.Join(buildTags, " "),
"-ldflags", ldflags,
"-o", output, "github.com/restic/restic/cmd/restic",
}
err = build(gopath, args...)
if err != nil {
die("build failed: %v\n", err)
}
if runTests {
verbosePrintf("running tests\n")
err = test(gopath, "github.com/restic/restic/...")
if err != nil {
die("running tests failed: %v\n", err)
}
}
}
src/restic/run_integration_tests.go
-426
View File
@@ -1,426 +0,0 @@
// +build ignore
package main
import (
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
)
var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests")
var minioServer = flag.String("minio", "", "path to the minio server binary")
func init() {
flag.Parse()
}
type CIEnvironment interface {
Prepare()
RunTests()
}
type TravisEnvironment struct {
goxArch []string
goxOS []string
minio string
}
func (env *TravisEnvironment) getMinio() {
if *minioServer != "" {
msg("using minio server at %q\n", *minioServer)
env.minio = *minioServer
return
}
tempfile, err := ioutil.TempFile("", "minio-server-")
if err != nil {
fmt.Fprintf(os.Stderr, "create tempfile failed: %v\n", err)
os.Exit(10)
}
url := fmt.Sprintf("https://dl.minio.io/server/minio/release/%s-%s/minio",
runtime.GOOS, runtime.GOARCH)
msg("downloading %v\n", url)
res, err := http.Get(url)
if err != nil {
msg("downloading minio failed: %v\n", err)
return
}
_, err = io.Copy(tempfile, res.Body)
if err != nil {
msg("downloading minio failed: %v\n", err)
return
}
err = res.Body.Close()
if err != nil {
msg("saving minio failed: %v\n", err)
return
}
err = tempfile.Close()
if err != nil {
msg("closing tempfile failed: %v\n", err)
return
}
err = os.Chmod(tempfile.Name(), 0755)
if err != nil {
msg("making minio server executable failed: %v\n", err)
return
}
msg("downloaded minio server to %v\n", tempfile.Name())
env.minio = tempfile.Name()
}
func (env *TravisEnvironment) Prepare() {
msg("preparing environment for Travis CI\n")
run("go", "get", "golang.org/x/tools/cmd/cover")
run("go", "get", "github.com/mattn/goveralls")
run("go", "get", "github.com/pierrre/gotestcover")
env.getMinio()
if runtime.GOOS == "darwin" {
// install the libraries necessary for fuse
run("brew", "update")
run("brew", "install", "caskroom/cask/brew-cask")
run("brew", "cask", "install", "osxfuse")
}
if *runCrossCompile {
// only test cross compilation on linux with Travis
run("go", "get", "github.com/mitchellh/gox")
if runtime.GOOS == "linux" {
env.goxArch = []string{"386", "amd64"}
if !strings.HasPrefix(runtime.Version(), "go1.3") {
env.goxArch = append(env.goxArch, "arm")
}
env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"}
} else {
env.goxArch = []string{runtime.GOARCH}
env.goxOS = []string{runtime.GOOS}
}
msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch)
v := runtime.Version()
if !strings.HasPrefix(v, "go1.5") && !strings.HasPrefix(v, "go1.6") {
run("gox", "-build-toolchain",
"-os", strings.Join(env.goxOS, " "),
"-arch", strings.Join(env.goxArch, " "))
}
}
}
func goVersionAtLeast151() bool {
v := runtime.Version()
if match, _ := regexp.MatchString(`^go1\.[0-4]`, v); match {
return false
}
if v == "go1.5" {
return false
}
return true
}
type MinioServer struct {
cmd *exec.Cmd
done bool
m sync.Mutex
}
func (env *TravisEnvironment) RunTests() {
// run fuse tests on darwin
if runtime.GOOS != "darwin" {
msg("skip fuse integration tests on %v\n", runtime.GOOS)
os.Setenv("RESTIC_TEST_FUSE", "0")
}
if *runCrossCompile {
// compile for all target architectures with tags
for _, tags := range []string{"release", "debug"} {
run("gox", "-verbose",
"-os", strings.Join(env.goxOS, " "),
"-arch", strings.Join(env.goxArch, " "),
"-tags", tags,
"-output", "/tmp/{{.Dir}}_{{.OS}}_{{.Arch}}",
"./cmd/restic")
}
}
// run the build script
run("go", "run", "build.go")
var (
testEnv map[string]string
srv *MinioServer
err error
)
if env.minio != "" {
srv, err = NewMinioServer(env.minio)
if err != nil {
fmt.Fprintf(os.Stderr, "error running minio server: %v", err)
os.Exit(8)
}
testEnv = minioEnv
}
// run the tests and gather coverage information
runWithEnv(testEnv, "gotestcover", "-coverprofile", "all.cov", "./...")
runGofmt()
srv.Stop()
}
type AppveyorEnvironment struct{}
func (env *AppveyorEnvironment) Prepare() {
msg("preparing environment for Appveyor CI\n")
}
func (env *AppveyorEnvironment) RunTests() {
run("go", "run", "build.go", "-v", "-T")
}
// findGoFiles returns a list of go source code file names below dir.
func findGoFiles(dir string) (list []string, err error) {
err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
if filepath.Base(name) == "Godeps" {
return filepath.SkipDir
}
if filepath.Ext(name) == ".go" {
relpath, err := filepath.Rel(dir, name)
if err != nil {
return err
}
list = append(list, relpath)
}
return err
})
return list, err
}
func msg(format string, args ...interface{}) {
fmt.Printf("CI: "+format, args...)
}
func updateEnv(env []string, override map[string]string) []string {
var newEnv []string
for _, s := range env {
d := strings.SplitN(s, "=", 2)
key := d[0]
if _, ok := override[key]; ok {
continue
}
newEnv = append(newEnv, s)
}
for k, v := range override {
newEnv = append(newEnv, k+"="+v)
}
return newEnv
}
func runGofmt() {
dir, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "Getwd(): %v\n", err)
os.Exit(5)
}
files, err := findGoFiles(dir)
if err != nil {
fmt.Fprintf(os.Stderr, "error finding Go files: %v\n", err)
os.Exit(4)
}
msg("runGofmt() with %d files\n", len(files))
args := append([]string{"-l"}, files...)
cmd := exec.Command("gofmt", args...)
cmd.Stderr = os.Stderr
buf, err := cmd.Output()
if err != nil {
fmt.Fprintf(os.Stderr, "error running gofmt: %v", err)
fmt.Fprintf(os.Stderr, "output:\n%s\n", buf)
os.Exit(3)
}
if len(buf) > 0 {
fmt.Fprintf(os.Stderr, "not formatted with `gofmt`:\n")
fmt.Fprintln(os.Stderr, string(buf))
os.Exit(6)
}
}
func run(command string, args ...string) {
msg("run %v %v\n", command, strings.Join(args, " "))
runWithEnv(nil, command, args...)
}
// runWithEnv calls a command with the current environment, except the entries
// of the env map are set additionally.
func runWithEnv(env map[string]string, command string, args ...string) {
msg("runWithEnv %v %v\n", command, strings.Join(args, " "))
cmd := exec.Command(command, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if env != nil {
cmd.Env = updateEnv(os.Environ(), env)
}
err := cmd.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "error running %v %v: %v",
command, strings.Join(args, " "), err)
os.Exit(3)
}
}
var minioConfig = `
{
"version": "2",
"credentials": {
"accessKeyId": "KEBIYDZ87HCIH5D17YCN",
"secretAccessKey": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe"
}
}
`
var minioEnv = map[string]string{
"RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000",
"AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN",
"AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
}
// NewMinioServer prepares and runs a minio server for the s3 backend tests in
// a temporary directory.
func NewMinioServer(minio string) (*MinioServer, error) {
msg("running minio server\n")
cfgdir, err := ioutil.TempDir("", "minio-config-")
if err != nil {
return nil, err
}
cfg, err := os.Create(filepath.Join(cfgdir, "config.json"))
if err != nil {
return nil, err
}
_, err = cfg.Write([]byte(minioConfig))
if err != nil {
return nil, err
}
err = cfg.Close()
if err != nil {
return nil, err
}
dir, err := ioutil.TempDir("", "minio-root")
if err != nil {
return nil, err
}
out := bytes.NewBuffer(nil)
cmd := exec.Command(minio,
"--config-folder", cfgdir,
"--address", "127.0.0.1:9000",
"server", dir)
cmd.Stdout = out
cmd.Stderr = out
err = cmd.Start()
if err != nil {
return nil, err
}
srv := &MinioServer{cmd: cmd}
go srv.Wait()
return srv, nil
}
func (m *MinioServer) Stop() {
if m == nil {
return
}
msg("stopping minio server\n")
m.m.Lock()
m.done = true
m.m.Unlock()
err := m.cmd.Process.Kill()
if err != nil {
fmt.Fprintf(os.Stderr, "error stopping minio server: %v", err)
os.Exit(8)
}
}
func (m *MinioServer) Wait() {
err := m.cmd.Wait()
msg("minio server exited\n")
m.m.Lock()
done := m.done
m.m.Unlock()
if err != nil && !done {
fmt.Fprintf(os.Stderr, "error running minio server: %#v, output:\n", err)
// io.Copy(os.Stderr, out)
os.Exit(12)
}
}
func isTravis() bool {
return os.Getenv("TRAVIS_BUILD_DIR") != ""
}
func isAppveyor() bool {
return runtime.GOOS == "windows"
}
func main() {
var env CIEnvironment
switch {
case isTravis():
env = &TravisEnvironment{}
case isAppveyor():
env = &AppveyorEnvironment{}
default:
fmt.Fprintln(os.Stderr, "unknown CI environment")
os.Exit(1)
}
for _, f := range []func(){env.Prepare, env.RunTests} {
f()
}
}