amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-07-09 00:35:10 +00:00
Code Issues Projects Releases Wiki Activity

Refactor backup pipeline

Browse Source
This commit is contained in:
Alexander Neumann
2015-03-07 11:53:32 +01:00
parent 1516cb5996
commit ba892e1ec2
9 changed files with 536 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipe/pipe.go
+59 -29
View File
@@ -10,22 +10,51 @@ import (
"github.com/restic/restic/debug"
)
type Entry struct {
Path string
Info os.FileInfo
Error error
Result chan<- interface{}
type Result interface{}
type Job interface {
Path() string
Fullpath() string
Error() error
Info() os.FileInfo
Result() chan<- Result
}
type Entry struct {
basedir string
path string
info os.FileInfo
error error
result chan<- Result
// points to the old node if available, interface{} is used to prevent
// circular import
Node interface{}
}
func (e Entry) Path() string { return e.path }
func (e Entry) Fullpath() string { return filepath.Join(e.basedir, e.path) }
func (e Entry) Error() error { return e.error }
func (e Entry) Info() os.FileInfo { return e.info }
func (e Entry) Result() chan<- Result { return e.result }
type Dir struct {
Path string
Error error
Info os.FileInfo
basedir string
path string
error error
info os.FileInfo
Entries [](<-chan interface{})
Result chan<- interface{}
Entries [](<-chan Result)
result chan<- Result
}
func (e Dir) Path() string { return e.path }
func (e Dir) Fullpath() string { return filepath.Join(e.basedir, e.path) }
func (e Dir) Error() error { return e.error }
func (e Dir) Info() os.FileInfo { return e.info }
func (e Dir) Result() chan<- Result { return e.result }
// readDirNames reads the directory named by dirname and returns
// a sorted list of directory entries.
// taken from filepath/path.go
@@ -53,15 +82,17 @@ func isFile(fi os.FileInfo) bool {
var errCancelled = errors.New("walk cancelled")
func walk(path string, done chan struct{}, jobs chan<- interface{}, res chan<- interface{}) error {
func walk(basedir, path string, done chan struct{}, jobs chan<- Job, res chan<- Result) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
relpath, _ := filepath.Rel(basedir, path)
if !info.IsDir() {
select {
case jobs <- Entry{Info: info, Path: path, Result: res}:
case jobs <- Entry{info: info, basedir: basedir, path: relpath, result: res}:
case <-done:
return errCancelled
}
@@ -73,18 +104,18 @@ func walk(path string, done chan struct{}, jobs chan<- interface{}, res chan<- i
return err
}
entries := make([]<-chan interface{}, 0, len(names))
entries := make([]<-chan Result, 0, len(names))
for _, name := range names {
subpath := filepath.Join(path, name)
ch := make(chan interface{}, 1)
ch := make(chan Result, 1)
entries = append(entries, ch)
fi, err := os.Lstat(subpath)
if err != nil {
select {
case jobs <- Entry{Info: fi, Error: err, Result: ch}:
case jobs <- Entry{info: fi, error: err, result: ch}:
case <-done:
return errCancelled
}
@@ -92,14 +123,14 @@ func walk(path string, done chan struct{}, jobs chan<- interface{}, res chan<- i
}
if isDir(fi) {
err = walk(subpath, done, jobs, ch)
err = walk(basedir, subpath, done, jobs, ch)
if err != nil {
return err
}
} else {
select {
case jobs <- Entry{Info: fi, Path: subpath, Result: ch}:
case jobs <- Entry{info: fi, basedir: basedir, path: filepath.Join(relpath, name), result: ch}:
case <-done:
return errCancelled
}
@@ -107,7 +138,7 @@ func walk(path string, done chan struct{}, jobs chan<- interface{}, res chan<- i
}
select {
case jobs <- Dir{Path: path, Info: info, Entries: entries, Result: res}:
case jobs <- Dir{basedir: basedir, path: relpath, info: info, Entries: entries, result: res}:
case <-done:
return errCancelled
}
@@ -116,31 +147,30 @@ func walk(path string, done chan struct{}, jobs chan<- interface{}, res chan<- i
// Walk sends a Job for each file and directory it finds below the paths. When
// the channel done is closed, processing stops.
func Walk(paths []string, done chan struct{}, jobs chan<- interface{}) (<-chan interface{}, error) {
resCh := make(chan interface{}, 1)
func Walk(paths []string, done chan struct{}, jobs chan<- Job, res chan<- Result) error {
defer func() {
close(resCh)
close(jobs)
debug.Log("pipe.Walk", "output channel closed")
close(res)
close(jobs)
}()
entries := make([]<-chan interface{}, 0, len(paths))
entries := make([]<-chan Result, 0, len(paths))
for _, path := range paths {
debug.Log("pipe.Walk", "start walker for %v", path)
ch := make(chan interface{}, 1)
ch := make(chan Result, 1)
entries = append(entries, ch)
err := walk(path, done, jobs, ch)
err := walk(filepath.Dir(path), path, done, jobs, ch)
if err != nil {
return nil, err
return err
}
debug.Log("pipe.Walk", "walker for %v done", path)
}
resCh <- Dir{Entries: entries}
return resCh, nil
res <- Dir{Entries: entries}
return nil
}
// Split feeds all elements read from inChan to dirChan and entChan.
func Split(inChan <-chan interface{}, dirChan chan<- Dir, entChan chan<- Entry) {
func Split(inChan <-chan Job, dirChan chan<- Dir, entChan chan<- Entry) {
debug.Log("pipe.Split", "start")
defer debug.Log("pipe.Split", "done")
pipe/pipe_test.go
+16 -13
View File
@@ -71,7 +71,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) {
after.files++
m.Unlock()
e.Result <- true
e.Result() <- true
case dir, ok := <-dirCh:
if !ok {
@@ -88,7 +88,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) {
after.dirs++
m.Unlock()
dir.Result <- true
dir.Result() <- true
case <-done:
// pipeline was cancelled
return
@@ -106,7 +106,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) {
go worker(&wg, done, entCh, dirCh)
}
jobs := make(chan interface{}, 200)
jobs := make(chan pipe.Job, 200)
wg.Add(1)
go func() {
pipe.Split(jobs, dirCh, entCh)
@@ -115,7 +115,8 @@ func TestPipelineWalkerWithSplit(t *testing.T) {
wg.Done()
}()
resCh, err := pipe.Walk([]string{*testWalkerPath}, done, jobs)
resCh := make(chan pipe.Result, 1)
err = pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh)
ok(t, err)
// wait for all workers to terminate
@@ -144,7 +145,7 @@ func TestPipelineWalker(t *testing.T) {
after := stats{}
m := sync.Mutex{}
worker := func(wg *sync.WaitGroup, done <-chan struct{}, jobs <-chan interface{}) {
worker := func(wg *sync.WaitGroup, done <-chan struct{}, jobs <-chan pipe.Job) {
defer wg.Done()
for {
select {
@@ -166,13 +167,13 @@ func TestPipelineWalker(t *testing.T) {
after.dirs++
m.Unlock()
j.Result <- true
j.Result() <- true
case pipe.Entry:
m.Lock()
after.files++
m.Unlock()
j.Result <- true
j.Result() <- true
}
case <-done:
@@ -184,14 +185,15 @@ func TestPipelineWalker(t *testing.T) {
var wg sync.WaitGroup
done := make(chan struct{})
jobs := make(chan interface{})
jobs := make(chan pipe.Job)
for i := 0; i < *maxWorkers; i++ {
wg.Add(1)
go worker(&wg, done, jobs)
}
resCh, err := pipe.Walk([]string{*testWalkerPath}, done, jobs)
resCh := make(chan pipe.Result, 1)
err = pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh)
ok(t, err)
// wait for all workers to terminate
@@ -227,7 +229,7 @@ func BenchmarkPipelineWalker(b *testing.B) {
// simulate backup
//time.Sleep(10 * time.Millisecond)
e.Result <- true
e.Result() <- true
case <-done:
// pipeline was cancelled
return
@@ -259,7 +261,7 @@ func BenchmarkPipelineWalker(b *testing.B) {
}
m.Unlock()
dir.Result <- true
dir.Result() <- true
case <-done:
// pipeline was cancelled
return
@@ -281,7 +283,7 @@ func BenchmarkPipelineWalker(b *testing.B) {
go fileWorker(&wg, done, entCh)
}
jobs := make(chan interface{}, 200)
jobs := make(chan pipe.Job, 200)
wg.Add(1)
go func() {
pipe.Split(jobs, dirCh, entCh)
@@ -290,7 +292,8 @@ func BenchmarkPipelineWalker(b *testing.B) {
wg.Done()
}()
resCh, err := pipe.Walk([]string{*testWalkerPath}, done, jobs)
resCh := make(chan pipe.Result, 1)
err := pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh)
ok(b, err)
// wait for all workers to terminate