mirror of
https://github.com/restic/restic.git
synced 2026-08-11 08:03:17 +00:00
Refactor backup progress
Move the shared logic into the progress Allows logic to be shared with forth coming restore status
This commit is contained in:
Dan Willoughby
committed by
Michael Eischer
Michael Eischer
parent
fa5ca8af81
commit
448419990c
+61
-292
@@ -2,11 +2,9 @@ package json
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/archiver"
|
||||
@@ -15,46 +13,14 @@ import (
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
)
|
||||
|
||||
type counter struct {
|
||||
Files, Dirs, Bytes uint64
|
||||
}
|
||||
|
||||
type fileWorkerMessage struct {
|
||||
filename string
|
||||
done bool
|
||||
}
|
||||
|
||||
// Backup reports progress for the `backup` command in JSON.
|
||||
type Backup struct {
|
||||
*ui.Message
|
||||
*ui.StdioWrapper
|
||||
|
||||
MinUpdatePause time.Duration
|
||||
|
||||
term *termstatus.Terminal
|
||||
v uint
|
||||
start time.Time
|
||||
dry bool
|
||||
|
||||
totalBytes uint64
|
||||
|
||||
totalCh chan counter
|
||||
processedCh chan counter
|
||||
errCh chan struct{}
|
||||
workerCh chan fileWorkerMessage
|
||||
finished chan struct{}
|
||||
closed chan struct{}
|
||||
|
||||
summary struct {
|
||||
sync.Mutex
|
||||
Files, Dirs struct {
|
||||
New uint
|
||||
Changed uint
|
||||
Unchanged uint
|
||||
}
|
||||
ProcessedBytes uint64
|
||||
archiver.ItemStats
|
||||
}
|
||||
term *termstatus.Terminal
|
||||
v uint
|
||||
dry bool
|
||||
}
|
||||
|
||||
// NewBackup returns a new backup progress reporter.
|
||||
@@ -64,17 +30,6 @@ func NewBackup(term *termstatus.Terminal, verbosity uint) *Backup {
|
||||
StdioWrapper: ui.NewStdioWrapper(term),
|
||||
term: term,
|
||||
v: verbosity,
|
||||
start: time.Now(),
|
||||
|
||||
// limit to 60fps by default
|
||||
MinUpdatePause: time.Second / 60,
|
||||
|
||||
totalCh: make(chan counter),
|
||||
processedCh: make(chan counter),
|
||||
errCh: make(chan struct{}),
|
||||
workerCh: make(chan fileWorkerMessage),
|
||||
finished: make(chan struct{}),
|
||||
closed: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,78 +50,11 @@ func (b *Backup) error(status interface{}) {
|
||||
b.term.Error(toJSONString(status))
|
||||
}
|
||||
|
||||
// Run regularly updates the status lines. It should be called in a separate
|
||||
// goroutine.
|
||||
func (b *Backup) Run(ctx context.Context) error {
|
||||
var (
|
||||
lastUpdate time.Time
|
||||
total, processed counter
|
||||
errors uint
|
||||
started bool
|
||||
currentFiles = make(map[string]struct{})
|
||||
secondsRemaining uint64
|
||||
)
|
||||
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
defer close(b.closed)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-b.finished:
|
||||
started = false
|
||||
case t, ok := <-b.totalCh:
|
||||
if ok {
|
||||
total = t
|
||||
started = true
|
||||
} else {
|
||||
// scan has finished
|
||||
b.totalCh = nil
|
||||
b.totalBytes = total.Bytes
|
||||
}
|
||||
case s := <-b.processedCh:
|
||||
processed.Files += s.Files
|
||||
processed.Dirs += s.Dirs
|
||||
processed.Bytes += s.Bytes
|
||||
started = true
|
||||
case <-b.errCh:
|
||||
errors++
|
||||
started = true
|
||||
case m := <-b.workerCh:
|
||||
if m.done {
|
||||
delete(currentFiles, m.filename)
|
||||
} else {
|
||||
currentFiles[m.filename] = struct{}{}
|
||||
}
|
||||
case <-t.C:
|
||||
if !started {
|
||||
continue
|
||||
}
|
||||
|
||||
if b.totalCh == nil {
|
||||
secs := float64(time.Since(b.start) / time.Second)
|
||||
todo := float64(total.Bytes - processed.Bytes)
|
||||
secondsRemaining = uint64(secs / float64(processed.Bytes) * todo)
|
||||
}
|
||||
}
|
||||
|
||||
// limit update frequency
|
||||
if time.Since(lastUpdate) < b.MinUpdatePause {
|
||||
continue
|
||||
}
|
||||
lastUpdate = time.Now()
|
||||
|
||||
b.update(total, processed, errors, currentFiles, secondsRemaining)
|
||||
}
|
||||
}
|
||||
|
||||
// update updates the status lines.
|
||||
func (b *Backup) update(total, processed counter, errors uint, currentFiles map[string]struct{}, secs uint64) {
|
||||
func (b *Backup) Update(total, processed ui.Counter, errors uint, currentFiles map[string]struct{}, start time.Time, secs uint64) {
|
||||
status := statusUpdate{
|
||||
MessageType: "status",
|
||||
SecondsElapsed: uint64(time.Since(b.start) / time.Second),
|
||||
SecondsElapsed: uint64(time.Since(start) / time.Second),
|
||||
SecondsRemaining: secs,
|
||||
TotalFiles: total.Files,
|
||||
FilesDone: processed.Files,
|
||||
@@ -207,211 +95,92 @@ func (b *Backup) Error(item string, fi os.FileInfo, err error) error {
|
||||
During: "archival",
|
||||
Item: item,
|
||||
})
|
||||
select {
|
||||
case b.errCh <- struct{}{}:
|
||||
case <-b.closed:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartFile is called when a file is being processed by a worker.
|
||||
func (b *Backup) StartFile(filename string) {
|
||||
select {
|
||||
case b.workerCh <- fileWorkerMessage{filename: filename}:
|
||||
case <-b.closed:
|
||||
}
|
||||
}
|
||||
|
||||
// CompleteBlob is called for all saved blobs for files.
|
||||
func (b *Backup) CompleteBlob(filename string, bytes uint64) {
|
||||
select {
|
||||
case b.processedCh <- counter{Bytes: bytes}:
|
||||
case <-b.closed:
|
||||
}
|
||||
}
|
||||
|
||||
// CompleteItem is the status callback function for the archiver when a
|
||||
// file/dir has been saved successfully.
|
||||
func (b *Backup) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
|
||||
b.summary.Lock()
|
||||
b.summary.ItemStats.Add(s)
|
||||
b.summary.Unlock()
|
||||
|
||||
if current == nil {
|
||||
// error occurred, tell the status display to remove the line
|
||||
select {
|
||||
case b.workerCh <- fileWorkerMessage{filename: item, done: true}:
|
||||
case <-b.closed:
|
||||
}
|
||||
func (b *Backup) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
|
||||
if b.v < 2 {
|
||||
return
|
||||
}
|
||||
|
||||
b.summary.ProcessedBytes += current.Size
|
||||
|
||||
switch current.Type {
|
||||
case "file":
|
||||
select {
|
||||
case b.processedCh <- counter{Files: 1}:
|
||||
case <-b.closed:
|
||||
}
|
||||
select {
|
||||
case b.workerCh <- fileWorkerMessage{filename: item, done: true}:
|
||||
case <-b.closed:
|
||||
}
|
||||
case "dir":
|
||||
select {
|
||||
case b.processedCh <- counter{Dirs: 1}:
|
||||
case <-b.closed:
|
||||
}
|
||||
}
|
||||
|
||||
if current.Type == "dir" {
|
||||
if previous == nil {
|
||||
if b.v >= 3 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "new",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
MetadataSize: s.TreeSize,
|
||||
})
|
||||
}
|
||||
b.summary.Lock()
|
||||
b.summary.Dirs.New++
|
||||
b.summary.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
if previous.Equals(*current) {
|
||||
if b.v >= 3 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "unchanged",
|
||||
Item: item,
|
||||
})
|
||||
}
|
||||
b.summary.Lock()
|
||||
b.summary.Dirs.Unchanged++
|
||||
b.summary.Unlock()
|
||||
} else {
|
||||
if b.v >= 3 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "modified",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
MetadataSize: s.TreeSize,
|
||||
})
|
||||
}
|
||||
b.summary.Lock()
|
||||
b.summary.Dirs.Changed++
|
||||
b.summary.Unlock()
|
||||
}
|
||||
|
||||
} else if current.Type == "file" {
|
||||
select {
|
||||
case b.workerCh <- fileWorkerMessage{done: true, filename: item}:
|
||||
case <-b.closed:
|
||||
}
|
||||
|
||||
if previous == nil {
|
||||
if b.v >= 3 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "new",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
})
|
||||
}
|
||||
b.summary.Lock()
|
||||
b.summary.Files.New++
|
||||
b.summary.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
if previous.Equals(*current) {
|
||||
if b.v >= 3 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "unchanged",
|
||||
Item: item,
|
||||
})
|
||||
}
|
||||
b.summary.Lock()
|
||||
b.summary.Files.Unchanged++
|
||||
b.summary.Unlock()
|
||||
} else {
|
||||
if b.v >= 3 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "modified",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
})
|
||||
}
|
||||
b.summary.Lock()
|
||||
b.summary.Files.Changed++
|
||||
b.summary.Unlock()
|
||||
}
|
||||
switch messageType {
|
||||
case "dir new":
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "new",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
MetadataSize: s.TreeSize,
|
||||
})
|
||||
case "dir unchanged":
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "unchanged",
|
||||
Item: item,
|
||||
})
|
||||
case "dir modified":
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "modified",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
MetadataSize: s.TreeSize,
|
||||
})
|
||||
case "file new":
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "new",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
})
|
||||
case "file unchanged":
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "unchanged",
|
||||
Item: item,
|
||||
})
|
||||
case "file modified":
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "verbose_status",
|
||||
Action: "modified",
|
||||
Item: item,
|
||||
Duration: d.Seconds(),
|
||||
DataSize: s.DataSize,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ReportTotal sets the total stats up to now
|
||||
func (b *Backup) ReportTotal(item string, s archiver.ScanStats) {
|
||||
select {
|
||||
case b.totalCh <- counter{Files: uint64(s.Files), Dirs: uint64(s.Dirs), Bytes: s.Bytes}:
|
||||
case <-b.closed:
|
||||
}
|
||||
|
||||
func (b *Backup) ReportTotal(item string, start time.Time, s archiver.ScanStats) {
|
||||
if item == "" {
|
||||
if b.v >= 2 {
|
||||
b.print(verboseUpdate{
|
||||
MessageType: "status",
|
||||
Action: "scan_finished",
|
||||
Duration: time.Since(b.start).Seconds(),
|
||||
Duration: time.Since(start).Seconds(),
|
||||
DataSize: s.Bytes,
|
||||
TotalFiles: s.Files,
|
||||
})
|
||||
}
|
||||
close(b.totalCh)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Finish prints the finishing messages.
|
||||
func (b *Backup) Finish(snapshotID restic.ID) {
|
||||
select {
|
||||
case b.finished <- struct{}{}:
|
||||
case <-b.closed:
|
||||
}
|
||||
|
||||
func (b *Backup) Finish(snapshotID restic.ID, start time.Time, summary *ui.Summary) {
|
||||
b.print(summaryOutput{
|
||||
MessageType: "summary",
|
||||
FilesNew: b.summary.Files.New,
|
||||
FilesChanged: b.summary.Files.Changed,
|
||||
FilesUnmodified: b.summary.Files.Unchanged,
|
||||
DirsNew: b.summary.Dirs.New,
|
||||
DirsChanged: b.summary.Dirs.Changed,
|
||||
DirsUnmodified: b.summary.Dirs.Unchanged,
|
||||
DataBlobs: b.summary.ItemStats.DataBlobs,
|
||||
TreeBlobs: b.summary.ItemStats.TreeBlobs,
|
||||
DataAdded: b.summary.ItemStats.DataSize + b.summary.ItemStats.TreeSize,
|
||||
TotalFilesProcessed: b.summary.Files.New + b.summary.Files.Changed + b.summary.Files.Unchanged,
|
||||
TotalBytesProcessed: b.summary.ProcessedBytes,
|
||||
TotalDuration: time.Since(b.start).Seconds(),
|
||||
SnapshotID: snapshotID.Str(),
|
||||
DryRun: b.dry,
|
||||
MessageType: "summary",
|
||||
SnapshotID: snapshotID.Str(),
|
||||
})
|
||||
}
|
||||
|
||||
// SetMinUpdatePause sets b.MinUpdatePause. It satisfies the
|
||||
// ArchiveProgressReporter interface.
|
||||
func (b *Backup) SetMinUpdatePause(d time.Duration) {
|
||||
b.MinUpdatePause = d
|
||||
// Reset no-op
|
||||
func (b *Backup) Reset() {
|
||||
}
|
||||
|
||||
// SetDryRun marks the backup as a "dry run".
|
||||
|
||||
Reference in New Issue
Block a user