backup - show excluded files and directories in verbose mode - text/JSON (#21887)

changelog/unreleased/issue-21870

@@ -0,0 +1,7 @@
+Enhancement: show excluded items during backup in verbose mode
+
+`restic backup -vv` now shows excluded items in verbose mode. Both text and JSON
+output are supported.
+
+https://github.com/restic/restic/issues/21870
+https://github.com/restic/restic/pull/21887

cmd/restic/cmd_backup.go

@@ -666,6 +666,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
 	arch.CompleteItem = progressReporter.CompleteItem
 	arch.StartFile = progressReporter.StartFile
 	arch.CompleteBlob = progressReporter.CompleteBlob
+	arch.ExcludedItem = progressReporter.ExcludedItem
 
 	if opts.IgnoreInode {
 		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not

cmd/restic/cmd_backup_integration_test.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
@@ -15,6 +18,7 @@ import (
 	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/backup"
 )
 
 func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) error {
@@ -30,6 +34,13 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts
 	})
 }
 
+func testRunBackupOutput(t testing.TB, opts BackupOptions, gopts global.Options, target []string) ([]byte, error) {
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		return runBackup(ctx, opts, gopts, gopts.Term, target)
+	})
+	return buf.Bytes(), err
+}
+
 func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) {
 	err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
 	rtest.Assert(t, err == nil, "Error while backing up: %v", err)
@@ -730,3 +741,40 @@ func TestBackupSkipIfUnchanged(t *testing.T) {
 
 	testRunCheck(t, env.gopts)
 }
+
+func TestBackupExcludeWithOutput(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	backupOptions := BackupOptions{}
+	backupOptions.Excludes = []string{"*.py"}
+
+	env.gopts.JSON = true
+	env.gopts.Verbosity = 2
+	output, err := testRunBackupOutput(t, backupOptions, env.gopts, []string{filepath.Join(env.testdata, "0", "for_cmd_ls")})
+	rtest.OK(t, err)
+
+	foundExclude := false
+	for _, line := range bytes.Split(output, []byte("\n")) {
+		if len(line) == 0 {
+			continue
+		}
+
+		type MessageType struct {
+			MessageType string `json:"message_type"` // any
+		}
+		var mType MessageType
+		rtest.OK(t, json.Unmarshal(line, &mType))
+		if mType.MessageType != "excluded_item" {
+			continue
+		}
+
+		var excludeLine backup.VerboseExclude
+		rtest.OK(t, json.Unmarshal(line, &excludeLine))
+		rtest.Assert(t, strings.Contains(excludeLine.Item, ".py"), "expected excluded pathname to be ending in .py, but contains %q",
+			excludeLine.Item)
+		foundExclude = true
+	}
+	rtest.Assert(t, foundExclude, "expected at least one excluded item, but found none")
+}

doc/075_scripting.rst

@@ -311,6 +311,15 @@ Verbose status provides details about the progress, including details about back
 | ``total_files``           | Total number of files                                    | uint64  |
 +---------------------------+----------------------------------------------------------+---------+
 
+Excluded backup items are detailed in this format:
+
++---------------------------+----------------------------------------------------------+---------+
+| ``message_type``          | Always "excluded_item"                                   | string  |
++---------------------------+----------------------------------------------------------+---------+
+| ``item``                  | The item in question                                     | string  |
++---------------------------+----------------------------------------------------------+---------+
+
+
 Summary
 ^^^^^^^
 

internal/archiver/archiver.go

@@ -129,6 +129,9 @@ type Archiver struct {
 
 	// Flags controlling change detection. See doc/040_backup.rst for details.
 	ChangeIgnoreFlags uint
+
+	// for excluded items
+	ExcludedItem func(path string)
 }
 
 // Flags for the ChangeIgnoreFlags bitfield.
@@ -183,6 +186,7 @@ func New(repo archiverRepo, filesystem fs.FS, opts Options) *Archiver {
 		CompleteItem: func(string, ItemAction, ItemStats, time.Duration) {},
 		StartFile:    func(string) {},
 		CompleteBlob: func(uint64) {},
+		ExcludedItem: func(string) {},
 	}
 
 	return arch
@@ -481,6 +485,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 	// exclude files by path before running Lstat to reduce number of lstat calls
 	if !explicit && !arch.SelectByName(abstarget) {
 		debug.Log("%v is excluded by path", target)
+		arch.ExcludedItem(abstarget)
 		return futureNode{}, true, nil
 	}
 
@@ -509,6 +514,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 	}
 	if !explicit && !arch.Select(abstarget, fi, arch.FS) {
 		debug.Log("%v is excluded", target)
+		arch.ExcludedItem(abstarget)
 		return futureNode{}, true, nil
 	}
 

internal/ui/backup/json.go

@@ -251,3 +251,18 @@ type summaryOutput struct {
 	SnapshotID          string    `json:"snapshot_id,omitempty"`
 	DryRun              bool      `json:"dry_run,omitempty"`
 }
+
+type VerboseExclude struct {
+	MessageType string `json:"message_type"` // "excluded_item"
+	Item        string `json:"item"`         // file or directory name
+}
+
+func (b *jsonProgress) ExcludedItem(path string) {
+	if b.v < 2 {
+		return
+	}
+	b.print(VerboseExclude{
+		MessageType: "excluded_item",
+		Item:        path,
+	})
+}

internal/ui/backup/progress.go

@@ -19,6 +19,7 @@ type ProgressPrinter interface {
 	ReportTotal(start time.Time, s archiver.ScanStats)
 	Finish(snapshotID restic.ID, summary *archiver.Summary, dryRun bool)
 	Reset()
+	ExcludedItem(path string)
 
 	restic.Printer
 }
@@ -162,3 +163,7 @@ func (p *Progress) Finish(snapshotID restic.ID, summary *archiver.Summary, dryru
 	p.Updater.Done()
 	p.printer.Finish(snapshotID, summary, dryrun)
 }
+
+func (p *Progress) ExcludedItem(path string) {
+	p.printer.ExcludedItem(path)
+}

internal/ui/backup/progress_test.go

@@ -41,7 +41,8 @@ func (p *mockPrinter) Finish(id restic.ID, _ *archiver.Summary, _ bool) {
 	p.id = id
 }
 
-func (p *mockPrinter) Reset() {}
+func (p *mockPrinter) Reset()                {}
+func (p *mockPrinter) ExcludedItem(_ string) {}
 
 func TestProgress(t *testing.T) {
 	t.Parallel()

internal/ui/backup/text.go

@@ -158,3 +158,7 @@ func (b *textProgress) Finish(id restic.ID, summary *archiver.Summary, dryRun bo
 		}
 	}
 }
+
+func (b *textProgress) ExcludedItem(path string) {
+	b.VV("excluded %s", path)
+}