archiver: ignore duplicate but excluded directory entry (#21900)

changelog/unreleased/issue-21899

@@ -0,0 +1,9 @@
+Bugfix: Fix excludes of duplicate directory entries during `backup`
+
+Since restic 0.19.0, creating a backup of a directory containing
+duplicate directory entries always resulted in "Warning: at least
+one source file could not be read" even if the files in question
+were excluded. This has been fixed.
+
+https://github.com/restic/restic/issues/21899
+https://github.com/restic/restic/pull/21900

internal/archiver/archiver.go

@@ -320,6 +320,8 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 	finder := data.NewTreeFinder(previous)
 	defer finder.Close()
 
+	var lastExcluded string
+
 	for _, name := range names {
 		// test if context has been cancelled
 		if ctx.Err() != nil {
@@ -327,6 +329,12 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 			return futureNode{}, ctx.Err()
 		}
 
+		if name == lastExcluded {
+			// Skip duplicate directory entry if it was already excluded.
+			// This avoids printing errors about duplicate directory entries even though the entry in question is ignored.
+			continue
+		}
+
 		pathname := arch.FS.Join(dir, name)
 		oldNode, err := finder.Find(name)
 		err = arch.error(pathname, err)
@@ -348,6 +356,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 		}
 
 		if excluded {
+			lastExcluded = name
 			continue
 		}
 

internal/archiver/archiver_test.go

@@ -894,6 +894,90 @@ func TestArchiverSaveDir(t *testing.T) {
 	}
 }
 
+type duplicateReaddirFS struct {
+	fs.FS
+	dir   string
+	names []string
+}
+
+func (d *duplicateReaddirFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
+	f, err := d.FS.OpenFile(name, flag, metadataOnly)
+	if err != nil {
+		return nil, err
+	}
+
+	if name == d.dir {
+		return &duplicateReaddirFile{File: f, names: d.names}, nil
+	}
+	return f, nil
+}
+
+type duplicateReaddirFile struct {
+	fs.File
+	names []string
+}
+
+func (f *duplicateReaddirFile) Readdirnames(int) ([]string, error) {
+	return append([]string(nil), f.names...), nil
+}
+
+func TestArchiverSaveDirDuplicateExcludedEntry(t *testing.T) {
+	const targetNodeName = "targetdir"
+
+	src := TestDir{
+		"excluded": TestFile{Content: "skip me"},
+		"keep":     TestFile{Content: "keep me"},
+	}
+	tempdir, repo := prepareTempdirRepoSrc(t, src)
+
+	testFS := fs.Track{FS: &duplicateReaddirFS{
+		FS:    &fs.Local{},
+		dir:   ".",
+		names: []string{"excluded", "excluded", "keep"},
+	}}
+	arch := New(repo, testFS, Options{})
+	arch.summary = &Summary{}
+	arch.Select = func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
+		return filepath.Base(item) != "excluded"
+	}
+	arch.Error = func(item string, err error) error {
+		t.Errorf("unexpected archiver error for %v: %v", item, err)
+		return err
+	}
+
+	back := rtest.Chdir(t, tempdir)
+	defer back()
+
+	// duplicate node check in tree finder is only done if the previous tree is not nil
+	previousTree, err := data.NewTreeNodeIterator(strings.NewReader(`{"nodes":[]}`))
+	rtest.OK(t, err)
+
+	var treeID restic.ID
+	err = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+		wg, ctx := errgroup.WithContext(ctx)
+		arch.runWorkers(ctx, wg, uploader)
+		meta, err := testFS.OpenFile(".", fs.O_NOFOLLOW, true)
+		rtest.OK(t, err)
+		ft, err := arch.saveDir(ctx, "/", ".", meta, previousTree, nil)
+		rtest.OK(t, err)
+		rtest.OK(t, meta.Close())
+
+		fnr := ft.take(ctx)
+		node := fnr.node
+		node.Name = targetNodeName
+		treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node})
+		arch.stopWorkers()
+		return wg.Wait()
+	})
+	rtest.OK(t, err)
+
+	TestEnsureTree(context.TODO(), t, "/", repo, treeID, TestDir{
+		"targetdir": TestDir{
+			"keep": TestFile{Content: "keep me"},
+		},
+	})
+}
+
 func TestArchiverSaveDirIncremental(t *testing.T) {
 	tempdir := rtest.TempDir(t)