amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-06-04 07:59:43 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2311 from vincentbernat/feature/negative-pattern

Browse Source 
filter: ability to use negative patterns
This commit is contained in:
Alexander Neumann
2022-03-20 14:02:30 +01:00
committed by GitHub
parent 13aae82635 29a5778626
commit 3a285f91bc
4 changed files with 139 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changelog/unreleased/issue-233
+31
View File
@@ -0,0 +1,31 @@
Enhancement: Add negative patterns for include/exclude
If a pattern starts with an exclamation mark and it matches a file that
was previously matched by a regular pattern, the match is cancelled.
Notably, this can be used with `--exclude-file` to cancel the
exclusion of some files.
It works similarly to `gitignore`, with the same limitation: once a
directory is excluded, it is not possible to include files inside the
directory.
Example of use (as an exclude pattern for backup):
$HOME/**/*
!$HOME/Documents
!$HOME/code
!$HOME/.emacs.d
!$HOME/games
# [...]
node_modules
*~
*.o
*.lo
*.pyc
# [...]
$HOME/code/linux/*
!$HOME/code/linux/.git
# [...]
https://github.com/restic/restic/issues/233
https://github.com/restic/restic/pull/2311
doc/040_backup.rst
+22
View File
@@ -289,6 +289,28 @@ On most Unixy shells, you can either quote or use backslashes. For example:
* ``--exclude="foo bar star/foo.txt"``
* ``--exclude=foo\ bar\ star/foo.txt``
If a pattern starts with exclamation mark and matches a file that
was previously matched by a regular pattern, the match is cancelled.
It works similarly to ``gitignore``, with the same limitation: once a
directory is excluded, it is not possible to include files inside the
directory. Here is a complete example to backup a selection of
directories inside the home directory. It works by excluding any
directory, then selectively add back some of them.
::
$HOME/**/*
!$HOME/Documents
!$HOME/code
!$HOME/.emacs.d
!$HOME/games
# [...]
node_modules
*~
*.o
*.lo
*.pyc
By specifying the option ``--one-file-system`` you can instruct restic
to only backup files from the file systems the initially specified files
or directories reside on. In other words, it will prevent restic from crossing
internal/filter/filter.go
+68 -45
View File
@@ -17,7 +17,10 @@ type patternPart struct {
}
// Pattern represents a preparsed filter pattern
type Pattern []patternPart
type Pattern struct {
parts []patternPart
isNegated bool
}
func prepareStr(str string) ([]string, error) {
if str == "" {
@@ -26,20 +29,26 @@ func prepareStr(str string) ([]string, error) {
return splitPath(str), nil
}
func preparePattern(pattern string) Pattern {
parts := splitPath(filepath.Clean(pattern))
patterns := make([]patternPart, len(parts))
for i, part := range parts {
func preparePattern(patternStr string) Pattern {
var negate bool
if patternStr[0] == '!' {
negate = true
patternStr = patternStr[1:]
}
pathParts := splitPath(filepath.Clean(patternStr))
parts := make([]patternPart, len(pathParts))
for i, part := range pathParts {
isSimple := !strings.ContainsAny(part, "\\[]*?")
// Replace "**" with the empty string to get faster comparisons
// (length-check only) in hasDoubleWildcard.
if part == "**" {
part = ""
}
patterns[i] = patternPart{part, isSimple}
parts[i] = patternPart{part, isSimple}
}
return patterns
return Pattern{parts, negate}
}
// Split p into path components. Assuming p has been Cleaned, no component
@@ -62,19 +71,19 @@ func splitPath(p string) []string {
// In addition patterns suitable for filepath.Match, pattern accepts a
// recursive wildcard '**', which greedily matches an arbitrary number of
// intermediate directories.
func Match(pattern, str string) (matched bool, err error) {
if pattern == "" {
func Match(patternStr, str string) (matched bool, err error) {
if patternStr == "" {
return true, nil
}
patterns := preparePattern(pattern)
pattern := preparePattern(patternStr)
strs, err := prepareStr(str)
if err != nil {
return false, err
}
return match(patterns, strs)
return match(pattern, strs)
}
// ChildMatch returns true if children of str can match the pattern. When the pattern is
@@ -87,28 +96,28 @@ func Match(pattern, str string) (matched bool, err error) {
// In addition patterns suitable for filepath.Match, pattern accepts a
// recursive wildcard '**', which greedily matches an arbitrary number of
// intermediate directories.
func ChildMatch(pattern, str string) (matched bool, err error) {
if pattern == "" {
func ChildMatch(patternStr, str string) (matched bool, err error) {
if patternStr == "" {
return true, nil
}
patterns := preparePattern(pattern)
pattern := preparePattern(patternStr)
strs, err := prepareStr(str)
if err != nil {
return false, err
}
return childMatch(patterns, strs)
return childMatch(pattern, strs)
}
func childMatch(patterns Pattern, strs []string) (matched bool, err error) {
if patterns[0].pattern != "/" {
func childMatch(pattern Pattern, strs []string) (matched bool, err error) {
if pattern.parts[0].pattern != "/" {
// relative pattern can always be nested down
return true, nil
}
ok, pos := hasDoubleWildcard(patterns)
ok, pos := hasDoubleWildcard(pattern)
if ok && len(strs) >= pos {
// cut off at the double wildcard
strs = strs[:pos]
@@ -116,16 +125,16 @@ func childMatch(patterns Pattern, strs []string) (matched bool, err error) {
// match path against absolute pattern prefix
l := 0
if len(strs) > len(patterns) {
l = len(patterns)
if len(strs) > len(pattern.parts) {
l = len(pattern.parts)
} else {
l = len(strs)
}
return match(patterns[0:l], strs)
return match(Pattern{pattern.parts[0:l], pattern.isNegated}, strs)
}
func hasDoubleWildcard(list Pattern) (ok bool, pos int) {
for i, item := range list {
for i, item := range list.parts {
if item.pattern == "" {
return true, i
}
@@ -134,22 +143,22 @@ func hasDoubleWildcard(list Pattern) (ok bool, pos int) {
return false, 0
}
func match(patterns Pattern, strs []string) (matched bool, err error) {
if ok, pos := hasDoubleWildcard(patterns); ok {
func match(pattern Pattern, strs []string) (matched bool, err error) {
if ok, pos := hasDoubleWildcard(pattern); ok {
// gradually expand '**' into separate wildcards
newPat := make(Pattern, len(strs))
newPat := make([]patternPart, len(strs))
// copy static prefix once
copy(newPat, patterns[:pos])
for i := 0; i <= len(strs)-len(patterns)+1; i++ {
copy(newPat, pattern.parts[:pos])
for i := 0; i <= len(strs)-len(pattern.parts)+1; i++ {
// limit to static prefix and already appended '*'
newPat := newPat[:pos+i]
// in the first iteration the wildcard expands to nothing
if i > 0 {
newPat[pos+i-1] = patternPart{"*", false}
}
newPat = append(newPat, patterns[pos+1:]...)
newPat = append(newPat, pattern.parts[pos+1:]...)
matched, err := match(newPat, strs)
matched, err := match(Pattern{newPat, pattern.isNegated}, strs)
if err != nil {
return false, err
}
@@ -162,20 +171,20 @@ func match(patterns Pattern, strs []string) (matched bool, err error) {
return false, nil
}
if len(patterns) == 0 && len(strs) == 0 {
if len(pattern.parts) == 0 && len(strs) == 0 {
return true, nil
}
// an empty pattern never matches a non-empty path
if len(patterns) == 0 {
if len(pattern.parts) == 0 {
return false, nil
}
if len(patterns) <= len(strs) {
if len(pattern.parts) <= len(strs) {
minOffset := 0
maxOffset := len(strs) - len(patterns)
maxOffset := len(strs) - len(pattern.parts)
// special case absolute patterns
if patterns[0].pattern == "/" {
if pattern.parts[0].pattern == "/" {
maxOffset = 0
} else if strs[0] == "/" {
// skip absolute path marker if pattern is not rooted
@@ -184,12 +193,12 @@ func match(patterns Pattern, strs []string) (matched bool, err error) {
outer:
for offset := maxOffset; offset >= minOffset; offset-- {
for i := len(patterns) - 1; i >= 0; i-- {
for i := len(pattern.parts) - 1; i >= 0; i-- {
var ok bool
if patterns[i].isSimple {
ok = patterns[i].pattern == strs[offset+i]
if pattern.parts[i].isSimple {
ok = pattern.parts[i].pattern == strs[offset+i]
} else {
ok, err = filepath.Match(patterns[i].pattern, strs[offset+i])
ok, err = filepath.Match(pattern.parts[i].pattern, strs[offset+i])
if err != nil {
return false, errors.Wrap(err, "Match")
}
@@ -208,9 +217,9 @@ func match(patterns Pattern, strs []string) (matched bool, err error) {
}
// ParsePatterns prepares a list of patterns for use with List.
func ParsePatterns(patterns []string) []Pattern {
func ParsePatterns(pattern []string) []Pattern {
patpat := make([]Pattern, 0)
for _, pat := range patterns {
for _, pat := range pattern {
if pat == "" {
continue
}
@@ -232,7 +241,9 @@ func ListWithChild(patterns []Pattern, str string) (matched bool, childMayMatch
return list(patterns, true, str)
}
// List returns true if str matches one of the patterns. Empty patterns are ignored.
// list returns true if str matches one of the patterns. Empty patterns are ignored.
// Patterns prefixed by "!" are negated: any matching file excluded by a previous pattern
// will become included again.
func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool, childMayMatch bool, err error) {
if len(patterns) == 0 {
return false, false, nil
@@ -242,6 +253,12 @@ func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool,
if err != nil {
return false, false, err
}
hasNegatedPattern := false
for _, pat := range patterns {
hasNegatedPattern = hasNegatedPattern || pat.isNegated
}
for _, pat := range patterns {
m, err := match(pat, strs)
if err != nil {
@@ -258,11 +275,17 @@ func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool,
c = true
}
matched = matched || m
childMayMatch = childMayMatch || c
if pat.isNegated {
matched = matched && !m
childMayMatch = childMayMatch && !m
} else {
matched = matched || m
childMayMatch = childMayMatch || c
if matched && childMayMatch {
return true, true, nil
if matched && childMayMatch && !hasNegatedPattern {
// without negative patterns the result cannot change any more
break
}
}
}
internal/filter/filter_test.go
+18
View File
@@ -248,6 +248,7 @@ var filterListTests = []struct {
}{
{[]string{}, "/foo/bar/test.go", false, false},
{[]string{"*.go"}, "/foo/bar/test.go", true, true},
{[]string{"*.go"}, "/foo/bar", false, true},
{[]string{"*.c"}, "/foo/bar/test.go", false, true},
{[]string{"*.go", "*.c"}, "/foo/bar/test.go", true, true},
{[]string{"*"}, "/foo/bar/test.go", true, true},
@@ -255,8 +256,25 @@ var filterListTests = []struct {
{[]string{"?"}, "/foo/bar/test.go", false, true},
{[]string{"?", "x"}, "/foo/bar/x", true, true},
{[]string{"/*/*/bar/test.*"}, "/foo/bar/test.go", false, false},
{[]string{"/*/*/bar/test.*"}, "/foo/bar/bar", false, true},
{[]string{"/*/*/bar/test.*", "*.go"}, "/foo/bar/test.go", true, true},
{[]string{"", "*.c"}, "/foo/bar/test.go", false, true},
{[]string{"!**", "*.go"}, "/foo/bar/test.go", true, true},
{[]string{"!**", "*.c"}, "/foo/bar/test.go", false, true},
{[]string{"/foo/*/test.*", "!*.c"}, "/foo/bar/test.c", false, false},
{[]string{"/foo/*/test.*", "!*.c"}, "/foo/bar/test.go", true, true},
{[]string{"/foo/*/*", "!test.*", "*.c"}, "/foo/bar/test.go", false, true},
{[]string{"/foo/*/*", "!test.*", "*.c"}, "/foo/bar/test.c", true, true},
{[]string{"/foo/*/*", "!test.*", "*.c"}, "/foo/bar/file.go", true, true},
{[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/other/test.go", true, true},
{[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/bar", false, false},
{[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/bar/test.go", false, false},
{[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/bar/test.go/child", false, false},
{[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar", "/foo/bar/test*"}, "/foo/bar/test.go/child", true, true},
{[]string{"/foo/bar/*"}, "/foo", false, true},
{[]string{"/foo/bar/*", "!/foo/bar/[a-m]*"}, "/foo", false, true},
{[]string{"/foo/**/test.c"}, "/foo/bar/foo/bar/test.c", true, true},
{[]string{"/foo/*/test.c"}, "/foo/bar/foo/bar/test.c", false, false},
}
func TestList(t *testing.T) {