Merge pull request #1090 from middelink/fix-1081

Update HasTags() and HasPaths() to follow #1081 feature request
This commit is contained in:
Alexander Neumann
2017-07-19 17:11:18 +02:00
18 changed files with 335 additions and 83 deletions
+4 -4
View File
@@ -68,12 +68,12 @@ func init() {
f := cmdBackup.Flags()
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringSliceVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}
@@ -392,7 +392,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentSnapshotID == nil {
id, err := restic.FindLatestSnapshot(context.TODO(), repo, target, opts.Tags, opts.Hostname)
id, err := restic.FindLatestSnapshot(context.TODO(), repo, target, []restic.TagList{opts.Tags}, opts.Hostname)
if err == nil {
parentSnapshotID = &id
} else if err != restic.ErrNoSnapshotFound {
+4 -4
View File
@@ -34,7 +34,7 @@ type FindOptions struct {
ListLong bool
Host string
Paths []string
Tags []string
Tags restic.TagLists
}
var findOptions FindOptions
@@ -45,13 +45,13 @@ func init() {
f := cmdFind.Flags()
f.StringVarP(&findOptions.Oldest, "oldest", "O", "", "oldest modification date/time")
f.StringVarP(&findOptions.Newest, "newest", "N", "", "newest modification date/time")
f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
f.StringArrayVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
f.Var(&findOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
f.StringArrayVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
}
type findPattern struct {
+5 -5
View File
@@ -31,10 +31,10 @@ type ForgetOptions struct {
Weekly int
Monthly int
Yearly int
KeepTags []string
KeepTags restic.TagLists
Host string
Tags []string
Tags restic.TagLists
Paths []string
GroupByTags bool
@@ -55,14 +55,14 @@ func init() {
f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "keep snapshots with this `tag` (can be specified multiple times)")
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
f.BoolVarP(&forgetOptions.GroupByTags, "group-by-tags", "G", false, "Group by host,paths,tags instead of just host,paths")
// Sadly the commonly used shortcut `H` is already used.
f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
// Deprecated since 2017-03-07.
f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
f.StringSliceVar(&forgetOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
f.StringSliceVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
+3 -3
View File
@@ -28,7 +28,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th
type LsOptions struct {
ListLong bool
Host string
Tags []string
Tags restic.TagLists
Paths []string
}
@@ -41,8 +41,8 @@ func init() {
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}
func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
+4 -3
View File
@@ -6,6 +6,7 @@ package main
import (
"context"
"os"
"restic"
"github.com/spf13/cobra"
@@ -37,7 +38,7 @@ type MountOptions struct {
AllowRoot bool
AllowOther bool
Host string
Tags []string
Tags restic.TagLists
Paths []string
}
@@ -52,8 +53,8 @@ func init() {
mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
}
func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
+5 -5
View File
@@ -31,7 +31,7 @@ type RestoreOptions struct {
Target string
Host string
Paths []string
Tags []string
Tags restic.TagLists
}
var restoreOptions RestoreOptions
@@ -40,13 +40,13 @@ func init() {
cmdRoot.AddCommand(cmdRestore)
flags := cmdRestore.Flags()
flags.StringSliceVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
flags.StringSliceVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
}
func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
+3 -3
View File
@@ -26,7 +26,7 @@ The "snapshots" command lists all snapshots stored in the repository.
// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
Host string
Tags []string
Tags restic.TagLists
Paths []string
}
@@ -37,8 +37,8 @@ func init() {
f := cmdSnapshots.Flags()
f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
}
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
+3 -3
View File
@@ -31,7 +31,7 @@ When no snapshot-ID is given, all snapshots matching the host, tag and path filt
type TagOptions struct {
Host string
Paths []string
Tags []string
Tags restic.TagLists
SetTags []string
AddTags []string
RemoveTags []string
@@ -48,8 +48,8 @@ func init() {
tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
tagFlags.Var(&tagOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
tagFlags.StringArrayVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
}
func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
+1 -1
View File
@@ -8,7 +8,7 @@ import (
)
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot)
go func() {
defer close(out)
+1 -1
View File
@@ -16,7 +16,7 @@ import (
type Config struct {
OwnerIsRoot bool
Host string
Tags []string
Tags []restic.TagList
Paths []string
}
+45 -25
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"os/user"
"path/filepath"
"restic/debug"
"time"
)
@@ -130,46 +131,65 @@ func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
return
}
// HasTags returns true if the snapshot has at least all of tags.
func (sn *Snapshot) HasTags(tags []string) bool {
nextTag:
for _, tag := range tags {
for _, snTag := range sn.Tags {
if tag == snTag {
continue nextTag
}
func (sn *Snapshot) hasTag(tag string) bool {
for _, snTag := range sn.Tags {
if tag == snTag {
return true
}
}
return false
}
return false
// HasTags returns true if the snapshot has all the tags in l.
func (sn *Snapshot) HasTags(l []string) bool {
for _, tag := range l {
if !sn.hasTag(tag) {
return false
}
}
return true
}
// HasPaths returns true if the snapshot has at least all of paths.
// HasTagList returns true if the snapshot satisfies at least one TagList,
// so there is a TagList in l for which all tags are included in sn.
func (sn *Snapshot) HasTagList(l []TagList) bool {
debug.Log("testing snapshot with tags %v against list: %v", sn.Tags, l)
if len(l) == 0 {
return true
}
for _, tags := range l {
if sn.HasTags(tags) {
debug.Log(" snapshot satisfies %v", tags, l)
return true
}
}
return false
}
func (sn *Snapshot) hasPath(path string) bool {
for _, snPath := range sn.Paths {
if path == snPath {
return true
}
}
return false
}
// HasPaths returns true if the snapshot has all of the paths.
func (sn *Snapshot) HasPaths(paths []string) bool {
nextPath:
for _, path := range paths {
for _, snPath := range sn.Paths {
if path == snPath {
continue nextPath
}
if !sn.hasPath(path) {
return false
}
return false
}
return true
}
// SamePaths returns true if the snapshot matches the entire paths set
func (sn *Snapshot) SamePaths(paths []string) bool {
if len(sn.Paths) != len(paths) {
return false
}
return sn.HasPaths(paths)
}
// Snapshots is a list of snapshots.
type Snapshots []*Snapshot
+17 -7
View File
@@ -12,7 +12,7 @@ import (
var ErrNoSnapshotFound = errors.New("no snapshot found")
// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters.
func FindLatestSnapshot(ctx context.Context, repo Repository, targets []string, tags []string, hostname string) (ID, error) {
func FindLatestSnapshot(ctx context.Context, repo Repository, targets []string, tagLists []TagList, hostname string) (ID, error) {
var (
latest time.Time
latestID ID
@@ -24,11 +24,21 @@ func FindLatestSnapshot(ctx context.Context, repo Repository, targets []string,
if err != nil {
return ID{}, errors.Errorf("Error listing snapshot: %v", err)
}
if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) {
latest = snapshot.Time
latestID = snapshotID
found = true
if snapshot.Time.Before(latest) || (hostname != "" && hostname != snapshot.Hostname) {
continue
}
if !snapshot.HasTagList(tagLists) {
continue
}
if !snapshot.HasPaths(targets) {
continue
}
latest = snapshot.Time
latestID = snapshotID
found = true
}
if !found {
@@ -53,7 +63,7 @@ func FindSnapshot(repo Repository, s string) (ID, error) {
// FindFilteredSnapshots yields Snapshots filtered from the list of all
// snapshots.
func FindFilteredSnapshots(ctx context.Context, repo Repository, host string, tags []string, paths []string) Snapshots {
func FindFilteredSnapshots(ctx context.Context, repo Repository, host string, tags []TagList, paths []string) Snapshots {
results := make(Snapshots, 0, 20)
for id := range repo.List(ctx, SnapshotFile) {
@@ -62,7 +72,7 @@ func FindFilteredSnapshots(ctx context.Context, repo Repository, host string, ta
fmt.Fprintf(os.Stderr, "could not load snapshot %v: %v\n", id.Str(), err)
continue
}
if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
if (host != "" && host != sn.Hostname) || !sn.HasTagList(tags) || !sn.HasPaths(paths) {
continue
}
+10 -9
View File
@@ -8,13 +8,13 @@ import (
// ExpirePolicy configures which snapshots should be automatically removed.
type ExpirePolicy struct {
Last int // keep the last n snapshots
Hourly int // keep the last n hourly snapshots
Daily int // keep the last n daily snapshots
Weekly int // keep the last n weekly snapshots
Monthly int // keep the last n monthly snapshots
Yearly int // keep the last n yearly snapshots
Tags []string // keep all snapshots with these tags
Last int // keep the last n snapshots
Hourly int // keep the last n hourly snapshots
Daily int // keep the last n daily snapshots
Weekly int // keep the last n weekly snapshots
Monthly int // keep the last n monthly snapshots
Yearly int // keep the last n yearly snapshots
Tags []TagList // keep all snapshots that include at least one of the tag lists.
}
// Sum returns the maximum number of snapshots to be kept according to this
@@ -94,11 +94,12 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) {
var keepSnap bool
// Tags are handled specially as they are not counted.
if len(p.Tags) > 0 {
if cur.HasTags(p.Tags) {
for _, l := range p.Tags {
if cur.HasTags(l) {
keepSnap = true
}
}
// Now update the other buckets and see if they have some counts left.
for i, b := range buckets {
if b.Count > 0 {
@@ -27,7 +27,7 @@ func TestExpireSnapshotOps(t *testing.T) {
p *restic.ExpirePolicy
}{
{true, 0, &restic.ExpirePolicy{}},
{true, 0, &restic.ExpirePolicy{Tags: []string{}}},
{true, 0, &restic.ExpirePolicy{Tags: []restic.TagList{}}},
{false, 22, &restic.ExpirePolicy{Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}},
}
for i, d := range data {
@@ -163,8 +163,9 @@ var expireTests = []restic.ExpirePolicy{
{Daily: 2, Weekly: 2, Monthly: 6},
{Yearly: 10},
{Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10},
{Tags: []string{"foo"}},
{Tags: []string{"foo", "bar"}},
{Tags: []restic.TagList{{"foo"}}},
{Tags: []restic.TagList{{"foo", "bar"}}},
{Tags: []restic.TagList{{"foo"}, {"bar"}}},
}
func TestApplyPolicy(t *testing.T) {
+62
View File
@@ -0,0 +1,62 @@
package restic
import (
"fmt"
"strings"
)
// TagList is a list of tags.
type TagList []string
// splitTagList splits a string into a list of tags. The tags in the string
// need to be separated by commas. Whitespace is stripped around the individual
// tags.
func splitTagList(s string) (l TagList) {
for _, t := range strings.Split(s, ",") {
l = append(l, strings.TrimSpace(t))
}
return l
}
func (l TagList) String() string {
return "[" + strings.Join(l, ", ") + "]"
}
// Set updates the TagList's value.
func (l *TagList) Set(s string) error {
*l = splitTagList(s)
return nil
}
// Type returns a description of the type.
func (TagList) Type() string {
return "TagList"
}
// TagLists consists of several TagList.
type TagLists []TagList
// splitTagLists splits a slice of strings into a slice of TagLists using
// SplitTagList.
func splitTagLists(s []string) (l TagLists) {
l = make([]TagList, 0, len(s))
for _, t := range s {
l = append(l, splitTagList(t))
}
return l
}
func (l TagLists) String() string {
return fmt.Sprintf("%v", []TagList(l))
}
// Set updates the TagList's value.
func (l *TagLists) Set(s string) error {
*l = append(*l, splitTagList(s))
return nil
}
// Type returns a description of the type.
func (TagLists) Type() string {
return "TagLists"
}
+131
View File
@@ -0,0 +1,131 @@
[
{
"time": "2014-11-15T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo",
"bar"
]
},
{
"time": "2014-11-13T10:20:30.1Z",
"tree": null,
"paths": null,
"tags": [
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-11-12T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-11-10T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-11-08T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-22T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-20T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-11T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-10T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-09T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-08T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-06T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-05T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-02T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
},
{
"time": "2014-10-01T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
}
]