Refactor repository structure

Merge Map data type into Tree.

cmd/restic/cmd_backup.go

@@ -59,7 +59,7 @@ func format_duration(d time.Duration) string {
 }
 
 func print_tree2(indent int, t *restic.Tree) {
-	for _, node := range *t {
+	for _, node := range t.Nodes {
 		if node.Tree() != nil {
 			fmt.Printf("%s%s/\n", strings.Repeat("  ", indent), node.Name)
 			print_tree2(indent+1, node.Tree())
@@ -120,22 +120,19 @@ func (cmd CmdBackup) Execute(args []string) error {
 		return err
 	}
 
-	var bl *restic.BlobList
 	if parentSnapshotID != nil {
 		fmt.Printf("load old snapshot\n")
-		ch := restic.NewContentHandler(s)
-		sn, err := ch.LoadSnapshot(parentSnapshotID)
+		sn, err := restic.LoadSnapshot(s, parentSnapshotID)
 		if err != nil {
 			return err
 		}
 
-		oldTree, err := restic.LoadTreeRecursive(filepath.Dir(sn.Dir), ch, sn.Tree)
+		oldTree, err := restic.LoadTreeRecursive(filepath.Dir(sn.Dir), s, sn.Tree)
 		if err != nil {
 			return err
 		}
 
-		bl = restic.NewBlobList()
-		err = newTree.CopyFrom(bl, oldTree, ch.BlobList())
+		err = newTree.CopyFrom(oldTree, &s)
 		if err != nil {
 			return err
 		}
@@ -175,14 +172,14 @@ func (cmd CmdBackup) Execute(args []string) error {
 		}
 	}
 
-	arch, err := restic.NewArchiver(s, bl, archiveProgress)
+	arch, err := restic.NewArchiver(s, archiveProgress)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "err: %v\n", err)
 	}
 
 	arch.Error = func(dir string, fi os.FileInfo, err error) error {
 		// TODO: make ignoring errors configurable
-		fmt.Fprintf(os.Stderr, "\nerror for %s: %v\n", dir, err)
+		fmt.Fprintf(os.Stderr, "\x1b[2K\rerror for %s: %v\n", dir, err)
 		return nil
 	}
 

cmd/restic/cmd_cat.go

@@ -53,66 +53,26 @@ func (cmd CmdCat) Execute(args []string) error {
 		}
 	}
 
-	ch := restic.NewContentHandler(s)
-
 	switch tpe {
 	case "blob":
-		err = ch.LoadAllMaps()
-		if err != nil {
-			return err
-		}
-
-		// try id
-		data, err := ch.Load(backend.Data, id)
+		// try storage id
+		data, err := s.LoadID(backend.Data, id)
 		if err == nil {
 			_, err = os.Stdout.Write(data)
 			return err
 		}
 
-		// try storage id
-		buf, err := s.Get(backend.Data, id)
-		if err != nil {
-			return err
-		}
-
-		// decrypt
-		buf, err = s.Decrypt(buf)
-		if err != nil {
-			return err
-		}
-
-		_, err = os.Stdout.Write(buf)
+		_, err = os.Stdout.Write(data)
 		return err
 
 	case "tree":
-		err = ch.LoadAllMaps()
+		// try storage id
+		tree := &restic.Tree{}
+		err := s.LoadJSONID(backend.Tree, id, tree)
 		if err != nil {
 			return err
 		}
 
-		var tree restic.Tree
-		// try id
-		err := ch.LoadJSON(backend.Tree, id, &tree)
-		if err != nil {
-			// try storage id
-			buf, err := s.Get(backend.Tree, id)
-			if err != nil {
-				return err
-			}
-
-			// decrypt
-			buf, err = s.Decrypt(buf)
-			if err != nil {
-				return err
-			}
-
-			// unmarshal
-			err = json.Unmarshal(backend.Uncompress(buf), &tree)
-			if err != nil {
-				return err
-			}
-		}
-
 		buf, err := json.MarshalIndent(&tree, "", "  ")
 		if err != nil {
 			return err
@@ -120,26 +80,10 @@ func (cmd CmdCat) Execute(args []string) error {
 
 		fmt.Println(string(buf))
 
-		return nil
-	case "map":
-		var bl restic.BlobList
-		err := ch.LoadJSONRaw(backend.Map, id, &bl)
-		if err != nil {
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&bl, "", "  ")
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(string(buf))
-
 		return nil
 	case "snapshot":
-		var sn restic.Snapshot
-
-		err = ch.LoadJSONRaw(backend.Snapshot, id, &sn)
+		sn := &restic.Snapshot{}
+		err = s.LoadJSONID(backend.Snapshot, id, sn)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_find.go

@@ -63,16 +63,15 @@ func parseTime(str string) (time.Time, error) {
 	return time.Time{}, fmt.Errorf("unable to parse time: %q", str)
 }
 
-func (c CmdFind) findInTree(ch *restic.ContentHandler, id backend.ID, path string) ([]findResult, error) {
-	debug("checking tree %v\n", id)
-
-	tree, err := restic.LoadTree(ch, id)
+func (c CmdFind) findInTree(s restic.Server, blob restic.Blob, path string) ([]findResult, error) {
+	debug("checking tree %v\n", blob)
+	tree, err := restic.LoadTree(s, blob)
 	if err != nil {
 		return nil, err
 	}
 
 	results := []findResult{}
-	for _, node := range tree {
+	for _, node := range tree.Nodes {
 		debug("  testing entry %q\n", node.Name)
 
 		m, err := filepath.Match(c.pattern, node.Name)
@@ -98,7 +97,12 @@ func (c CmdFind) findInTree(ch *restic.ContentHandler, id backend.ID, path strin
 		}
 
 		if node.Type == "dir" {
-			subdirResults, err := c.findInTree(ch, node.Subtree, filepath.Join(path, node.Name))
+			b, err := tree.Map.FindID(node.Subtree)
+			if err != nil {
+				return nil, err
+			}
+
+			subdirResults, err := c.findInTree(s, b, filepath.Join(path, node.Name))
 			if err != nil {
 				return nil, err
 			}
@@ -113,13 +117,12 @@ func (c CmdFind) findInTree(ch *restic.ContentHandler, id backend.ID, path strin
 func (c CmdFind) findInSnapshot(s restic.Server, id backend.ID) error {
 	debug("searching in snapshot %s\n  for entries within [%s %s]", id, c.oldest, c.newest)
 
-	ch := restic.NewContentHandler(s)
-	sn, err := ch.LoadSnapshot(id)
+	sn, err := restic.LoadSnapshot(s, id)
 	if err != nil {
 		return err
 	}
 
-	results, err := c.findInTree(ch, sn.Tree, "")
+	results, err := c.findInTree(s, sn.Tree, "")
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_fsck.go

@@ -16,9 +16,8 @@ type CmdFsck struct {
 	RemoveOrphaned bool   `short:"r" long:"remove-orphaned" description:"Remove orphaned blobs (implies -o)"`
 
 	// lists checking for orphaned blobs
-	o_data  *restic.BlobList
-	o_trees *restic.BlobList
-	o_maps  *restic.BlobList
+	o_data  *backend.IDSet
+	o_trees *backend.IDSet
 }
 
 func init() {
@@ -31,91 +30,103 @@ func init() {
 	}
 }
 
-func fsckFile(opts CmdFsck, ch *restic.ContentHandler, IDs []backend.ID) error {
+func fsckFile(opts CmdFsck, s restic.Server, m *restic.Map, IDs []backend.ID) (uint64, error) {
+	var bytes uint64
+
 	for _, id := range IDs {
 		debug("checking data blob %v\n", id)
 
+		// test if blob is in map
+		blob, err := m.FindID(id)
+		if err != nil {
+			return 0, fmt.Errorf("storage ID for data blob %v not found", id)
+		}
+
+		bytes += blob.Size
+
 		if opts.CheckData {
 			// load content
-			_, err := ch.Load(backend.Data, id)
+			_, err := s.Load(backend.Data, blob)
 			if err != nil {
-				return err
+				return 0, err
 			}
 		} else {
 			// test if data blob is there
-			ok, err := ch.Test(backend.Data, id)
+			ok, err := s.Test(backend.Data, blob.Storage)
 			if err != nil {
-				return err
+				return 0, err
 			}
 
 			if !ok {
-				return fmt.Errorf("data blob %v not found", id)
+				return 0, fmt.Errorf("data blob %v not found", id)
 			}
 		}
 
 		// if orphan check is active, record storage id
 		if opts.o_data != nil {
-			// lookup storage ID
-			sid, err := ch.Lookup(id)
-			if err != nil {
-				return err
-			}
-
-			// add ID to list
-			opts.o_data.Insert(restic.Blob{ID: sid})
+			opts.o_data.Insert(blob.Storage)
 		}
 	}
 
-	return nil
+	return bytes, nil
 }
 
-func fsckTree(opts CmdFsck, ch *restic.ContentHandler, id backend.ID) error {
-	debug("checking tree %v\n", id)
+func fsckTree(opts CmdFsck, s restic.Server, blob restic.Blob) error {
+	debug("checking tree %v\n", blob.ID)
 
-	tree, err := restic.LoadTree(ch, id)
+	tree, err := restic.LoadTree(s, blob)
 	if err != nil {
 		return err
 	}
 
 	// if orphan check is active, record storage id
 	if opts.o_trees != nil {
-		// lookup storage ID
-		sid, err := ch.Lookup(id)
-		if err != nil {
-			return err
-		}
-
 		// add ID to list
-		opts.o_trees.Insert(restic.Blob{ID: sid})
+		opts.o_trees.Insert(blob.Storage)
 	}
 
 	var firstErr error
 
-	for i, node := range tree {
+	for i, node := range tree.Nodes {
 		if node.Name == "" {
-			return fmt.Errorf("node %v of tree %v has no name", i, id)
+			return fmt.Errorf("node %v of tree %v has no name", i, blob.ID)
 		}
 
 		if node.Type == "" {
-			return fmt.Errorf("node %q of tree %v has no type", node.Name, id)
+			return fmt.Errorf("node %q of tree %v has no type", node.Name, blob.ID)
 		}
 
 		switch node.Type {
 		case "file":
-			if node.Content == nil && node.Error == "" {
-				return fmt.Errorf("file node %q of tree %v has no content", node.Name, id)
+			if node.Content == nil {
+				return fmt.Errorf("file node %q of tree %v has no content: %v", node.Name, blob.ID, node)
 			}
 
-			err := fsckFile(opts, ch, node.Content)
+			if node.Content == nil && node.Error == "" {
+				return fmt.Errorf("file node %q of tree %v has no content", node.Name, blob.ID)
+			}
+
+			bytes, err := fsckFile(opts, s, tree.Map, node.Content)
 			if err != nil {
 				return err
 			}
+
+			if bytes != node.Size {
+				return fmt.Errorf("file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, blob, node.Size, bytes)
+			}
 		case "dir":
 			if node.Subtree == nil {
-				return fmt.Errorf("dir node %q of tree %v has no subtree", node.Name, id)
+				return fmt.Errorf("dir node %q of tree %v has no subtree", node.Name, blob.ID)
 			}
 
-			err := fsckTree(opts, ch, node.Subtree)
+			// lookup blob
+			subtreeBlob, err := tree.Map.FindID(node.Subtree)
+			if err != nil {
+				firstErr = err
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+			}
+
+			err = fsckTree(opts, s, subtreeBlob)
 			if err != nil {
 				firstErr = err
 				fmt.Fprintf(os.Stderr, "%v\n", err)
@@ -129,27 +140,22 @@ func fsckTree(opts CmdFsck, ch *restic.ContentHandler, id backend.ID) error {
 func fsck_snapshot(opts CmdFsck, s restic.Server, id backend.ID) error {
 	debug("checking snapshot %v\n", id)
 
-	ch := restic.NewContentHandler(s)
-
-	sn, err := ch.LoadSnapshot(id)
+	sn, err := restic.LoadSnapshot(s, id)
 	if err != nil {
-		return err
+		return fmt.Errorf("loading snapshot %v failed: %v", id, err)
 	}
 
-	if sn.Tree == nil {
-		return fmt.Errorf("snapshot %v has no content", sn.ID)
+	if !sn.Tree.Valid() {
+		return fmt.Errorf("snapshot %v has invalid tree %v", sn.ID, sn.Tree)
 	}
 
-	if sn.Map == nil {
-		return fmt.Errorf("snapshot %v has no map", sn.ID)
+	err = fsckTree(opts, s, sn.Tree)
+	if err != nil {
+		debug("  checking tree %v for snapshot %v\n", sn.Tree, id)
+		fmt.Fprintf(os.Stderr, "snapshot %v:\n  error for tree %v:\n    %v\n", id, sn.Tree, err)
 	}
 
-	// if orphan check is active, record storage id for map
-	if opts.o_maps != nil {
-		opts.o_maps.Insert(restic.Blob{ID: sn.Map})
-	}
-
-	return fsckTree(opts, ch, sn.Tree)
+	return err
 }
 
 func (cmd CmdFsck) Usage() string {
@@ -185,9 +191,8 @@ func (cmd CmdFsck) Execute(args []string) error {
 	}
 
 	if cmd.Orphaned {
-		cmd.o_data = restic.NewBlobList()
-		cmd.o_trees = restic.NewBlobList()
-		cmd.o_maps = restic.NewBlobList()
+		cmd.o_data = backend.NewIDSet()
+		cmd.o_trees = backend.NewIDSet()
 	}
 
 	list, err := s.List(backend.Snapshot)
@@ -214,11 +219,10 @@ func (cmd CmdFsck) Execute(args []string) error {
 	l := []struct {
 		desc string
 		tpe  backend.Type
-		list *restic.BlobList
+		set  *backend.IDSet
 	}{
 		{"data blob", backend.Data, cmd.o_data},
 		{"tree", backend.Tree, cmd.o_trees},
-		{"maps", backend.Map, cmd.o_maps},
 	}
 
 	for _, d := range l {
@@ -230,8 +234,8 @@ func (cmd CmdFsck) Execute(args []string) error {
 		}
 
 		for _, id := range blobs {
-			_, err := d.list.Find(restic.Blob{ID: id})
-			if err == restic.ErrBlobNotFound {
+			err := d.set.Find(id)
+			if err != nil {
 				if !cmd.RemoveOrphaned {
 					fmt.Printf("orphaned %v %v\n", d.desc, id)
 					continue

cmd/restic/cmd_list.go

@@ -46,8 +46,6 @@ func (cmd CmdList) Execute(args []string) error {
 		each = s.EachDecrypted
 	case "snapshots":
 		t = backend.Snapshot
-	case "maps":
-		t = backend.Map
 	case "keys":
 		t = backend.Key
 	case "locks":

cmd/restic/cmd_ls.go

@@ -37,19 +37,22 @@ func print_node(prefix string, n *restic.Node) string {
 	}
 }
 
-func print_tree(prefix string, ch *restic.ContentHandler, id backend.ID) error {
-	tree := &restic.Tree{}
-
-	err := ch.LoadJSON(backend.Tree, id, tree)
+func print_tree(prefix string, s restic.Server, blob restic.Blob) error {
+	tree, err := restic.LoadTree(s, blob)
 	if err != nil {
 		return err
 	}
 
-	for _, entry := range *tree {
+	for _, entry := range tree.Nodes {
 		fmt.Println(print_node(prefix, entry))
 
 		if entry.Type == "dir" && entry.Subtree != nil {
-			err = print_tree(filepath.Join(prefix, entry.Name), ch, entry.Subtree)
+			b, err := tree.Map.FindID(entry.Subtree)
+			if err != nil {
+				return err
+			}
+
+			err = print_tree(filepath.Join(prefix, entry.Name), s, b)
 			if err != nil {
 				return err
 			}
@@ -78,17 +81,12 @@ func (cmd CmdLs) Execute(args []string) error {
 		return err
 	}
 
-	ch := restic.NewContentHandler(s)
-	if err != nil {
-		return err
-	}
-
-	sn, err := ch.LoadSnapshot(id)
+	sn, err := restic.LoadSnapshot(s, id)
 	if err != nil {
 		return err
 	}
 
 	fmt.Printf("snapshot of %s at %s:\n", sn.Dir, sn.Time)
 
-	return print_tree("", ch, sn.Tree)
+	return print_tree("", s, sn.Tree)
 }

cmd/restic/cmd_snapshots.go

@@ -97,15 +97,13 @@ func (cmd CmdSnapshots) Execute(args []string) error {
 		return err
 	}
 
-	ch := restic.NewContentHandler(s)
-
 	tab := NewTable()
 	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %s", "ID", "Date", "Source", "Directory")
 	tab.RowFormat = "%-8s  %-19s  %-10s  %s"
 
 	list := []*restic.Snapshot{}
 	s.EachID(backend.Snapshot, func(id backend.ID) {
-		sn, err := restic.LoadSnapshot(ch, id)
+		sn, err := restic.LoadSnapshot(s, id)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
 			return