Major restructure, bundle blobs

This commit also breaks the repository format.

cmd/restic/cmd_backup.go

@@ -96,26 +96,6 @@ func (cmd CmdBackup) Usage() string {
 	return "DIR/FILE [snapshot-ID]"
 }
 
-func newCacheRefreshProgress() *restic.Progress {
-	p := restic.NewProgress(time.Second)
-	p.OnStart = func() {
-		fmt.Printf("refreshing cache\n")
-	}
-
-	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
-		return p
-	}
-
-	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
-		fmt.Printf("\x1b[2K[%s] %d trees loaded\r", formatDuration(d), s.Trees)
-	}
-	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
-		fmt.Printf("\x1b[2Krefreshed cache in %s\n", formatDuration(d))
-	}
-
-	return p
-}
-
 func newScanProgress() *restic.Progress {
 	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
 		return nil
@@ -200,6 +180,11 @@ func (cmd CmdBackup) Execute(args []string) error {
 		return err
 	}
 
+	err = s.LoadIndex()
+	if err != nil {
+		return err
+	}
+
 	var (
 		parentSnapshot   string
 		parentSnapshotID backend.ID
@@ -278,16 +263,7 @@ func (cmd CmdBackup) Execute(args []string) error {
 		return nil
 	}
 
-	err = arch.Cache().RefreshSnapshots(s, newCacheRefreshProgress())
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("loading blobs\n")
-	err = arch.Preload()
-	if err != nil {
-		return err
-	}
+	// TODO: load index
 
 	_, id, err := arch.Snapshot(newArchiveProgress(stat), target, parentSnapshotID)
 	if err != nil {

cmd/restic/cmd_cat.go

@@ -4,10 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/restic/restic"
 	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
+	"github.com/restic/restic/pack"
 	"github.com/restic/restic/server"
 )
 
@@ -24,7 +27,7 @@ func init() {
 }
 
 func (cmd CmdCat) Usage() string {
-	return "[data|tree|snapshot|key|masterkey|lock] ID"
+	return "[pack|blob|tree|snapshot|key|masterkey|lock] ID"
 }
 
 func (cmd CmdCat) Execute(args []string) error {
@@ -62,37 +65,20 @@ func (cmd CmdCat) Execute(args []string) error {
 		}
 	}
 
+	// handle all types that don't need an index
 	switch tpe {
-	case "data":
-		// try storage id
-		data, err := s.LoadID(backend.Data, id)
-		if err == nil {
-			_, err = os.Stdout.Write(data)
+	case "index":
+		buf, err := s.Load(backend.Index, id)
+		if err != nil {
 			return err
 		}
 
-		_, err = os.Stdout.Write(data)
+		_, err = os.Stdout.Write(append(buf, '\n'))
 		return err
 
-	case "tree":
-		// try storage id
-		tree := &restic.Tree{}
-		err := s.LoadJSONID(backend.Tree, id, tree)
-		if err != nil {
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&tree, "", "  ")
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(string(buf))
-
-		return nil
 	case "snapshot":
 		sn := &restic.Snapshot{}
-		err = s.LoadJSONID(backend.Snapshot, id, sn)
+		err = s.LoadJSONEncrypted(backend.Snapshot, id, sn)
 		if err != nil {
 			return err
 		}
@@ -136,6 +122,52 @@ func (cmd CmdCat) Execute(args []string) error {
 		return nil
 	case "lock":
 		return errors.New("not yet implemented")
+	}
+
+	// load index, handle all the other types
+	err = s.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	switch tpe {
+	case "pack":
+		rd, err := s.Backend().Get(backend.Data, id.String())
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(os.Stdout, rd)
+		return err
+
+	case "blob":
+		data, err := s.LoadBlob(pack.Data, id)
+		if err == nil {
+			_, err = os.Stdout.Write(data)
+			return err
+		}
+
+		_, err = os.Stdout.Write(data)
+		return err
+
+	case "tree":
+		debug.Log("cat", "cat tree %v", id.Str())
+		tree := restic.NewTree()
+		err = s.LoadJSONPack(pack.Tree, id, tree)
+		if err != nil {
+			debug.Log("cat", "unable to load tree %v: %v", id.Str(), err)
+			return err
+		}
+
+		buf, err := json.MarshalIndent(&tree, "", "  ")
+		if err != nil {
+			debug.Log("cat", "error json.MarshalIndent(): %v", err)
+			return err
+		}
+
+		_, err = os.Stdout.Write(append(buf, '\n'))
+		return nil
+
 	default:
 		return errors.New("invalid type")
 	}

cmd/restic/cmd_find.go

@@ -59,9 +59,9 @@ func parseTime(str string) (time.Time, error) {
 	return time.Time{}, fmt.Errorf("unable to parse time: %q", str)
 }
 
-func (c CmdFind) findInTree(s *server.Server, blob server.Blob, path string) ([]findResult, error) {
-	debug.Log("restic.find", "checking tree %v\n", blob)
-	tree, err := restic.LoadTree(s, blob)
+func (c CmdFind) findInTree(s *server.Server, id backend.ID, path string) ([]findResult, error) {
+	debug.Log("restic.find", "checking tree %v\n", id)
+	tree, err := restic.LoadTree(s, id)
 	if err != nil {
 		return nil, err
 	}
@@ -93,12 +93,7 @@ func (c CmdFind) findInTree(s *server.Server, blob server.Blob, path string) ([]
 		}
 
 		if node.Type == "dir" {
-			b, err := tree.Map.FindID(node.Subtree)
-			if err != nil {
-				return nil, err
-			}
-
-			subdirResults, err := c.findInTree(s, b, filepath.Join(path, node.Name))
+			subdirResults, err := c.findInTree(s, id, filepath.Join(path, node.Name))
 			if err != nil {
 				return nil, err
 			}

cmd/restic/cmd_fsck.go

@@ -7,7 +7,9 @@ import (
 
 	"github.com/restic/restic"
 	"github.com/restic/restic/backend"
+	"github.com/restic/restic/crypto"
 	"github.com/restic/restic/debug"
+	"github.com/restic/restic/pack"
 	"github.com/restic/restic/server"
 )
 
@@ -32,31 +34,31 @@ func init() {
 	}
 }
 
-func fsckFile(opts CmdFsck, s *server.Server, m *restic.Map, IDs []backend.ID) (uint64, error) {
+func fsckFile(opts CmdFsck, s *server.Server, IDs []backend.ID) (uint64, error) {
 	debug.Log("restic.fsckFile", "checking file %v", IDs)
 	var bytes uint64
 
 	for _, id := range IDs {
 		debug.Log("restic.fsck", "  checking data blob %v\n", id)
 
-		// test if blob is in map
-		blob, err := m.FindID(id)
+		// test if blob is in the index
+		packID, tpe, _, length, err := s.Index().Lookup(id)
 		if err != nil {
-			return 0, fmt.Errorf("storage ID for data blob %v not found", id)
+			return 0, fmt.Errorf("storage for blob %v (%v) not found", id, tpe)
 		}
 
-		bytes += blob.Size
-		debug.Log("restic.fsck", "  data blob found: %v\n", blob)
+		bytes += uint64(length - crypto.Extension)
+		debug.Log("restic.fsck", "  blob found in pack %v\n", packID)
 
 		if opts.CheckData {
 			// load content
-			_, err := s.Load(backend.Data, blob)
+			_, err := s.LoadBlob(pack.Data, id)
 			if err != nil {
 				return 0, err
 			}
 		} else {
 			// test if data blob is there
-			ok, err := s.Test(backend.Data, blob.Storage.String())
+			ok, err := s.Test(backend.Data, packID.String())
 			if err != nil {
 				return 0, err
 			}
@@ -68,17 +70,17 @@ func fsckFile(opts CmdFsck, s *server.Server, m *restic.Map, IDs []backend.ID) (
 
 		// if orphan check is active, record storage id
 		if opts.o_data != nil {
-			opts.o_data.Insert(blob.Storage)
+			opts.o_data.Insert(id)
 		}
 	}
 
 	return bytes, nil
 }
 
-func fsckTree(opts CmdFsck, s *server.Server, blob server.Blob) error {
-	debug.Log("restic.fsckTree", "checking tree %v", blob)
+func fsckTree(opts CmdFsck, s *server.Server, id backend.ID) error {
+	debug.Log("restic.fsckTree", "checking tree %v", id.Str())
 
-	tree, err := restic.LoadTree(s, blob)
+	tree, err := restic.LoadTree(s, id)
 	if err != nil {
 		return err
 	}
@@ -86,7 +88,7 @@ func fsckTree(opts CmdFsck, s *server.Server, blob server.Blob) error {
 	// if orphan check is active, record storage id
 	if opts.o_trees != nil {
 		// add ID to list
-		opts.o_trees.Insert(blob.Storage)
+		opts.o_trees.Insert(id)
 	}
 
 	var firstErr error
@@ -95,23 +97,23 @@ func fsckTree(opts CmdFsck, s *server.Server, blob server.Blob) error {
 
 	for i, node := range tree.Nodes {
 		if node.Name == "" {
-			return fmt.Errorf("node %v of tree %v has no name", i, blob.ID)
+			return fmt.Errorf("node %v of tree %v has no name", i, id.Str())
 		}
 
 		if node.Type == "" {
-			return fmt.Errorf("node %q of tree %v has no type", node.Name, blob.ID)
+			return fmt.Errorf("node %q of tree %v has no type", node.Name, id.Str())
 		}
 
 		switch node.Type {
 		case "file":
 			if node.Content == nil {
-				debug.Log("restic.fsckTree", "file node %q of tree %v has no content: %v", node.Name, blob.ID, node)
-				return fmt.Errorf("file node %q of tree %v has no content: %v", node.Name, blob.ID, node)
+				debug.Log("restic.fsckTree", "file node %q of tree %v has no content: %v", node.Name, id, node)
+				return fmt.Errorf("file node %q of tree %v has no content: %v", node.Name, id, node)
 			}
 
 			if node.Content == nil && node.Error == "" {
-				debug.Log("restic.fsckTree", "file node %q of tree %v has no content", node.Name, blob.ID)
-				return fmt.Errorf("file node %q of tree %v has no content", node.Name, blob.ID)
+				debug.Log("restic.fsckTree", "file node %q of tree %v has no content", node.Name, id)
+				return fmt.Errorf("file node %q of tree %v has no content", node.Name, id)
 			}
 
 			// record ids
@@ -119,32 +121,25 @@ func fsckTree(opts CmdFsck, s *server.Server, blob server.Blob) error {
 				seenIDs.Insert(id)
 			}
 
-			debug.Log("restic.fsckTree", "check file %v (%v)", node.Name, blob.ID.Str())
-			bytes, err := fsckFile(opts, s, tree.Map, node.Content)
+			debug.Log("restic.fsckTree", "check file %v (%v)", node.Name, id.Str())
+			bytes, err := fsckFile(opts, s, node.Content)
 			if err != nil {
 				return err
 			}
 
 			if bytes != node.Size {
-				debug.Log("restic.fsckTree", "file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, blob, node.Size, bytes)
-				return fmt.Errorf("file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, blob, node.Size, bytes)
+				debug.Log("restic.fsckTree", "file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, id, node.Size, bytes)
+				return fmt.Errorf("file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, id, node.Size, bytes)
 			}
 		case "dir":
 			if node.Subtree == nil {
-				return fmt.Errorf("dir node %q of tree %v (storage id %v) has no subtree", node.Name, blob.ID, blob.Storage)
-			}
-
-			// lookup blob
-			subtreeBlob, err := tree.Map.FindID(node.Subtree)
-			if err != nil {
-				firstErr = err
-				fmt.Fprintf(os.Stderr, "%v\n", err)
+				return fmt.Errorf("dir node %q of tree %v has no subtree", node.Name, id)
 			}
 
 			// record id
 			seenIDs.Insert(node.Subtree)
 
-			err = fsckTree(opts, s, subtreeBlob)
+			err = fsckTree(opts, s, node.Subtree)
 			if err != nil {
 				firstErr = err
 				fmt.Fprintf(os.Stderr, "%v\n", err)
@@ -153,11 +148,11 @@ func fsckTree(opts CmdFsck, s *server.Server, blob server.Blob) error {
 	}
 
 	// check map for unused ids
-	for _, id := range tree.Map.IDs() {
-		if seenIDs.Find(id) != nil {
-			return fmt.Errorf("tree %v: map contains unused ID %v", blob.ID, id)
-		}
-	}
+	// for _, id := range tree.Map.IDs() {
+	// 	if seenIDs.Find(id) != nil {
+	// 		return fmt.Errorf("tree %v: map contains unused ID %v", id, id)
+	// 	}
+	// }
 
 	return firstErr
 }
@@ -170,10 +165,6 @@ func fsckSnapshot(opts CmdFsck, s *server.Server, id backend.ID) error {
 		return fmt.Errorf("loading snapshot %v failed: %v", id, err)
 	}
 
-	if !sn.Tree.Valid() {
-		return fmt.Errorf("snapshot %s has invalid tree %v", sn.ID(), sn.Tree)
-	}
-
 	err = fsckTree(opts, s, sn.Tree)
 	if err != nil {
 		debug.Log("restic.fsck", "  checking tree %v for snapshot %v\n", sn.Tree, id)
@@ -201,6 +192,11 @@ func (cmd CmdFsck) Execute(args []string) error {
 		return err
 	}
 
+	err = s.LoadIndex()
+	if err != nil {
+		return err
+	}
+
 	if cmd.Snapshot != "" {
 		name, err := s.FindSnapshot(cmd.Snapshot)
 		if err != nil {

cmd/restic/cmd_list.go

@@ -20,7 +20,7 @@ func init() {
 }
 
 func (cmd CmdList) Usage() string {
-	return "[data|trees|snapshots|keys|locks]"
+	return "[blobs|packs|index|snapshots|keys|locks]"
 }
 
 func (cmd CmdList) Execute(args []string) error {
@@ -35,10 +35,21 @@ func (cmd CmdList) Execute(args []string) error {
 
 	var t backend.Type
 	switch args[0] {
-	case "data":
+	case "blobs":
+		err = s.LoadIndex()
+		if err != nil {
+			return err
+		}
+
+		for blob := range s.Index().Each(nil) {
+			fmt.Println(blob.ID)
+		}
+
+		return nil
+	case "packs":
 		t = backend.Data
-	case "trees":
-		t = backend.Tree
+	case "index":
+		t = backend.Index
 	case "snapshots":
 		t = backend.Snapshot
 	case "keys":

cmd/restic/cmd_ls.go

@@ -38,8 +38,8 @@ func printNode(prefix string, n *restic.Node) string {
 	}
 }
 
-func printTree(prefix string, s *server.Server, blob server.Blob) error {
-	tree, err := restic.LoadTree(s, blob)
+func printTree(prefix string, s *server.Server, id backend.ID) error {
+	tree, err := restic.LoadTree(s, id)
 	if err != nil {
 		return err
 	}
@@ -48,12 +48,7 @@ func printTree(prefix string, s *server.Server, blob server.Blob) error {
 		fmt.Println(printNode(prefix, entry))
 
 		if entry.Type == "dir" && entry.Subtree != nil {
-			b, err := tree.Map.FindID(entry.Subtree)
-			if err != nil {
-				return err
-			}
-
-			err = printTree(filepath.Join(prefix, entry.Name), s, b)
+			err = printTree(filepath.Join(prefix, entry.Name), s, id)
 			if err != nil {
 				return err
 			}

cmd/restic/cmd_restore.go

@@ -35,6 +35,11 @@ func (cmd CmdRestore) Execute(args []string) error {
 		return err
 	}
 
+	err = s.LoadIndex()
+	if err != nil {
+		return err
+	}
+
 	name, err := backend.FindSnapshot(s, args[0])
 	if err != nil {
 		errx(1, "invalid id %q: %v", args[0], err)