index: support incremental index loading

Do not require a full index reload if only a few additional index files
have been added. This can drastically speed up loading the index in the
mount command.

internal/repository/index/master_index.go

@@ -22,7 +22,7 @@ type MasterIndex struct {
 
 // NewMasterIndex creates a new master index.
 func NewMasterIndex() *MasterIndex {
-	mi := &MasterIndex{pendingBlobs: make(map[restic.BlobHandle]uint)}
+	mi := &MasterIndex{}
 	mi.clear()
 	return mi
 }
@@ -31,6 +31,11 @@ func (mi *MasterIndex) clear() {
 	// Always add an empty final index, such that MergeFinalIndexes can merge into this.
 	mi.idx = []*Index{NewIndex()}
 	mi.idx[0].Finalize()
+	mi.clearPendingBlobs()
+}
+
+func (mi *MasterIndex) clearPendingBlobs() {
+	mi.pendingBlobs = make(map[restic.BlobHandle]uint)
 }
 
 // Lookup queries all known Indexes for the ID and returns all matches.
@@ -265,10 +270,17 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
 	if err != nil {
 		return err
 	}
-
+	loadedIDs, err := mi.prepareIncrementalLoad(ctx, indexList)
+	if err != nil {
+		return err
+	}
 	if p != nil {
 		var numIndexFiles uint64
-		err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error {
+		err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
+			if loadedIDs.Has(id) {
+				// skip already loaded indexes
+				return nil
+			}
 			numIndexFiles++
 			return nil
 		})
@@ -280,6 +292,10 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
 	}
 
 	err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {
+		if loadedIDs.Has(id) {
+			// skip already loaded indexes
+			return nil
+		}
 		if p != nil {
 			p.Add(1)
 		}
@@ -304,6 +320,37 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
 	return mi.MergeFinalIndexes()
 }
 
+func (mi *MasterIndex) prepareIncrementalLoad(ctx context.Context, indexList restic.Lister) (restic.IDSet, error) {
+	mi.idxMutex.Lock()
+	// support incremental loading, while also ensuring that the result is identical to the result of a full load into a new MasterIndex
+	mi.clearPendingBlobs()
+	defer mi.idxMutex.Unlock()
+
+	// the first index is always final so this can't actually fail
+	loadedIDList, err := mi.idx[0].IDs()
+	if err != nil {
+		panic("internal error - failed to get index IDs")
+	}
+	loadedIDs := restic.NewIDSet(loadedIDList...)
+
+	indexFiles := restic.NewIDSet()
+	err = indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
+		indexFiles.Insert(id)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(loadedIDs.Sub(indexFiles)) > 0 {
+		// indexes can only be removed by prune, which shouldn't happen concurrently, but behave correctly anyways
+		mi.clear()
+		loadedIDs = nil
+	}
+
+	return loadedIDs, nil
+}
+
 type MasterIndexRewriteOpts struct {
 	SaveProgress   *progress.Counter
 	DeleteProgress func() *progress.Counter

internal/repository/index/master_index_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/data"
@@ -15,6 +16,7 @@ import (
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/progress"
 )
 
 func TestMasterIndex(t *testing.T) {
@@ -520,6 +522,50 @@ func testIndexSavePartial(t *testing.T, version uint) {
 	checker.TestCheckRepo(t, repo)
 }
 
+func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} {
+	p := progress.NewCounter(0, 0, nil)
+	rtest.OK(t, master.Load(context.TODO(), repo, p, nil))
+	v, max := p.Get()
+	rtest.Equals(t, uint64(indexCount), v)
+	rtest.Equals(t, uint64(indexCount), max)
+	return collectBlobs(master)
+}
+
+func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} {
+	s := make(map[restic.PackedBlob]struct{})
+	for pb := range master.Values() {
+		s[pb] = struct{}{}
+	}
+	return s
+}
+
+func TestMasterIndexIncrementalLoad(t *testing.T) {
+	repo, _ := createFilledRepo(t, 3, restic.StableRepoVersion)
+
+	// Normal full index load
+	master1 := index.NewMasterIndex()
+	blobs1 := loadIndexAndCollectBlobs(t, repo, master1, 3)
+
+	// Noop reload should not change the index content
+	blobs1NoopLoad := loadIndexAndCollectBlobs(t, repo, master1, 0)
+	if !cmp.Equal(blobs1, blobs1NoopLoad) {
+		t.Fatalf("index content mismatch after noop reload: %v", cmp.Diff(blobs1, blobs1NoopLoad))
+	}
+
+	// Add new snapshot, which also results in a new index file
+	data.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(4)*time.Second), depth)
+
+	// Incremental load should only load the new index
+	blobs1IncrementalLoad := loadIndexAndCollectBlobs(t, repo, master1, 1)
+
+	// Reload index from scratch and compare with incremental load
+	master2 := index.NewMasterIndex()
+	blobs2 := loadIndexAndCollectBlobs(t, repo, master2, 4)
+	if !cmp.Equal(blobs1IncrementalLoad, blobs2) {
+		t.Fatalf("index content mismatch compared to full reload: %v", cmp.Diff(blobs1IncrementalLoad, blobs2))
+	}
+}
+
 func listPacks(t testing.TB, repo restic.Lister) restic.IDSet {
 	s := restic.NewIDSet()
 	rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, _ int64) error {

internal/repository/repository.go

@@ -709,9 +709,6 @@ func (r *Repository) LoadIndex(ctx context.Context, p restic.TerminalCounterFact
 func (r *Repository) loadIndexWithCallback(ctx context.Context, p restic.TerminalCounterFactory, cb func(id restic.ID, idx *index.Index, err error) error) error {
 	debug.Log("Loading index")
 
-	// reset in-memory index before loading it from the repository
-	r.clearIndex()
-
 	var bar *progress.Counter
 	if p != nil {
 		bar = p.NewCounterTerminalOnly("index files loaded")

internal/ui/progress/counter.go

@@ -26,7 +26,9 @@ func NewCounter(interval time.Duration, total uint64, report Func) *Counter {
 	c.max.Store(total)
 	c.Updater = *NewUpdater(interval, func(runtime time.Duration, final bool) {
 		v, maxV := c.Get()
-		report(v, maxV, runtime, final)
+		if report != nil {
+			report(v, maxV, runtime, final)
+		}
 	})
 	return c
 }