From 4c0dc9e202d9b1308d0c0fc308c45426c5cd767e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 18 Feb 2026 20:07:12 +0100 Subject: [PATCH] index: support incremental index loading Do not require a full index reload if only a few additional index files have been added. This can drastically speed up loading the index in the mount command. --- internal/repository/index/master_index.go | 53 +++++++++++++++++-- .../repository/index/master_index_test.go | 46 ++++++++++++++++ internal/repository/repository.go | 3 -- internal/ui/progress/counter.go | 4 +- 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index f410ebf61..bc0198882 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -22,7 +22,7 @@ type MasterIndex struct { // NewMasterIndex creates a new master index. func NewMasterIndex() *MasterIndex { - mi := &MasterIndex{pendingBlobs: make(map[restic.BlobHandle]uint)} + mi := &MasterIndex{} mi.clear() return mi } @@ -31,6 +31,11 @@ func (mi *MasterIndex) clear() { // Always add an empty final index, such that MergeFinalIndexes can merge into this. mi.idx = []*Index{NewIndex()} mi.idx[0].Finalize() + mi.clearPendingBlobs() +} + +func (mi *MasterIndex) clearPendingBlobs() { + mi.pendingBlobs = make(map[restic.BlobHandle]uint) } // Lookup queries all known Indexes for the ID and returns all matches. @@ -265,10 +270,17 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, if err != nil { return err } - + loadedIDs, err := mi.prepareIncrementalLoad(ctx, indexList) + if err != nil { + return err + } if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error { + err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error { + if loadedIDs.Has(id) { + // skip already loaded indexes + return nil + } numIndexFiles++ return nil }) @@ -280,6 +292,10 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, } err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error { + if loadedIDs.Has(id) { + // skip already loaded indexes + return nil + } if p != nil { p.Add(1) } @@ -304,6 +320,37 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, return mi.MergeFinalIndexes() } +func (mi *MasterIndex) prepareIncrementalLoad(ctx context.Context, indexList restic.Lister) (restic.IDSet, error) { + mi.idxMutex.Lock() + // support incremental loading, while also ensuring that the result is identical to the result of a full load into a new MasterIndex + mi.clearPendingBlobs() + defer mi.idxMutex.Unlock() + + // the first index is always final so this can't actually fail + loadedIDList, err := mi.idx[0].IDs() + if err != nil { + panic("internal error - failed to get index IDs") + } + loadedIDs := restic.NewIDSet(loadedIDList...) + + indexFiles := restic.NewIDSet() + err = indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error { + indexFiles.Insert(id) + return nil + }) + if err != nil { + return nil, err + } + + if len(loadedIDs.Sub(indexFiles)) > 0 { + // indexes can only be removed by prune, which shouldn't happen concurrently, but behave correctly anyways + mi.clear() + loadedIDs = nil + } + + return loadedIDs, nil +} + type MasterIndexRewriteOpts struct { SaveProgress *progress.Counter DeleteProgress func() *progress.Counter diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index 39837f53e..5cae25873 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/data" @@ -15,6 +16,7 @@ import ( "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" ) func TestMasterIndex(t *testing.T) { @@ -520,6 +522,50 @@ func testIndexSavePartial(t *testing.T, version uint) { checker.TestCheckRepo(t, repo) } +func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} { + p := progress.NewCounter(0, 0, nil) + rtest.OK(t, master.Load(context.TODO(), repo, p, nil)) + v, max := p.Get() + rtest.Equals(t, uint64(indexCount), v) + rtest.Equals(t, uint64(indexCount), max) + return collectBlobs(master) +} + +func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} { + s := make(map[restic.PackedBlob]struct{}) + for pb := range master.Values() { + s[pb] = struct{}{} + } + return s +} + +func TestMasterIndexIncrementalLoad(t *testing.T) { + repo, _ := createFilledRepo(t, 3, restic.StableRepoVersion) + + // Normal full index load + master1 := index.NewMasterIndex() + blobs1 := loadIndexAndCollectBlobs(t, repo, master1, 3) + + // Noop reload should not change the index content + blobs1NoopLoad := loadIndexAndCollectBlobs(t, repo, master1, 0) + if !cmp.Equal(blobs1, blobs1NoopLoad) { + t.Fatalf("index content mismatch after noop reload: %v", cmp.Diff(blobs1, blobs1NoopLoad)) + } + + // Add new snapshot, which also results in a new index file + data.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(4)*time.Second), depth) + + // Incremental load should only load the new index + blobs1IncrementalLoad := loadIndexAndCollectBlobs(t, repo, master1, 1) + + // Reload index from scratch and compare with incremental load + master2 := index.NewMasterIndex() + blobs2 := loadIndexAndCollectBlobs(t, repo, master2, 4) + if !cmp.Equal(blobs1IncrementalLoad, blobs2) { + t.Fatalf("index content mismatch compared to full reload: %v", cmp.Diff(blobs1IncrementalLoad, blobs2)) + } +} + func listPacks(t testing.TB, repo restic.Lister) restic.IDSet { s := restic.NewIDSet() rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, _ int64) error { diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f1704291b..c27d2b0da 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -709,9 +709,6 @@ func (r *Repository) LoadIndex(ctx context.Context, p restic.TerminalCounterFact func (r *Repository) loadIndexWithCallback(ctx context.Context, p restic.TerminalCounterFactory, cb func(id restic.ID, idx *index.Index, err error) error) error { debug.Log("Loading index") - // reset in-memory index before loading it from the repository - r.clearIndex() - var bar *progress.Counter if p != nil { bar = p.NewCounterTerminalOnly("index files loaded") diff --git a/internal/ui/progress/counter.go b/internal/ui/progress/counter.go index fbb5722e0..271c483f6 100644 --- a/internal/ui/progress/counter.go +++ b/internal/ui/progress/counter.go @@ -26,7 +26,9 @@ func NewCounter(interval time.Duration, total uint64, report Func) *Counter { c.max.Store(total) c.Updater = *NewUpdater(interval, func(runtime time.Duration, final bool) { v, maxV := c.Get() - report(v, maxV, runtime, final) + if report != nil { + report(v, maxV, runtime, final) + } }) return c }