From e428de5f8416a1256bd25474b3c3ef55944122ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20D=C3=B6tsch?= Date: Wed, 8 Jul 2026 06:01:15 +0000 Subject: [PATCH] Reduce in-memory index entry size by storing packIndex as uint32 Each blob in a repository requires one index entry in memory, which is one of the largest contributors to restic's memory usage. The indexEntry.packIndex field, an index into the per-Index packs slice, was stored as a machine-word int (8 bytes on amd64) although the value is always a small array index. Store it as a uint32 instead. This shrinks indexEntry from 64 to 56 bytes on 64-bit platforms (56 % 8 == 0, so no padding is added), reducing index memory by roughly 12% -- about 100 MB for a repository with ~13 million blobs, as confirmed by profiling. uint32 allows 4.29 billion packs (>30 billion blobs at the assumed 8 blobs/pack), far beyond any real repository; addToPacks guards against overflow defensively. The on-disk repository format is unaffected. --- changelog/unreleased/pull-XXXXX | 14 ++++++++++++++ internal/repository/index/index.go | 13 +++++++++++-- internal/repository/index/indexmap.go | 4 ++-- internal/repository/index/indexmap_test.go | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/pull-XXXXX diff --git a/changelog/unreleased/pull-XXXXX b/changelog/unreleased/pull-XXXXX new file mode 100644 index 000000000..72d3c27d0 --- /dev/null +++ b/changelog/unreleased/pull-XXXXX @@ -0,0 +1,14 @@ +Enhancement: Reduce memory usage of the in-memory index + +For most operations restic keeps one index entry per blob of a repository in +memory, which is one of the largest contributors to its memory usage. Each +entry referenced its containing pack using a machine-word-sized integer, +even though only a small array index is needed. + +The pack reference is now stored as a 32-bit integer. This shrinks each index +entry from 64 to 56 bytes on 64-bit platforms, reducing the memory required +for the index by roughly 12% (about 100 MB for a repository with ~13 million +blobs). Repositories on disk are unaffected; only the in-memory representation +changed. + +https://github.com/restic/restic/pull/XXXXX diff --git a/internal/repository/index/index.go b/internal/repository/index/index.go index bbdad92d5..289032202 100644 --- a/internal/repository/index/index.go +++ b/internal/repository/index/index.go @@ -31,6 +31,9 @@ import ( // bytes each, plus 8/4 = 2 bytes of unused pointers on average, not counting // malloc and header struct overhead and ignoring duplicates (those are only // present in edge cases and are also removed by prune runs). +// (The 56-byte entry is: id 32 + next 8 + packIndex 4 + offset 4 + length 4 + +// uncompressedLength 4. The `next` word also carries a bloom filter in its +// upper bits, see indexmap.go.) // // In the index entries, we need to reference the packID. As one pack may // contain many blobs the packIDs are saved in a separate array and only the index @@ -70,7 +73,13 @@ func NewIndex() *Index { // This procedere allows to use pack IDs which can be easily garbage collected after. func (idx *Index) addToPacks(id restic.ID) int { idx.packs = append(idx.packs, id) - return len(idx.packs) - 1 + packIdx := len(idx.packs) - 1 + // packIdx is stored as a uint32 in each indexEntry; guard against an + // (unrealistic) overflow so the cast in indexMap.add can never wrap. + if uint64(packIdx) > math.MaxUint32 { + panic("repository index pack count overflow") + } + return packIdx } func (idx *Index) store(packIndex int, blob pack.Blob) { @@ -500,7 +509,7 @@ func (idx *Index) merge(idx2 *Index) error { for e2 := range m2.values() { if !hasIdenticalEntry(e2) { // packIndex needs to be changed as idx2.pack was appended to idx.pack, see above - m.add(e2.id, e2.packIndex+packlen, e2.offset, e2.length, e2.uncompressedLength) + m.add(e2.id, int(e2.packIndex)+packlen, e2.offset, e2.length, e2.uncompressedLength) } } } diff --git a/internal/repository/index/indexmap.go b/internal/repository/index/indexmap.go index 159abe7fe..dbee0ad30 100644 --- a/internal/repository/index/indexmap.go +++ b/internal/repository/index/indexmap.go @@ -50,7 +50,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr e, idx := m.newEntry() e.id = id e.next = m.buckets[h] // Prepend to existing chain. - e.packIndex = packIdx + e.packIndex = uint32(packIdx) e.offset = offset e.length = length e.uncompressedLength = uncompressedLength @@ -246,7 +246,7 @@ func bloomInsertID(idx uint, nextIdx uint, id restic.ID) uint { type indexEntry struct { id restic.ID next uint - packIndex int // Position in containing Index's packs field. + packIndex uint32 // Position in containing Index's packs field. offset uint32 length uint32 uncompressedLength uint32 diff --git a/internal/repository/index/indexmap_test.go b/internal/repository/index/indexmap_test.go index 704f2cbf0..18089191c 100644 --- a/internal/repository/index/indexmap_test.go +++ b/internal/repository/index/indexmap_test.go @@ -51,7 +51,7 @@ func TestIndexMapForeach(t *testing.T) { for e := range m.values() { i := int(e.id[0]) rtest.Assert(t, i < N, "unknown id %v in indexMap", e.id) - rtest.Equals(t, i, e.packIndex) + rtest.Equals(t, uint32(i), e.packIndex) rtest.Equals(t, i, int(e.length)) rtest.Equals(t, i, int(e.offset)) rtest.Equals(t, i/2, int(e.uncompressedLength))