From e428de5f8416a1256bd25474b3c3ef55944122ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20D=C3=B6tsch?= Date: Wed, 8 Jul 2026 06:01:15 +0000 Subject: [PATCH 1/2] Reduce in-memory index entry size by storing packIndex as uint32 Each blob in a repository requires one index entry in memory, which is one of the largest contributors to restic's memory usage. The indexEntry.packIndex field, an index into the per-Index packs slice, was stored as a machine-word int (8 bytes on amd64) although the value is always a small array index. Store it as a uint32 instead. This shrinks indexEntry from 64 to 56 bytes on 64-bit platforms (56 % 8 == 0, so no padding is added), reducing index memory by roughly 12% -- about 100 MB for a repository with ~13 million blobs, as confirmed by profiling. uint32 allows 4.29 billion packs (>30 billion blobs at the assumed 8 blobs/pack), far beyond any real repository; addToPacks guards against overflow defensively. The on-disk repository format is unaffected. --- changelog/unreleased/pull-XXXXX | 14 ++++++++++++++ internal/repository/index/index.go | 13 +++++++++++-- internal/repository/index/indexmap.go | 4 ++-- internal/repository/index/indexmap_test.go | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/pull-XXXXX diff --git a/changelog/unreleased/pull-XXXXX b/changelog/unreleased/pull-XXXXX new file mode 100644 index 000000000..72d3c27d0 --- /dev/null +++ b/changelog/unreleased/pull-XXXXX @@ -0,0 +1,14 @@ +Enhancement: Reduce memory usage of the in-memory index + +For most operations restic keeps one index entry per blob of a repository in +memory, which is one of the largest contributors to its memory usage. Each +entry referenced its containing pack using a machine-word-sized integer, +even though only a small array index is needed. + +The pack reference is now stored as a 32-bit integer. This shrinks each index +entry from 64 to 56 bytes on 64-bit platforms, reducing the memory required +for the index by roughly 12% (about 100 MB for a repository with ~13 million +blobs). Repositories on disk are unaffected; only the in-memory representation +changed. + +https://github.com/restic/restic/pull/XXXXX diff --git a/internal/repository/index/index.go b/internal/repository/index/index.go index bbdad92d5..289032202 100644 --- a/internal/repository/index/index.go +++ b/internal/repository/index/index.go @@ -31,6 +31,9 @@ import ( // bytes each, plus 8/4 = 2 bytes of unused pointers on average, not counting // malloc and header struct overhead and ignoring duplicates (those are only // present in edge cases and are also removed by prune runs). +// (The 56-byte entry is: id 32 + next 8 + packIndex 4 + offset 4 + length 4 + +// uncompressedLength 4. The `next` word also carries a bloom filter in its +// upper bits, see indexmap.go.) // // In the index entries, we need to reference the packID. As one pack may // contain many blobs the packIDs are saved in a separate array and only the index @@ -70,7 +73,13 @@ func NewIndex() *Index { // This procedere allows to use pack IDs which can be easily garbage collected after. func (idx *Index) addToPacks(id restic.ID) int { idx.packs = append(idx.packs, id) - return len(idx.packs) - 1 + packIdx := len(idx.packs) - 1 + // packIdx is stored as a uint32 in each indexEntry; guard against an + // (unrealistic) overflow so the cast in indexMap.add can never wrap. + if uint64(packIdx) > math.MaxUint32 { + panic("repository index pack count overflow") + } + return packIdx } func (idx *Index) store(packIndex int, blob pack.Blob) { @@ -500,7 +509,7 @@ func (idx *Index) merge(idx2 *Index) error { for e2 := range m2.values() { if !hasIdenticalEntry(e2) { // packIndex needs to be changed as idx2.pack was appended to idx.pack, see above - m.add(e2.id, e2.packIndex+packlen, e2.offset, e2.length, e2.uncompressedLength) + m.add(e2.id, int(e2.packIndex)+packlen, e2.offset, e2.length, e2.uncompressedLength) } } } diff --git a/internal/repository/index/indexmap.go b/internal/repository/index/indexmap.go index 159abe7fe..dbee0ad30 100644 --- a/internal/repository/index/indexmap.go +++ b/internal/repository/index/indexmap.go @@ -50,7 +50,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr e, idx := m.newEntry() e.id = id e.next = m.buckets[h] // Prepend to existing chain. - e.packIndex = packIdx + e.packIndex = uint32(packIdx) e.offset = offset e.length = length e.uncompressedLength = uncompressedLength @@ -246,7 +246,7 @@ func bloomInsertID(idx uint, nextIdx uint, id restic.ID) uint { type indexEntry struct { id restic.ID next uint - packIndex int // Position in containing Index's packs field. + packIndex uint32 // Position in containing Index's packs field. offset uint32 length uint32 uncompressedLength uint32 diff --git a/internal/repository/index/indexmap_test.go b/internal/repository/index/indexmap_test.go index 704f2cbf0..18089191c 100644 --- a/internal/repository/index/indexmap_test.go +++ b/internal/repository/index/indexmap_test.go @@ -51,7 +51,7 @@ func TestIndexMapForeach(t *testing.T) { for e := range m.values() { i := int(e.id[0]) rtest.Assert(t, i < N, "unknown id %v in indexMap", e.id) - rtest.Equals(t, i, e.packIndex) + rtest.Equals(t, uint32(i), e.packIndex) rtest.Equals(t, i, int(e.length)) rtest.Equals(t, i, int(e.offset)) rtest.Equals(t, i/2, int(e.uncompressedLength)) From 905ca56ee88ad654607b366b6c731fd1499b702c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20D=C3=B6tsch?= Date: Wed, 8 Jul 2026 08:49:54 +0200 Subject: [PATCH 2/2] Address review feedback on #21937: - addToPacks now returns uint32 directly and the value is passed through store() to indexMap.add unchanged, instead of casting between int and uint32. - merge() appends idx2.packs to idx.packs without going through addToPacks, so it now checks the pack count against the uint32 limit itself + error if exceeded. - optimize changelog message --- changelog/unreleased/pull-21937 | 8 ++++++++ changelog/unreleased/pull-XXXXX | 14 -------------- internal/repository/index/index.go | 20 ++++++++++++-------- internal/repository/index/indexmap.go | 4 ++-- internal/repository/index/indexmap_test.go | 7 ++++--- 5 files changed, 26 insertions(+), 27 deletions(-) create mode 100644 changelog/unreleased/pull-21937 delete mode 100644 changelog/unreleased/pull-XXXXX diff --git a/changelog/unreleased/pull-21937 b/changelog/unreleased/pull-21937 new file mode 100644 index 000000000..078b48684 --- /dev/null +++ b/changelog/unreleased/pull-21937 @@ -0,0 +1,8 @@ +Enhancement: Reduce memory usage of the in-memory index + +Restic keeps an index of the repository contents in memory during most +operations. This index now requires roughly 12% less memory, which is +especially noticeable for large repositories with many snapshots, where +a lot of index data has to be loaded. + +https://github.com/restic/restic/pull/21937 diff --git a/changelog/unreleased/pull-XXXXX b/changelog/unreleased/pull-XXXXX deleted file mode 100644 index 72d3c27d0..000000000 --- a/changelog/unreleased/pull-XXXXX +++ /dev/null @@ -1,14 +0,0 @@ -Enhancement: Reduce memory usage of the in-memory index - -For most operations restic keeps one index entry per blob of a repository in -memory, which is one of the largest contributors to its memory usage. Each -entry referenced its containing pack using a machine-word-sized integer, -even though only a small array index is needed. - -The pack reference is now stored as a 32-bit integer. This shrinks each index -entry from 64 to 56 bytes on 64-bit platforms, reducing the memory required -for the index by roughly 12% (about 100 MB for a repository with ~13 million -blobs). Repositories on disk are unaffected; only the in-memory representation -changed. - -https://github.com/restic/restic/pull/XXXXX diff --git a/internal/repository/index/index.go b/internal/repository/index/index.go index 289032202..9f05b92b9 100644 --- a/internal/repository/index/index.go +++ b/internal/repository/index/index.go @@ -71,18 +71,17 @@ func NewIndex() *Index { // addToPacks saves the given pack ID and return the index. // This procedere allows to use pack IDs which can be easily garbage collected after. -func (idx *Index) addToPacks(id restic.ID) int { +func (idx *Index) addToPacks(id restic.ID) uint32 { idx.packs = append(idx.packs, id) - packIdx := len(idx.packs) - 1 - // packIdx is stored as a uint32 in each indexEntry; guard against an - // (unrealistic) overflow so the cast in indexMap.add can never wrap. - if uint64(packIdx) > math.MaxUint32 { + // packIndex is stored as a uint32 in each indexEntry; guard against + // an (unrealistic) overflow so the cast below can never wrap. + if uint64(len(idx.packs)) > math.MaxUint32 { panic("repository index pack count overflow") } - return packIdx + return uint32(len(idx.packs) - 1) } -func (idx *Index) store(packIndex int, blob pack.Blob) { +func (idx *Index) store(packIndex uint32, blob pack.Blob) { // assert that offset and length fit into uint32! if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 { panic("offset or length does not fit in uint32. You have packs > 4GB!") @@ -487,6 +486,11 @@ func (idx *Index) merge(idx2 *Index) error { packlen := len(idx.packs) // first append packs as they might be accessed when looking for duplicates below idx.packs = append(idx.packs, idx2.packs...) + // packIndex is stored as a uint32 in each indexEntry; make sure the + // merged packs list stays within that limit. + if uint64(len(idx.packs)) > math.MaxUint32 { + return errors.New("index merge: too many packs") + } // copy all index entries of idx2 to idx for typ := range idx2.byType { @@ -509,7 +513,7 @@ func (idx *Index) merge(idx2 *Index) error { for e2 := range m2.values() { if !hasIdenticalEntry(e2) { // packIndex needs to be changed as idx2.pack was appended to idx.pack, see above - m.add(e2.id, int(e2.packIndex)+packlen, e2.offset, e2.length, e2.uncompressedLength) + m.add(e2.id, e2.packIndex+uint32(packlen), e2.offset, e2.length, e2.uncompressedLength) } } } diff --git a/internal/repository/index/indexmap.go b/internal/repository/index/indexmap.go index dbee0ad30..d38385aba 100644 --- a/internal/repository/index/indexmap.go +++ b/internal/repository/index/indexmap.go @@ -42,7 +42,7 @@ const ( // add inserts an indexEntry for the given arguments into the map, // using id as the key. -func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompressedLength uint32) { +func (m *indexMap) add(id restic.ID, packIdx uint32, offset, length uint32, uncompressedLength uint32) { // Make sure there is enough space for the new entry. m.preallocate(int(m.numentries) + 1) @@ -50,7 +50,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr e, idx := m.newEntry() e.id = id e.next = m.buckets[h] // Prepend to existing chain. - e.packIndex = uint32(packIdx) + e.packIndex = packIdx e.offset = offset e.length = length e.uncompressedLength = uncompressedLength diff --git a/internal/repository/index/indexmap_test.go b/internal/repository/index/indexmap_test.go index 18089191c..085d6acb1 100644 --- a/internal/repository/index/indexmap_test.go +++ b/internal/repository/index/indexmap_test.go @@ -1,6 +1,7 @@ package index import ( + "math" "math/rand" "testing" "time" @@ -44,7 +45,7 @@ func TestIndexMapForeach(t *testing.T) { for i := 0; i < N; i++ { var id restic.ID id[0] = byte(i) - m.add(id, i, uint32(i), uint32(i), uint32(i/2)) + m.add(id, uint32(i), uint32(i), uint32(i), uint32(i/2)) } seen := make(map[int]struct{}) @@ -90,13 +91,13 @@ func TestIndexMapForeachWithID(t *testing.T) { // Test insertion and retrieval of duplicates. for i := 0; i < ndups; i++ { - m.add(id, i, 0, 0, 0) + m.add(id, uint32(i), 0, 0, 0) } for i := 0; i < 100; i++ { var otherid restic.ID r.Read(otherid[:]) - m.add(otherid, -1, 0, 0, 0) + m.add(otherid, math.MaxUint32, 0, 0, 0) } n = 0