Merge pull request #21937 from brainexe/reduce-index-memory-packindex-uint32

Reduce in-memory index entry size by storing packIndex as uint32

changelog/unreleased/pull-21937

@@ -0,0 +1,8 @@
+Enhancement: Reduce memory usage of the in-memory index
+
+Restic keeps an index of the repository contents in memory during most
+operations. This index now requires roughly 12% less memory, which is
+especially noticeable for large repositories with many snapshots, where
+a lot of index data has to be loaded.
+
+https://github.com/restic/restic/pull/21937

internal/repository/index/index.go

@@ -31,6 +31,9 @@ import (
 // bytes each, plus 8/4 = 2 bytes of unused pointers on average, not counting
 // malloc and header struct overhead and ignoring duplicates (those are only
 // present in edge cases and are also removed by prune runs).
+// (The 56-byte entry is: id 32 + next 8 + packIndex 4 + offset 4 + length 4 +
+// uncompressedLength 4. The `next` word also carries a bloom filter in its
+// upper bits, see indexmap.go.)
 //
 // In the index entries, we need to reference the packID. As one pack may
 // contain many blobs the packIDs are saved in a separate array and only the index
@@ -68,12 +71,17 @@ func NewIndex() *Index {
 
 // addToPacks saves the given pack ID and return the index.
 // This procedere allows to use pack IDs which can be easily garbage collected after.
-func (idx *Index) addToPacks(id restic.ID) int {
+func (idx *Index) addToPacks(id restic.ID) uint32 {
 	idx.packs = append(idx.packs, id)
-	return len(idx.packs) - 1
+	// packIndex is stored as a uint32 in each indexEntry; guard against
+	// an (unrealistic) overflow so the cast below can never wrap.
+	if uint64(len(idx.packs)) > math.MaxUint32 {
+		panic("repository index pack count overflow")
+	}
+	return uint32(len(idx.packs) - 1)
 }
 
-func (idx *Index) store(packIndex int, blob pack.Blob) {
+func (idx *Index) store(packIndex uint32, blob pack.Blob) {
 	// assert that offset and length fit into uint32!
 	if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 {
 		panic("offset or length does not fit in uint32. You have packs > 4GB!")
@@ -478,6 +486,11 @@ func (idx *Index) merge(idx2 *Index) error {
 	packlen := len(idx.packs)
 	// first append packs as they might be accessed when looking for duplicates below
 	idx.packs = append(idx.packs, idx2.packs...)
+	// packIndex is stored as a uint32 in each indexEntry; make sure the
+	// merged packs list stays within that limit.
+	if uint64(len(idx.packs)) > math.MaxUint32 {
+		return errors.New("index merge: too many packs")
+	}
 
 	// copy all index entries of idx2 to idx
 	for typ := range idx2.byType {
@@ -500,7 +513,7 @@ func (idx *Index) merge(idx2 *Index) error {
 		for e2 := range m2.values() {
 			if !hasIdenticalEntry(e2) {
 				// packIndex needs to be changed as idx2.pack was appended to idx.pack, see above
-				m.add(e2.id, e2.packIndex+packlen, e2.offset, e2.length, e2.uncompressedLength)
+				m.add(e2.id, e2.packIndex+uint32(packlen), e2.offset, e2.length, e2.uncompressedLength)
 			}
 		}
 	}

internal/repository/index/indexmap.go

@@ -42,7 +42,7 @@ const (
 
 // add inserts an indexEntry for the given arguments into the map,
 // using id as the key.
-func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompressedLength uint32) {
+func (m *indexMap) add(id restic.ID, packIdx uint32, offset, length uint32, uncompressedLength uint32) {
 	// Make sure there is enough space for the new entry.
 	m.preallocate(int(m.numentries) + 1)
 
@@ -246,7 +246,7 @@ func bloomInsertID(idx uint, nextIdx uint, id restic.ID) uint {
 type indexEntry struct {
 	id                 restic.ID
 	next               uint
-	packIndex          int // Position in containing Index's packs field.
+	packIndex          uint32 // Position in containing Index's packs field.
 	offset             uint32
 	length             uint32
 	uncompressedLength uint32

internal/repository/index/indexmap_test.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"math"
 	"math/rand"
 	"testing"
 	"time"
@@ -44,14 +45,14 @@ func TestIndexMapForeach(t *testing.T) {
 	for i := 0; i < N; i++ {
 		var id restic.ID
 		id[0] = byte(i)
-		m.add(id, i, uint32(i), uint32(i), uint32(i/2))
+		m.add(id, uint32(i), uint32(i), uint32(i), uint32(i/2))
 	}
 
 	seen := make(map[int]struct{})
 	for e := range m.values() {
 		i := int(e.id[0])
 		rtest.Assert(t, i < N, "unknown id %v in indexMap", e.id)
-		rtest.Equals(t, i, e.packIndex)
+		rtest.Equals(t, uint32(i), e.packIndex)
 		rtest.Equals(t, i, int(e.length))
 		rtest.Equals(t, i, int(e.offset))
 		rtest.Equals(t, i/2, int(e.uncompressedLength))
@@ -90,13 +91,13 @@ func TestIndexMapForeachWithID(t *testing.T) {
 
 	// Test insertion and retrieval of duplicates.
 	for i := 0; i < ndups; i++ {
-		m.add(id, i, 0, 0, 0)
+		m.add(id, uint32(i), 0, 0, 0)
 	}
 
 	for i := 0; i < 100; i++ {
 		var otherid restic.ID
 		r.Read(otherid[:])
-		m.add(otherid, -1, 0, 0, 0)
+		m.add(otherid, math.MaxUint32, 0, 0, 0)
 	}
 
 	n = 0