Reduce in-memory index entry size by storing packIndex as uint32

Each blob in a repository requires one index entry in memory, which is one
of the largest contributors to restic's memory usage. The indexEntry.packIndex
field, an index into the per-Index packs slice, was stored as a machine-word
int (8 bytes on amd64) although the value is always a small array index.

Store it as a uint32 instead. This shrinks indexEntry from 64 to 56 bytes on
64-bit platforms (56 % 8 == 0, so no padding is added), reducing index memory
by roughly 12% -- about 100 MB for a repository with ~13 million blobs, as
confirmed by profiling. uint32 allows 4.29 billion packs (>30 billion blobs at
the assumed 8 blobs/pack), far beyond any real repository; addToPacks guards
against overflow defensively. The on-disk repository format is unaffected.

changelog/unreleased/pull-XXXXX

@@ -0,0 +1,14 @@
+Enhancement: Reduce memory usage of the in-memory index
+
+For most operations restic keeps one index entry per blob of a repository in
+memory, which is one of the largest contributors to its memory usage. Each
+entry referenced its containing pack using a machine-word-sized integer,
+even though only a small array index is needed.
+
+The pack reference is now stored as a 32-bit integer. This shrinks each index
+entry from 64 to 56 bytes on 64-bit platforms, reducing the memory required
+for the index by roughly 12% (about 100 MB for a repository with ~13 million
+blobs). Repositories on disk are unaffected; only the in-memory representation
+changed.
+
+https://github.com/restic/restic/pull/XXXXX

internal/repository/index/index.go

@@ -31,6 +31,9 @@ import (
 // bytes each, plus 8/4 = 2 bytes of unused pointers on average, not counting
 // malloc and header struct overhead and ignoring duplicates (those are only
 // present in edge cases and are also removed by prune runs).
+// (The 56-byte entry is: id 32 + next 8 + packIndex 4 + offset 4 + length 4 +
+// uncompressedLength 4. The `next` word also carries a bloom filter in its
+// upper bits, see indexmap.go.)
 //
 // In the index entries, we need to reference the packID. As one pack may
 // contain many blobs the packIDs are saved in a separate array and only the index
@@ -70,7 +73,13 @@ func NewIndex() *Index {
 // This procedere allows to use pack IDs which can be easily garbage collected after.
 func (idx *Index) addToPacks(id restic.ID) int {
 	idx.packs = append(idx.packs, id)
-	return len(idx.packs) - 1
+	packIdx := len(idx.packs) - 1
+	// packIdx is stored as a uint32 in each indexEntry; guard against an
+	// (unrealistic) overflow so the cast in indexMap.add can never wrap.
+	if uint64(packIdx) > math.MaxUint32 {
+		panic("repository index pack count overflow")
+	}
+	return packIdx
 }
 
 func (idx *Index) store(packIndex int, blob pack.Blob) {
@@ -500,7 +509,7 @@ func (idx *Index) merge(idx2 *Index) error {
 		for e2 := range m2.values() {
 			if !hasIdenticalEntry(e2) {
 				// packIndex needs to be changed as idx2.pack was appended to idx.pack, see above
-				m.add(e2.id, e2.packIndex+packlen, e2.offset, e2.length, e2.uncompressedLength)
+				m.add(e2.id, int(e2.packIndex)+packlen, e2.offset, e2.length, e2.uncompressedLength)
 			}
 		}
 	}

internal/repository/index/indexmap.go

@@ -50,7 +50,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr
 	e, idx := m.newEntry()
 	e.id = id
 	e.next = m.buckets[h] // Prepend to existing chain.
-	e.packIndex = packIdx
+	e.packIndex = uint32(packIdx)
 	e.offset = offset
 	e.length = length
 	e.uncompressedLength = uncompressedLength
@@ -246,7 +246,7 @@ func bloomInsertID(idx uint, nextIdx uint, id restic.ID) uint {
 type indexEntry struct {
 	id                 restic.ID
 	next               uint
-	packIndex          int // Position in containing Index's packs field.
+	packIndex          uint32 // Position in containing Index's packs field.
 	offset             uint32
 	length             uint32
 	uncompressedLength uint32

internal/repository/index/indexmap_test.go

@@ -51,7 +51,7 @@ func TestIndexMapForeach(t *testing.T) {
 	for e := range m.values() {
 		i := int(e.id[0])
 		rtest.Assert(t, i < N, "unknown id %v in indexMap", e.id)
-		rtest.Equals(t, i, e.packIndex)
+		rtest.Equals(t, uint32(i), e.packIndex)
 		rtest.Equals(t, i, int(e.length))
 		rtest.Equals(t, i, int(e.offset))
 		rtest.Equals(t, i/2, int(e.uncompressedLength))