index: support index preallocation

internal/repository/index/index.go

@@ -132,6 +132,15 @@ var Oversized = func(idx *Index) bool {
 	return blobs >= indexMaxBlobs+pack.MaxHeaderEntries
 }
 
+// Preallocate preallocates space for the given blob type.
+// This is used to avoid reallocations when adding a large number of blobs to the index.
+func (idx *Index) Preallocate(t restic.BlobType, numEntries int) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	idx.byType[t].preallocate(numEntries)
+}
+
 // StorePack remembers the ids of all blobs of a given pack
 // in the index
 func (idx *Index) StorePack(id restic.ID, blobs []restic.Blob) {

internal/repository/index/index_test.go

@@ -427,6 +427,8 @@ func NewRandomTestID(rng *rand.Rand) restic.ID {
 
 func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupBh restic.BlobHandle) {
 	idx = index.NewIndex()
+	// the expectation is slightly above 8 blobs per pack, so preallocate 9 to be safe
+	idx.Preallocate(restic.DataBlob, packfiles*9)
 
 	// create index with given number of pack files
 	for i := 0; i < packfiles; i++ {

internal/repository/index/indexmap.go

@@ -37,19 +37,14 @@ type indexMap struct {
 }
 
 const (
-	growthFactor = 2 // Must be a power of 2.
-	maxLoad      = 4 // Max. number of entries per bucket.
+	maxLoad = 4 // Max. number of entries per bucket.
 )
 
 // add inserts an indexEntry for the given arguments into the map,
 // using id as the key.
 func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompressedLength uint32) {
-	switch {
-	case m.numentries == 0: // Lazy initialization.
-		m.init()
-	case m.numentries >= maxLoad*uint(len(m.buckets)):
-		m.grow()
-	}
+	// Make sure there is enough space for the new entry.
+	m.preallocate(int(m.numentries) + 1)
 
 	h := m.hash(id)
 	e, idx := m.newEntry()
@@ -144,8 +139,24 @@ func (m *indexMap) firstIndex(id restic.ID) int {
 	return idx
 }
 
-func (m *indexMap) grow() {
-	m.buckets = make([]uint, growthFactor*len(m.buckets))
+func (m *indexMap) preallocate(numEntries int) {
+	if numEntries == 0 {
+		return
+	}
+	if len(m.buckets) == 0 {
+		m.init() // Perform lazy initialization.
+	}
+
+	// new size must be a power of two
+	newSize := len(m.buckets)
+	for newSize < (numEntries+maxLoad-1)/maxLoad {
+		newSize *= 2
+	}
+	if newSize == len(m.buckets) {
+		return
+	}
+
+	m.buckets = make([]uint, newSize)
 
 	blockCount := m.blockList.Size()
 	for i := uint(1); i < blockCount; i++ {

internal/repository/index/master_index.go

@@ -243,6 +243,20 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
 	mi.idxMutex.Lock()
 	defer mi.idxMutex.Unlock()
 
+	if len(mi.idx) == 0 {
+		return nil
+	}
+
+	// preallocate space for all blob types
+	for typ := range restic.NumBlobTypes {
+		size := 0
+		for _, idx := range mi.idx {
+			size += int(idx.Len(typ))
+		}
+
+		mi.idx[0].Preallocate(typ, size)
+	}
+
 	// The first index is always final and the one to merge into
 	newIdx := mi.idx[:1]
 	for i := 1; i < len(mi.idx); i++ {