prune: Use a single CountedBlobSet to track blobs

The set covers necessary, existing and duplicate blobs. This removes the
duplicate sets used to track whether all necessary blobs also exist.
This reduces the memory usage of prune by about 20-30%.
This commit is contained in:
Michael Eischer committed 2022-10-22 18:45:12 +02:00
1 parent b21241ec1c
commit c4fc5c97f9
4 files changed
+90 -51

No files matched your search

+4
View File
@@ -31,6 +31,10 @@ func (s BlobSet) Delete(h BlobHandle) {
delete(s, h)
}
func (s BlobSet) Len() int {
return len(s)
}
// Equals returns true iff s equals other.
func (s BlobSet) Equals(other BlobSet) bool {
if len(s) != len(other) {
+6 -1
View File
@@ -15,9 +15,14 @@ type Loader interface {
Connections() uint
}
type findBlobSet interface {
Has(bh BlobHandle) bool
Insert(bh BlobHandle)
}
// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data
// blobs) to the set blobs. Already seen tree blobs will not be visited again.
func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs BlobSet, p *progress.Counter) error {
func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs findBlobSet, p *progress.Counter) error {
var lock sync.Mutex
wg, ctx := errgroup.WithContext(ctx)