diff --git a/.golangci.yml b/.golangci.yml
index 2365a5b04..0482253b0 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -32,7 +32,6 @@ linters:
         backend-imports:
           files:
             - "**/internal/backend/**"
-            - "!**/internal/backend/cache/**"
             - "!**/internal/backend/test/**"
             - "!**/*_test.go"
           deny:
diff --git a/internal/backend/cache/backend.go b/internal/backend/cache/backend.go
index 8a561c411..34dd07903 100644
--- a/internal/backend/cache/backend.go
+++ b/internal/backend/cache/backend.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/restic"
 )
 
 // Backend wraps a restic.Backend and adds a cache.
@@ -224,18 +223,11 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen
 		return b.Backend.List(ctx, t, fn)
 	}
 
-	// will contain the IDs of the files that are in the repository
-	ids := restic.NewIDSet()
+	ids := make(map[string]struct{})
 
 	// wrap the original function to also add the file to the ids set
 	wrapFn := func(f backend.FileInfo) error {
-		id, err := restic.ParseID(f.Name)
-		if err != nil {
-			// ignore files with invalid name
-			return nil
-		}
-
-		ids.Insert(id)
+		ids[f.Name] = struct{}{}
 
 		// execute the original function
 		return fn(f)
diff --git a/internal/backend/cache/cache.go b/internal/backend/cache/cache.go
index d1bfa47a0..22c3532ce 100644
--- a/internal/backend/cache/cache.go
+++ b/internal/backend/cache/cache.go
@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/restic"
 )
 
 // Cache manages a local cache.
@@ -43,10 +42,10 @@ func readVersion(dir string) (v uint, err error) {
 
 const cacheVersion = 1
 
-var cacheLayoutPaths = map[restic.FileType]string{
-	restic.PackFile:     "data",
-	restic.SnapshotFile: "snapshots",
-	restic.IndexFile:    "index",
+var cacheLayoutPaths = map[backend.FileType]string{
+	backend.PackFile:     "data",
+	backend.SnapshotFile: "snapshots",
+	backend.IndexFile:    "index",
 }
 
 const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"
diff --git a/internal/backend/cache/file.go b/internal/backend/cache/file.go
index 062d6ea3f..71613bfbf 100644
--- a/internal/backend/cache/file.go
+++ b/internal/backend/cache/file.go
@@ -12,7 +12,6 @@ import (
 	"github.com/restic/restic/internal/backend/util"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/restic"
 )
 
 func (c *Cache) filename(h backend.Handle) string {
@@ -171,7 +170,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
 
 // Clear removes all files of type t from the cache that are not contained in
 // the set valid.
-func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
+func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
 	debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
 	if !c.canBeCached(t) {
 		return nil
@@ -183,12 +182,12 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
 	}
 
 	for id := range list {
-		if valid.Has(id) {
+		if _, ok := valid[id]; ok {
 			continue
 		}
 
 		// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
-		if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
+		if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id})); err != nil && !errors.Is(err, os.ErrNotExist) {
 			return err
 		}
 	}
@@ -201,12 +200,12 @@ func isFile(fi os.FileInfo) bool {
 }
 
 // list returns a list of all files of type T in the cache.
-func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
+func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
 	if !c.canBeCached(t) {
 		return nil, errors.New("cannot be cached")
 	}
 
-	list := restic.NewIDSet()
+	list := make(map[string]struct{})
 	dir := filepath.Join(c.path, cacheLayoutPaths[t])
 	err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
 		if err != nil {
@@ -221,12 +220,8 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
 			return nil
 		}
 
-		id, err := restic.ParseID(filepath.Base(name))
-		if err != nil {
-			return nil
-		}
-
-		list.Insert(id)
+		id := filepath.Base(name)
+		list[id] = struct{}{}
 		return nil
 	})
 
diff --git a/internal/backend/cache/file_test.go b/internal/backend/cache/file_test.go
index 942f71f91..ace3080e0 100644
--- a/internal/backend/cache/file_test.go
+++ b/internal/backend/cache/file_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"maps"
 	"math/rand"
 	"os"
 	"runtime"
@@ -18,8 +19,8 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
-	ids := restic.NewIDSet()
+func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
+	ids := make(map[string]struct{})
 	for i := 0; i < random.Intn(15)+10; i++ {
 		buf := rtest.Random(random.Int(), 1<<19)
 		id := restic.Hash(buf)
@@ -33,13 +34,13 @@ func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType,
 		if err != nil {
 			t.Fatal(err)
 		}
-		ids.Insert(id)
+		ids[id.String()] = struct{}{}
 	}
 	return ids
 }
 
 // randomID returns a random ID from s.
-func randomID(s restic.IDSet) restic.ID {
+func randomID(s map[string]struct{}) string {
 	for id := range s {
 		return id
 	}
@@ -69,7 +70,7 @@ func load(t testing.TB, c *Cache, h backend.Handle) []byte {
 	return buf
 }
 
-func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
+func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{} {
 	list, err := c.list(tpe)
 	if err != nil {
 		t.Errorf("listing failed: %v", err)
@@ -78,7 +79,7 @@ func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
 	return list
 }
 
-func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
+func clearFiles(t testing.TB, c *Cache, tpe backend.FileType, valid map[string]struct{}) {
 	if err := c.Clear(tpe, valid); err != nil {
 		t.Error(err)
 	}
@@ -102,34 +103,34 @@ func TestFiles(t *testing.T) {
 			ids := generateRandomFiles(t, random, tpe, c)
 			id := randomID(ids)
 
-			h := backend.Handle{Type: tpe, Name: id.String()}
+			h := backend.Handle{Type: tpe, Name: id}
 			id2 := restic.Hash(load(t, c, h))
 
-			if !id.Equal(id2) {
-				t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
+			if id != id2.String() {
+				t.Errorf("wrong data returned, want %v, got %v", id, id2.String())
 			}
 
 			if !c.Has(h) {
-				t.Errorf("cache thinks index %v isn't present", id.Str())
+				t.Errorf("cache thinks index %v isn't present", id)
 			}
 
 			list := listFiles(t, c, tpe)
-			if !ids.Equals(list) {
+			if !maps.Equal(ids, list) {
 				t.Errorf("wrong list of index IDs returned, want:\n  %v\ngot:\n  %v", ids, list)
 			}
 
-			clearFiles(t, c, tpe, restic.NewIDSet(id))
+			clearFiles(t, c, tpe, map[string]struct{}{id: {}})
 			list2 := listFiles(t, c, tpe)
-			ids.Delete(id)
-			want := restic.NewIDSet(id)
-			if !list2.Equals(want) {
+			delete(ids, id)
+			want := map[string]struct{}{id: {}}
+			if !maps.Equal(list2, want) {
 				t.Errorf("ClearIndexes removed indexes, want:\n  %v\ngot:\n  %v", list2, want)
 			}
 
-			clearFiles(t, c, tpe, restic.NewIDSet())
-			want = restic.NewIDSet()
+			clearFiles(t, c, tpe, map[string]struct{}{})
+			want = map[string]struct{}{}
 			list3 := listFiles(t, c, tpe)
-			if !list3.Equals(want) {
+			if !maps.Equal(list3, want) {
 				t.Errorf("ClearIndexes returned a wrong list, want:\n  %v\ngot:\n  %v", want, list3)
 			}
 		})
diff --git a/internal/backend/cache/testing.go b/internal/backend/cache/testing.go
index d4ce8c2ff..f88b72b13 100644
--- a/internal/backend/cache/testing.go
+++ b/internal/backend/cache/testing.go
@@ -3,16 +3,17 @@ package cache
 import (
 	"testing"
 
-	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
 )
 
+const testCacheID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
 // TestNewCache returns a cache in a temporary directory which is removed when
 // cleanup is called.
 func TestNewCache(t testing.TB) *Cache {
 	dir := test.TempDir(t)
 	t.Logf("created new cache at %v", dir)
-	cache, err := New(restic.NewRandomID().String(), dir)
+	cache, err := New(testCacheID, dir)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 7bc61bcc0..91e1fcff9 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -842,8 +842,13 @@ func (r *Repository) prepareCache() error {
 
 	packs := r.idx.Packs(restic.NewIDSet())
 
+	ids := make(map[string]struct{})
+	for id := range packs {
+		ids[id.String()] = struct{}{}
+	}
+
 	// clear old packs
-	return r.cache.Clear(restic.PackFile, packs)
+	return r.cache.Clear(backend.PackFile, ids)
 }
 
 // SearchKey finds a key with the supplied password, afterwards the config is