mirror of
https://github.com/restic/restic.git
synced 2026-06-28 11:34:18 +00:00
repository: expose ZeroChunk via chunker factory
This commit is contained in:
@@ -19,11 +19,15 @@ func (c *baseChunker) NextSplitPoint(buf []byte) int {
|
||||
}
|
||||
|
||||
type chunkerFactory struct {
|
||||
pol chunker.Pol
|
||||
pol chunker.Pol
|
||||
zeroChunk func() restic.ID
|
||||
}
|
||||
|
||||
func newChunkerFactory(pol chunker.Pol) *chunkerFactory {
|
||||
return &chunkerFactory{pol: pol}
|
||||
func newChunkerFactory(r *Repository) *chunkerFactory {
|
||||
return &chunkerFactory{
|
||||
pol: r.Config().ChunkerPolynomial,
|
||||
zeroChunk: r.zeroChunk,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *chunkerFactory) NewChunker() restic.Chunker {
|
||||
@@ -34,6 +38,10 @@ func (f *chunkerFactory) MaxChunkSize() int {
|
||||
return chunker.MaxSize
|
||||
}
|
||||
|
||||
func (r *Repository) ChunkerFactory() restic.ChunkerFactory {
|
||||
return newChunkerFactory(r.Config().ChunkerPolynomial)
|
||||
func (f *chunkerFactory) ZeroChunk() restic.ID {
|
||||
return f.zeroChunk()
|
||||
}
|
||||
|
||||
func (r *Repository) ChunkerFactory() restic.ChunkerFactory {
|
||||
return newChunkerFactory(r)
|
||||
}
|
||||
|
||||
@@ -51,6 +51,9 @@ type Repository struct {
|
||||
allocDec sync.Once
|
||||
enc *zstd.Encoder
|
||||
dec *zstd.Decoder
|
||||
|
||||
zeroChunkOnce sync.Once
|
||||
zeroChunkID restic.ID
|
||||
}
|
||||
|
||||
// internalRepository allows using SaveUnpacked and RemoveUnpacked with all FileTypes
|
||||
@@ -1023,7 +1026,7 @@ func (r *Repository) saveBlob(ctx context.Context, t restic.BlobType, buf []byte
|
||||
// useful for sparse files containing large all zero regions. For these we can
|
||||
// process chunks as fast as we can read the from disk.
|
||||
if len(buf) == chunker.MinSize && restic.ZeroPrefixLen(buf) == chunker.MinSize {
|
||||
newID = ZeroChunk()
|
||||
newID = r.zeroChunk()
|
||||
} else {
|
||||
newID = restic.Hash(buf)
|
||||
}
|
||||
@@ -1340,13 +1343,9 @@ func (b *packBlobIterator) Next() (packBlobValue, error) {
|
||||
return packBlobValue{entry.BlobHandle, plaintext, err}, nil
|
||||
}
|
||||
|
||||
var zeroChunkOnce sync.Once
|
||||
var zeroChunkID restic.ID
|
||||
|
||||
// ZeroChunk computes and returns (cached) the ID of an all-zero chunk with size chunker.MinSize
|
||||
func ZeroChunk() restic.ID {
|
||||
zeroChunkOnce.Do(func() {
|
||||
zeroChunkID = restic.Hash(make([]byte, chunker.MinSize))
|
||||
func (r *Repository) zeroChunk() restic.ID {
|
||||
r.zeroChunkOnce.Do(func() {
|
||||
r.zeroChunkID = restic.Hash(make([]byte, chunker.MinSize))
|
||||
})
|
||||
return zeroChunkID
|
||||
return r.zeroChunkID
|
||||
}
|
||||
|
||||
@@ -17,4 +17,6 @@ type ChunkerFactory interface {
|
||||
NewChunker() Chunker
|
||||
// MaxChunkSize is the maximum size of a single chunk (used for output buffer pools).
|
||||
MaxChunkSize() int
|
||||
// ZeroChunk returns the ID of an all-zero chunk with minimum chunk size.
|
||||
ZeroChunk() ID
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/feature"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
@@ -72,7 +71,8 @@ func newFileRestorer(dst string,
|
||||
sparse bool,
|
||||
allowRecursiveDelete bool,
|
||||
startWarmup startWarmupFn,
|
||||
progress ProgressReporter) *fileRestorer {
|
||||
progress ProgressReporter,
|
||||
zeroChunk restic.ID) *fileRestorer {
|
||||
|
||||
// as packs are streamed the concurrency is limited by IO
|
||||
workerCount := int(connections)
|
||||
@@ -82,7 +82,7 @@ func newFileRestorer(dst string,
|
||||
blobsLoader: blobsLoader,
|
||||
startWarmup: startWarmup,
|
||||
filesWriter: newFilesWriter(workerCount, allowRecursiveDelete),
|
||||
zeroChunk: repository.ZeroChunk(),
|
||||
zeroChunk: zeroChunk,
|
||||
sparse: sparse,
|
||||
progress: progressOrNoop(progress),
|
||||
allowRecursiveDelete: allowRecursiveDelete,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/feature"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
@@ -208,7 +209,8 @@ func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files ma
|
||||
t.Helper()
|
||||
repo := newTestRepo(content)
|
||||
|
||||
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, false, repo.StartWarmup, nil)
|
||||
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, false, repo.StartWarmup, nil,
|
||||
repository.TestRepository(t).ChunkerFactory().ZeroChunk())
|
||||
|
||||
if files == nil {
|
||||
r.files = repo.files
|
||||
@@ -358,7 +360,8 @@ func TestErrorRestoreFiles(t *testing.T) {
|
||||
return loadError
|
||||
}
|
||||
|
||||
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, repo.StartWarmup, nil)
|
||||
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, repo.StartWarmup, nil,
|
||||
repository.TestRepository(t).ChunkerFactory().ZeroChunk())
|
||||
r.files = repo.files
|
||||
|
||||
err := r.restoreFiles(context.TODO())
|
||||
@@ -399,7 +402,8 @@ func TestFatalDownloadError(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, repo.StartWarmup, nil)
|
||||
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, repo.StartWarmup, nil,
|
||||
repository.TestRepository(t).ChunkerFactory().ZeroChunk())
|
||||
r.files = repo.files
|
||||
|
||||
var errors []string
|
||||
|
||||
@@ -362,7 +362,8 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error)
|
||||
|
||||
idx := data.NewHardlinkIndex[string]()
|
||||
filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.LookupBlob,
|
||||
res.repo.Connections(), res.opts.Sparse, res.opts.Delete, res.repo.StartWarmup, res.opts.Progress)
|
||||
res.repo.Connections(), res.opts.Sparse, res.opts.Delete, res.repo.StartWarmup, res.opts.Progress,
|
||||
res.repo.ChunkerFactory().ZeroChunk())
|
||||
filerestorer.Error = res.Error
|
||||
filerestorer.Info = res.Info
|
||||
|
||||
|
||||
Reference in New Issue
Block a user