amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-07-14 19:24:53 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3704 from MichaelEischer/compression-migrations

Browse Source 
Support migration to repository format with compression
This commit is contained in:
Alexander Neumann
2022-05-29 15:52:21 +02:00
committed by GitHub
parent 78a21bbccf 5c6db534d4
commit d2c5843c68
26 changed files with 439 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/restic/cmd_check.go
+13 -9
View File
@@ -219,15 +219,20 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
Verbosef("load indexes\n")
hints, errs := chkr.LoadIndex(gopts.ctx)
dupFound := false
errorsFound := false
suggestIndexRebuild := false
for _, hint := range hints {
Printf("%v\n", hint)
if _, ok := hint.(checker.ErrDuplicatePacks); ok {
dupFound = true
switch hint.(type) {
case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
Printf("%v\n", hint)
suggestIndexRebuild = true
default:
Warnf("error: %v\n", hint)
errorsFound = true
}
}
if dupFound {
if suggestIndexRebuild {
Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
}
@@ -238,7 +243,6 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("LoadIndex returned errors")
}
errorsFound := false
orphanedPacks := 0
errChan := make(chan error)
@@ -252,11 +256,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
continue
}
errorsFound = true
Warnf("%v\n", err)
Warnf("error: %v\n", err)
}
if orphanedPacks > 0 {
Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks)
Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
}
Verbosef("check snapshots, trees and blobs\n")
@@ -273,7 +277,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
for err := range errChan {
errorsFound = true
if e, ok := err.(checker.TreeError); ok {
if e, ok := err.(*checker.TreeError); ok {
Warnf("error for tree %v:\n", e.ID.Str())
for _, treeErr := range e.Errors {
Warnf(" %v\n", treeErr)
cmd/restic/cmd_migrate.go
+26 -4
View File
@@ -8,11 +8,12 @@ import (
)
var cmdMigrate = &cobra.Command{
Use: "migrate [flags] [name]",
Use: "migrate [flags] [migration name] [...]",
Short: "Apply migrations",
Long: `
The "migrate" command applies migrations to a repository. When no migration
name is explicitly given, a list of migrations that can be applied is printed.
The "migrate" command checks which migrations can be applied for a repository
and prints a list with available migration names. If one or more migration
names are specified, these migrations are applied.
EXIT STATUS
===========
@@ -41,6 +42,8 @@ func init() {
func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
ctx := gopts.ctx
Printf("available migrations:\n")
found := false
for _, m := range migrations.All {
ok, err := m.Check(ctx, repo)
if err != nil {
@@ -48,10 +51,15 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
}
if ok {
Printf(" %v: %v\n", m.Name(), m.Desc())
Printf(" %v\t%v\n", m.Name(), m.Desc())
found = true
}
}
if !found {
Printf("no migrations found")
}
return nil
}
@@ -76,6 +84,20 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
Warnf("check for migration %v failed, continuing anyway\n", m.Name())
}
repoCheckOpts := m.RepoCheckOptions()
if repoCheckOpts != nil {
Printf("checking repository integrity...\n")
checkOptions := CheckOptions{}
checkGopts := gopts
// the repository is already locked
checkGopts.NoLock = true
err = runCheck(checkOptions, checkGopts, []string{})
if err != nil {
return err
}
}
Printf("applying migration %v...\n", m.Name())
if err = m.Apply(ctx, repo); err != nil {
Warnf("migration %v failed: %v\n", m.Name(), err)
cmd/restic/cmd_prune.go
+28 -3
View File
@@ -51,6 +51,7 @@ type PruneOptions struct {
MaxRepackBytes uint64
RepackCachableOnly bool
RepackUncompressed bool
}
var pruneOptions PruneOptions
@@ -68,6 +69,7 @@ func addPruneOptions(c *cobra.Command) {
f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")
}
func verifyPruneOptions(opts *PruneOptions) error {
@@ -135,6 +137,10 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
return err
}
if opts.RepackUncompressed && gopts.Compression == repository.CompressionOff {
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
@@ -144,6 +150,10 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
return errors.Fatal("prune requires a backend connection limit of at least two")
}
if repo.Config().Version < 2 && opts.RepackUncompressed {
return errors.Fatal("compression requires at least repository format version 2")
}
if opts.UnsafeNoSpaceRecovery != "" {
repoID := repo.Config().ID
if opts.UnsafeNoSpaceRecovery != repoID {
@@ -191,6 +201,7 @@ type packInfo struct {
usedSize uint64
unusedSize uint64
tpe restic.BlobType
uncompressed bool
}
type packInfoWithID struct {
@@ -299,6 +310,9 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
ip.unusedSize += size
ip.unusedBlobs++
}
if !blob.IsCompressed() {
ip.uncompressed = true
}
// update indexPack
indexPack[blob.PackID] = ip
}
@@ -318,6 +332,8 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
}
}
repoVersion := repo.Config().Version
// loop over all packs and decide what to do
bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
@@ -350,6 +366,15 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
stats.packs.partlyUsed++
}
mustCompress := false
if repoVersion >= 2 {
// repo v2: always repack tree blobs if uncompressed
// compress data blobs if requested
mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
}
// use a flag that pack must be compressed
p.uncompressed = mustCompress
// decide what to do
switch {
case p.usedBlobs == 0 && p.duplicateBlobs == 0:
@@ -362,7 +387,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
// if this is a data pack and --repack-cacheable-only is set => keep pack!
keep(p)
case p.unusedBlobs == 0 && p.duplicateBlobs == 0 && p.tpe != restic.InvalidBlob:
case p.unusedBlobs == 0 && p.duplicateBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
// All blobs in pack are used and not duplicates/mixed => keep pack!
keep(p)
@@ -447,8 +472,8 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
case reachedRepackSize:
keep(p.packInfo)
case p.duplicateBlobs > 0, p.tpe != restic.DataBlob:
// repacking duplicates/non-data is only limited by repackSize
case p.duplicateBlobs > 0, p.tpe != restic.DataBlob, p.uncompressed:
// repacking duplicates/non-data/uncompressed-trees is only limited by repackSize
repack(p.ID, p.packInfo)
case reachedUnusedSizeAfter: