amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-06-17 22:24:17 +00:00
Code Issues Projects Releases Wiki Activity

Json prune (#5239)

Browse Source 
Co-authored-by: Alexander Weiss <alex@weissfam.de>
Co-authored-by: Michael Eischer <michael.eischer@fau.de>
This commit is contained in:
darkdragon-001
2026-06-13 12:38:57 +02:00
committed by GitHub
parent bf56d71b09
commit e5dba15367
7 changed files with 192 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changelog/unreleased/issue-3129
+7
View File
@@ -0,0 +1,7 @@
Enhancement: Add JSON support to prune
Restic `prune` now also supports the `--json` option and gives all
statistics in JSON format.
https://github.com/restic/restic/issues/3129
https://github.com/restic/restic/pull/5239
cmd/restic/cmd_forget.go
+1 -1
View File
@@ -348,7 +348,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
}
pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(ctx, pruneOptions, repo, removeSnIDs, printer)
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, printer)
}
return nil
cmd/restic/cmd_prune.go
+16 -17
View File
@@ -175,7 +175,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for prune command")
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock, printer)
if err != nil {
return err
@@ -190,11 +190,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term
opts.unsafeRecovery = true
}
return runPruneWithRepo(ctx, opts, repo, restic.NewIDSet(), printer)
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), printer)
}
func runPruneWithRepo(ctx context.Context, opts PruneOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) error {
if repo.Cache() == nil {
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts global.Options, repo *repository.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) error {
if repo.Cache() == nil && !gopts.JSON {
printer.S("warning: running prune without a cache, this may be very slow!")
}
@@ -230,9 +230,13 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, repo *repository.R
printer.P("\nWould have made the following changes:")
}
err = printPruneStats(printer, plan.Stats())
if err != nil {
return err
if !gopts.JSON {
err = printPruneStats(printer, plan.Stats())
if err != nil {
return err
}
} else {
gopts.Term.Print(ui.ToJSONString(plan.Stats()))
}
// Trigger GC to reset garbage collection threshold
@@ -251,24 +255,19 @@ func printPruneStats(printer progress.Printer, stats repository.PruneStats) erro
if stats.Size.Unref > 0 {
printer.V("unreferenced: %s", ui.FormatBytes(stats.Size.Unref))
}
totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate
totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref
unusedSize := stats.Size.Duplicate + stats.Size.Unused
printer.V("total: %10d blobs / %s", totalBlobs, ui.FormatBytes(totalSize))
printer.V("unused size: %s of total size", ui.FormatPercent(unusedSize, totalSize))
printer.V("total: %10d blobs / %s", stats.Blobs.Total, ui.FormatBytes(stats.Size.Total))
printer.V("unused size: %s of total size", ui.FormatPercent(stats.Size.Duplicate+stats.Size.Unused, stats.Size.Total))
printer.P("\nto repack: %10d blobs / %s", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack))
printer.P("this removes: %10d blobs / %s", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm))
printer.P("to delete: %10d blobs / %s", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref))
totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref
printer.P("total prune: %10d blobs / %s", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize))
printer.P("total prune: %10d blobs / %s", stats.Blobs.RemoveTotal, ui.FormatBytes(stats.Size.RemoveTotal))
if stats.Size.Uncompressed > 0 {
printer.P("not yet compressed: %s", ui.FormatBytes(stats.Size.Uncompressed))
}
printer.P("remaining: %10d blobs / %s", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize))
unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm
printer.P("remaining: %10d blobs / %s", stats.Blobs.Remain, ui.FormatBytes(stats.Size.Remain))
printer.P("unused size after prune: %s (%s of remaining size)",
ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
ui.FormatBytes(stats.Size.RemainUnused), ui.FormatPercent(stats.Size.RemainUnused, stats.Size.Remain))
printer.P("")
printer.V("totally used packs: %10d", stats.Packs.Used)
printer.V("partly used packs: %10d", stats.Packs.PartlyUsed)
cmd/restic/cmd_prune_integration_test.go
+25
View File
@@ -258,3 +258,28 @@ func TestPruneRepackSmallerThanSmoke(t *testing.T) {
MaxUnused: "5%",
})
}
func TestPruneJSON(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
createPrunableRepo(t, env)
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
gopts.JSON = true
oldHook := gopts.BackendTestHook
gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
gopts.BackendTestHook = oldHook
}()
return runPrune(ctx, pruneDefaultOptions, gopts, gopts.Term)
})
rtest.OK(t, err)
var stats repository.PruneStats
rtest.OK(t, json.Unmarshal(buf.Bytes(), &stats))
rtest.Equals(t, "summary", stats.MessageType)
rtest.Assert(t, stats.Blobs.Total > 0, "expected non-zero total blobs, got %v", stats.Blobs.Total)
rtest.Assert(t, stats.Packs.Total > 0, "expected non-zero total packs, got %v", stats.Packs.Total)
}
doc/060_forget.rst
+2
View File
@@ -507,6 +507,8 @@ The ``prune`` command accepts the following options:
- ``--verbose`` increased verbosity shows additional statistics for ``prune``.
- ``--json`` gives the statistics in JSON format.
Recovering from "no free space" errors
**************************************
doc/075_scripting.rst
+94 -3
View File
@@ -534,9 +534,6 @@ forget
The ``forget`` command prints a single JSON document containing an array of
ForgetGroups. If specific snapshot IDs are specified, then no output is generated.
The ``prune`` command does not yet support JSON such that ``forget --prune``
results in a mix of JSON and text output.
ForgetGroup
^^^^^^^^^^^
@@ -599,6 +596,100 @@ KeepReason object
+--------------+--------------------------------------------------------+--------------------+
prune
-----
The ``prune`` command uses the JSON lines format, but only outputs a single message.
+------------------+----------------------------------------+--------------------------+
| ``message_type`` | Always "summary" | string |
+------------------+----------------------------------------+--------------------------+
| ``blobs`` | Statistics regarding data blobs | `PruneBlobs object`_ |
+------------------+----------------------------------------+--------------------------+
| ``bytes`` | Statistics regarding sizes in bytes | `PruneSizes object`_ |
+------------------+----------------------------------------+--------------------------+
| ``packfiles`` | Statistics regarding packfiles | `PrunePackfiles object`_ |
+------------------+----------------------------------------+--------------------------+
.. _PruneBlobs object:
PruneBlobs object
+-----------------+----------------------------------------+------+
| ``used`` | Number of used blobs | uint |
+-----------------+----------------------------------------+------+
| ``duplicate`` | Number of duplicate blobs | uint |
+-----------------+----------------------------------------+------+
| ``unused`` | Number of unused blobs | uint |
+-----------------+----------------------------------------+------+
| ``total`` | Total number of blobs | uint |
+-----------------+----------------------------------------+------+
| ``repack`` | Number of blobs to be repacked | uint |
+-----------------+----------------------------------------+------+
| ``repack_remove`` | Number of blobs removed by repacking | uint |
+-----------------+----------------------------------------+------+
| ``remove`` | Number of blobs removed by pack deletion | uint |
+-----------------+----------------------------------------+------+
| ``remove_total`` | Total number of blobs to be removed | uint |
+-----------------+----------------------------------------+------+
| ``remaining`` | Number of blobs remaining | uint |
+-----------------+----------------------------------------+------+
.. _PruneSizes object:
PruneSizes object
+--------------------+-------------------------------------+--------+
| ``used`` | Size of used blobs | uint64 |
+--------------------+-------------------------------------+--------+
| ``duplicate`` | Size of duplicate blobs | uint64 |
+--------------------+-------------------------------------+--------+
| ``unused`` | Size of unused blobs | uint64 |
+--------------------+-------------------------------------+--------+
| ``unreferenced`` | Size of unreferenced pack files | uint64 |
+--------------------+-------------------------------------+--------+
| ``uncompressed`` | Size of uncompressed pack files | uint64 |
+--------------------+-------------------------------------+--------+
| ``total`` | Total size of blobs | uint64 |
+--------------------+-------------------------------------+--------+
| ``repack`` | Size of blobs to be repacked | uint64 |
+--------------------+-------------------------------------+--------+
| ``repack_remove`` | Size of blobs removed by repacking | uint64 |
+--------------------+-------------------------------------+--------+
| ``remove`` | Size of blobs removed by pack deletion | uint64 |
+--------------------+-------------------------------------+--------+
| ``remove_total`` | Total size of blobs to be removed | uint64 |
+--------------------+-------------------------------------+--------+
| ``remaining`` | Size of blobs remaining | uint64 |
+--------------------+-------------------------------------+--------+
| ``remaining_unused`` | Size of remaining unused blobs | uint64 |
+--------------------+-------------------------------------+--------+
.. _PrunePackfiles object:
PrunePackfiles object
+------------------+---------------------------------------+------+
| ``used`` | Number of used pack files | uint |
+------------------+---------------------------------------+------+
| ``unused`` | Number of unused pack files | uint |
+------------------+---------------------------------------+------+
| ``partly_used`` | Number of partially used pack files | uint |
+------------------+---------------------------------------+------+
| ``unreferenced`` | Number of unreferenced pack files | uint |
+------------------+---------------------------------------+------+
| ``total`` | Total number of pack files | uint |
+------------------+---------------------------------------+------+
| ``keep`` | Number of pack files to keep | uint |
+------------------+---------------------------------------+------+
| ``repack`` | Number of pack files to repack | uint |
+------------------+---------------------------------------+------+
| ``remove`` | Number of pack files to remove | uint |
+------------------+---------------------------------------+------+
| ``remove_total`` | Total number of pack files to remove | uint |
+------------------+---------------------------------------+------+
init
----
internal/repository/prune.go
+47 -26
View File
@@ -34,33 +34,43 @@ type PruneOptions struct {
}
type PruneStats struct {
Blobs struct {
Used uint
Duplicate uint
Unused uint
Remove uint
Repack uint
Repackrm uint
}
MessageType string `json:"message_type"`
Blobs struct {
Used uint `json:"used"`
Duplicate uint `json:"duplicate"`
Unused uint `json:"unused"`
Total uint `json:"total"`
Repack uint `json:"repack"`
Repackrm uint `json:"repack_remove"`
Remove uint `json:"remove"`
RemoveTotal uint `json:"remove_total"`
Remain uint `json:"remaining"`
} `json:"blobs"`
Size struct {
Used uint64
Duplicate uint64
Unused uint64
Remove uint64
Repack uint64
Repackrm uint64
Unref uint64
Uncompressed uint64
}
Used uint64 `json:"used"`
Duplicate uint64 `json:"duplicate"`
Unused uint64 `json:"unused"`
Unref uint64 `json:"unreferenced"`
Uncompressed uint64 `json:"uncompressed"`
Total uint64 `json:"total"`
Repack uint64 `json:"repack"`
Repackrm uint64 `json:"repack_remove"`
Remove uint64 `json:"remove"`
RemoveTotal uint64 `json:"remove_total"`
Remain uint64 `json:"remaining"`
RemainUnused uint64 `json:"remaining_unused"`
} `json:"bytes"`
Packs struct {
Used uint
Unused uint
PartlyUsed uint
Unref uint
Keep uint
Repack uint
Remove uint
}
Used uint `json:"used"`
Unused uint `json:"unused"`
PartlyUsed uint `json:"partly_used"`
Unref uint `json:"unreferenced"`
Total uint `json:"total"`
Keep uint `json:"keep"`
Repack uint `json:"repack"`
Remove uint `json:"remove"`
RemoveTotal uint `json:"remove_total"`
} `json:"packfiles"`
}
type PrunePlan struct {
@@ -95,7 +105,7 @@ type packInfoWithID struct {
// PlanPrune selects which files to rewrite and which to delete and which blobs to keep.
// Also some summary statistics are returned.
func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error, printer progress.Printer) (*PrunePlan, error) {
var stats PruneStats
stats := PruneStats{MessageType: "summary"}
if opts.UnsafeRecovery {
// prevent repacking data to make sure users cannot get stuck.
@@ -147,6 +157,17 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed
}
plan.keepBlobs = keepBlobs
// calculate totals for statistics
stats.Blobs.Total = stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate
stats.Blobs.RemoveTotal = stats.Blobs.Remove + stats.Blobs.Repackrm
stats.Blobs.Remain = stats.Blobs.Total - stats.Blobs.RemoveTotal
stats.Size.Total = stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref
stats.Size.RemoveTotal = stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref
stats.Size.Remain = stats.Size.Total - stats.Size.RemoveTotal
stats.Size.RemainUnused = stats.Size.Duplicate + stats.Size.Unused - stats.Size.Remove - stats.Size.Repackrm
stats.Packs.Total = stats.Packs.Used + stats.Packs.PartlyUsed + stats.Packs.Unused + stats.Packs.Unref
stats.Packs.RemoveTotal = stats.Packs.Unref + stats.Packs.Remove
plan.repo = repo
plan.stats = stats
plan.opts = opts