diff --git a/changelog/unreleased/issue-3129 b/changelog/unreleased/issue-3129 new file mode 100644 index 000000000..3340dca0e --- /dev/null +++ b/changelog/unreleased/issue-3129 @@ -0,0 +1,7 @@ +Enhancement: Add JSON support to prune + +Restic `prune` now also supports the `--json` option and gives all +statistics in JSON format. + +https://github.com/restic/restic/issues/3129 +https://github.com/restic/restic/pull/5239 diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 976db4d0d..f1521f1c2 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -348,7 +348,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs)) } pruneOptions.DryRun = opts.DryRun - return runPruneWithRepo(ctx, pruneOptions, repo, removeSnIDs, printer) + return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, printer) } return nil diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index df08c3a33..c0d148990 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -175,7 +175,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term return errors.Fatal("--no-lock is only applicable in combination with --dry-run for prune command") } - printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) + printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term) ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock, printer) if err != nil { return err @@ -190,11 +190,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term opts.unsafeRecovery = true } - return runPruneWithRepo(ctx, opts, repo, restic.NewIDSet(), printer) + return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), printer) } -func runPruneWithRepo(ctx context.Context, opts PruneOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) error { - if repo.Cache() == nil { +func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts global.Options, repo *repository.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) error { + if repo.Cache() == nil && !gopts.JSON { printer.S("warning: running prune without a cache, this may be very slow!") } @@ -230,9 +230,13 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, repo *repository.R printer.P("\nWould have made the following changes:") } - err = printPruneStats(printer, plan.Stats()) - if err != nil { - return err + if !gopts.JSON { + err = printPruneStats(printer, plan.Stats()) + if err != nil { + return err + } + } else { + gopts.Term.Print(ui.ToJSONString(plan.Stats())) } // Trigger GC to reset garbage collection threshold @@ -251,24 +255,19 @@ func printPruneStats(printer progress.Printer, stats repository.PruneStats) erro if stats.Size.Unref > 0 { printer.V("unreferenced: %s", ui.FormatBytes(stats.Size.Unref)) } - totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate - totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref - unusedSize := stats.Size.Duplicate + stats.Size.Unused - printer.V("total: %10d blobs / %s", totalBlobs, ui.FormatBytes(totalSize)) - printer.V("unused size: %s of total size", ui.FormatPercent(unusedSize, totalSize)) + printer.V("total: %10d blobs / %s", stats.Blobs.Total, ui.FormatBytes(stats.Size.Total)) + printer.V("unused size: %s of total size", ui.FormatPercent(stats.Size.Duplicate+stats.Size.Unused, stats.Size.Total)) printer.P("\nto repack: %10d blobs / %s", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack)) printer.P("this removes: %10d blobs / %s", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm)) printer.P("to delete: %10d blobs / %s", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref)) - totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref - printer.P("total prune: %10d blobs / %s", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize)) + printer.P("total prune: %10d blobs / %s", stats.Blobs.RemoveTotal, ui.FormatBytes(stats.Size.RemoveTotal)) if stats.Size.Uncompressed > 0 { printer.P("not yet compressed: %s", ui.FormatBytes(stats.Size.Uncompressed)) } - printer.P("remaining: %10d blobs / %s", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize)) - unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm + printer.P("remaining: %10d blobs / %s", stats.Blobs.Remain, ui.FormatBytes(stats.Size.Remain)) printer.P("unused size after prune: %s (%s of remaining size)", - ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize)) + ui.FormatBytes(stats.Size.RemainUnused), ui.FormatPercent(stats.Size.RemainUnused, stats.Size.Remain)) printer.P("") printer.V("totally used packs: %10d", stats.Packs.Used) printer.V("partly used packs: %10d", stats.Packs.PartlyUsed) diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 74dc6d363..200d36f06 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -258,3 +258,28 @@ func TestPruneRepackSmallerThanSmoke(t *testing.T) { MaxUnused: "5%", }) } + +func TestPruneJSON(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + createPrunableRepo(t, env) + + buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error { + gopts.JSON = true + oldHook := gopts.BackendTestHook + gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } + defer func() { + gopts.BackendTestHook = oldHook + }() + return runPrune(ctx, pruneDefaultOptions, gopts, gopts.Term) + }) + rtest.OK(t, err) + + var stats repository.PruneStats + rtest.OK(t, json.Unmarshal(buf.Bytes(), &stats)) + + rtest.Equals(t, "summary", stats.MessageType) + rtest.Assert(t, stats.Blobs.Total > 0, "expected non-zero total blobs, got %v", stats.Blobs.Total) + rtest.Assert(t, stats.Packs.Total > 0, "expected non-zero total packs, got %v", stats.Packs.Total) +} diff --git a/doc/060_forget.rst b/doc/060_forget.rst index 9dcb45531..fe737add4 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -507,6 +507,8 @@ The ``prune`` command accepts the following options: - ``--verbose`` increased verbosity shows additional statistics for ``prune``. +- ``--json`` gives the statistics in JSON format. + Recovering from "no free space" errors ************************************** diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 70bde985e..5c44514bd 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -534,9 +534,6 @@ forget The ``forget`` command prints a single JSON document containing an array of ForgetGroups. If specific snapshot IDs are specified, then no output is generated. -The ``prune`` command does not yet support JSON such that ``forget --prune`` -results in a mix of JSON and text output. - ForgetGroup ^^^^^^^^^^^ @@ -599,6 +596,100 @@ KeepReason object +--------------+--------------------------------------------------------+--------------------+ +prune +----- + +The ``prune`` command uses the JSON lines format, but only outputs a single message. + ++------------------+----------------------------------------+--------------------------+ +| ``message_type`` | Always "summary" | string | ++------------------+----------------------------------------+--------------------------+ +| ``blobs`` | Statistics regarding data blobs | `PruneBlobs object`_ | ++------------------+----------------------------------------+--------------------------+ +| ``bytes`` | Statistics regarding sizes in bytes | `PruneSizes object`_ | ++------------------+----------------------------------------+--------------------------+ +| ``packfiles`` | Statistics regarding packfiles | `PrunePackfiles object`_ | ++------------------+----------------------------------------+--------------------------+ + +.. _PruneBlobs object: + +PruneBlobs object + ++-----------------+----------------------------------------+------+ +| ``used`` | Number of used blobs | uint | ++-----------------+----------------------------------------+------+ +| ``duplicate`` | Number of duplicate blobs | uint | ++-----------------+----------------------------------------+------+ +| ``unused`` | Number of unused blobs | uint | ++-----------------+----------------------------------------+------+ +| ``total`` | Total number of blobs | uint | ++-----------------+----------------------------------------+------+ +| ``repack`` | Number of blobs to be repacked | uint | ++-----------------+----------------------------------------+------+ +| ``repack_remove`` | Number of blobs removed by repacking | uint | ++-----------------+----------------------------------------+------+ +| ``remove`` | Number of blobs removed by pack deletion | uint | ++-----------------+----------------------------------------+------+ +| ``remove_total`` | Total number of blobs to be removed | uint | ++-----------------+----------------------------------------+------+ +| ``remaining`` | Number of blobs remaining | uint | ++-----------------+----------------------------------------+------+ + +.. _PruneSizes object: + +PruneSizes object + ++--------------------+-------------------------------------+--------+ +| ``used`` | Size of used blobs | uint64 | ++--------------------+-------------------------------------+--------+ +| ``duplicate`` | Size of duplicate blobs | uint64 | ++--------------------+-------------------------------------+--------+ +| ``unused`` | Size of unused blobs | uint64 | ++--------------------+-------------------------------------+--------+ +| ``unreferenced`` | Size of unreferenced pack files | uint64 | ++--------------------+-------------------------------------+--------+ +| ``uncompressed`` | Size of uncompressed pack files | uint64 | ++--------------------+-------------------------------------+--------+ +| ``total`` | Total size of blobs | uint64 | ++--------------------+-------------------------------------+--------+ +| ``repack`` | Size of blobs to be repacked | uint64 | ++--------------------+-------------------------------------+--------+ +| ``repack_remove`` | Size of blobs removed by repacking | uint64 | ++--------------------+-------------------------------------+--------+ +| ``remove`` | Size of blobs removed by pack deletion | uint64 | ++--------------------+-------------------------------------+--------+ +| ``remove_total`` | Total size of blobs to be removed | uint64 | ++--------------------+-------------------------------------+--------+ +| ``remaining`` | Size of blobs remaining | uint64 | ++--------------------+-------------------------------------+--------+ +| ``remaining_unused`` | Size of remaining unused blobs | uint64 | ++--------------------+-------------------------------------+--------+ + +.. _PrunePackfiles object: + +PrunePackfiles object + ++------------------+---------------------------------------+------+ +| ``used`` | Number of used pack files | uint | ++------------------+---------------------------------------+------+ +| ``unused`` | Number of unused pack files | uint | ++------------------+---------------------------------------+------+ +| ``partly_used`` | Number of partially used pack files | uint | ++------------------+---------------------------------------+------+ +| ``unreferenced`` | Number of unreferenced pack files | uint | ++------------------+---------------------------------------+------+ +| ``total`` | Total number of pack files | uint | ++------------------+---------------------------------------+------+ +| ``keep`` | Number of pack files to keep | uint | ++------------------+---------------------------------------+------+ +| ``repack`` | Number of pack files to repack | uint | ++------------------+---------------------------------------+------+ +| ``remove`` | Number of pack files to remove | uint | ++------------------+---------------------------------------+------+ +| ``remove_total`` | Total number of pack files to remove | uint | ++------------------+---------------------------------------+------+ + + init ---- diff --git a/internal/repository/prune.go b/internal/repository/prune.go index 393c2e45b..d534d20e3 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -34,33 +34,43 @@ type PruneOptions struct { } type PruneStats struct { - Blobs struct { - Used uint - Duplicate uint - Unused uint - Remove uint - Repack uint - Repackrm uint - } + MessageType string `json:"message_type"` + Blobs struct { + Used uint `json:"used"` + Duplicate uint `json:"duplicate"` + Unused uint `json:"unused"` + Total uint `json:"total"` + Repack uint `json:"repack"` + Repackrm uint `json:"repack_remove"` + Remove uint `json:"remove"` + RemoveTotal uint `json:"remove_total"` + Remain uint `json:"remaining"` + } `json:"blobs"` Size struct { - Used uint64 - Duplicate uint64 - Unused uint64 - Remove uint64 - Repack uint64 - Repackrm uint64 - Unref uint64 - Uncompressed uint64 - } + Used uint64 `json:"used"` + Duplicate uint64 `json:"duplicate"` + Unused uint64 `json:"unused"` + Unref uint64 `json:"unreferenced"` + Uncompressed uint64 `json:"uncompressed"` + Total uint64 `json:"total"` + Repack uint64 `json:"repack"` + Repackrm uint64 `json:"repack_remove"` + Remove uint64 `json:"remove"` + RemoveTotal uint64 `json:"remove_total"` + Remain uint64 `json:"remaining"` + RemainUnused uint64 `json:"remaining_unused"` + } `json:"bytes"` Packs struct { - Used uint - Unused uint - PartlyUsed uint - Unref uint - Keep uint - Repack uint - Remove uint - } + Used uint `json:"used"` + Unused uint `json:"unused"` + PartlyUsed uint `json:"partly_used"` + Unref uint `json:"unreferenced"` + Total uint `json:"total"` + Keep uint `json:"keep"` + Repack uint `json:"repack"` + Remove uint `json:"remove"` + RemoveTotal uint `json:"remove_total"` + } `json:"packfiles"` } type PrunePlan struct { @@ -95,7 +105,7 @@ type packInfoWithID struct { // PlanPrune selects which files to rewrite and which to delete and which blobs to keep. // Also some summary statistics are returned. func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error, printer progress.Printer) (*PrunePlan, error) { - var stats PruneStats + stats := PruneStats{MessageType: "summary"} if opts.UnsafeRecovery { // prevent repacking data to make sure users cannot get stuck. @@ -147,6 +157,17 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed } plan.keepBlobs = keepBlobs + // calculate totals for statistics + stats.Blobs.Total = stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate + stats.Blobs.RemoveTotal = stats.Blobs.Remove + stats.Blobs.Repackrm + stats.Blobs.Remain = stats.Blobs.Total - stats.Blobs.RemoveTotal + stats.Size.Total = stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref + stats.Size.RemoveTotal = stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref + stats.Size.Remain = stats.Size.Total - stats.Size.RemoveTotal + stats.Size.RemainUnused = stats.Size.Duplicate + stats.Size.Unused - stats.Size.Remove - stats.Size.Repackrm + stats.Packs.Total = stats.Packs.Used + stats.Packs.PartlyUsed + stats.Packs.Unused + stats.Packs.Unref + stats.Packs.RemoveTotal = stats.Packs.Unref + stats.Packs.Remove + plan.repo = repo plan.stats = stats plan.opts = opts