mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-25 13:24:57 +00:00
Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c573fa61cb | ||
|
|
5d0fb476cf | ||
|
|
e3d8bcb102 | ||
|
|
9558ede894 | ||
|
|
437a219658 | ||
|
|
396c0b539c | ||
|
|
50dec14653 | ||
|
|
cc4917efec | ||
|
|
672fc6a0d4 | ||
|
|
943d322196 | ||
|
|
4d956fa13b | ||
|
|
3963581af0 | ||
|
|
581fbab8f3 | ||
|
|
7044d8ae5f | ||
|
|
5e21ba891d | ||
|
|
c2c30541bb | ||
|
|
1ea9775ab4 | ||
|
|
02b97fbaaf | ||
|
|
0e538acdef | ||
|
|
7f58d16758 | ||
|
|
344414bb47 | ||
|
|
8263f75643 | ||
|
|
72d031a7e2 | ||
|
|
1a253d0f26 | ||
|
|
dc4cb829a6 | ||
|
|
cad2d2d151 | ||
|
|
8db09d7c92 | ||
|
|
b02a151e45 | ||
|
|
0083443e77 | ||
|
|
2cd9b94c43 | ||
|
|
b76bd9a7bb | ||
|
|
87a9fcadc8 | ||
|
|
1c4f6b6144 | ||
|
|
2d985961b0 | ||
|
|
08a2186de1 | ||
|
|
a89fdfb033 | ||
|
|
c0fc6c0475 | ||
|
|
28f291c503 | ||
|
|
8263460b7b | ||
|
|
f4e997e524 | ||
|
|
1944208de4 | ||
|
|
db14bd6b61 | ||
|
|
f41181f992 | ||
|
|
b4729ec5d2 | ||
|
|
3cd772d8e9 | ||
|
|
1203203e08 | ||
|
|
6cf3c47814 | ||
|
|
24aee98ad5 | ||
|
|
2ab1ac7cb8 |
@@ -272,6 +272,8 @@ optional arguments:
|
|||||||
-sm, --split-manifest
|
-sm, --split-manifest
|
||||||
-z, --zip
|
-z, --zip
|
||||||
-zn, --zip-name
|
-zn, --zip-name
|
||||||
|
--zip-compression
|
||||||
|
--zip-compression-level
|
||||||
--data-only
|
--data-only
|
||||||
--no-progress-bar
|
--no-progress-bar
|
||||||
--passphrase
|
--passphrase
|
||||||
@@ -334,6 +336,19 @@ If `-z` or `--zip` is provided, the export will be a zip file
|
|||||||
in the target directory, named according to the current local date or the
|
in the target directory, named according to the current local date or the
|
||||||
value set in `-zn` or `--zip-name`.
|
value set in `-zn` or `--zip-name`.
|
||||||
|
|
||||||
|
The compression method for the zip can be set with `--zip-compression`
|
||||||
|
(`stored`, `deflated` (default), `bzip2`, `lzma`, or `zstd`) and tuned with
|
||||||
|
`--zip-compression-level` (deflated: 0–9, bzip2: 1–9, zstd: -22–22; ignored
|
||||||
|
for `stored` and `lzma`). Both options require `--zip`.
|
||||||
|
|
||||||
|
!!! warning
|
||||||
|
|
||||||
|
`zstd` compression requires Python 3.14 or newer on **both** the machine
|
||||||
|
creating the export and any machine importing it. An archive compressed with
|
||||||
|
`zstd` (or `lzma`/`bzip2` where those modules are unavailable) cannot be
|
||||||
|
imported on a runtime that lacks the codec; the importer will refuse it with
|
||||||
|
a clear error. The default `deflated` is universally readable.
|
||||||
|
|
||||||
If `--data-only` is provided, only the database will be exported. This option is intended
|
If `--data-only` is provided, only the database will be exported. This option is intended
|
||||||
to facilitate database upgrades without needing to clean documents and thumbnails from the media directory.
|
to facilitate database upgrades without needing to clean documents and thumbnails from the media directory.
|
||||||
|
|
||||||
|
|||||||
+783
-1
@@ -1,5 +1,787 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## paperless-ngx 3.0.0
|
||||||
|
|
||||||
|
### Breaking Changes
|
||||||
|
|
||||||
|
- [BREAKING] Remove the positional arguments from the pre/post consume scripts [@stumpylog](https://github.com/stumpylog) ([#12573](https://github.com/paperless-ngx/paperless-ngx/pull/12573))
|
||||||
|
- [BREAKING] Decouple OCR control from archive file control [@stumpylog](https://github.com/stumpylog) ([#12448](https://github.com/paperless-ngx/paperless-ngx/pull/12448))
|
||||||
|
- [BREAKING] Chore: drop support for api versions \< 9 [@shamoon](https://github.com/shamoon) ([#12284](https://github.com/paperless-ngx/paperless-ngx/pull/12284))
|
||||||
|
- [BREAKING] Chore: Drop support for Python 3.10 [@stumpylog](https://github.com/stumpylog) ([#12234](https://github.com/paperless-ngx/paperless-ngx/pull/12234))
|
||||||
|
- [BREAKING] Chore: Refactor advanced database settings to allow more user configuration [@stumpylog](https://github.com/stumpylog) ([#12165](https://github.com/paperless-ngx/paperless-ngx/pull/12165))
|
||||||
|
- [BREAKING] Remove API v1 compatibility [@stumpylog](https://github.com/stumpylog) ([#12166](https://github.com/paperless-ngx/paperless-ngx/pull/12166))
|
||||||
|
- [BREAKING] Remove pybzar as a barcode reader [@stumpylog](https://github.com/stumpylog) ([#12065](https://github.com/paperless-ngx/paperless-ngx/pull/12065))
|
||||||
|
- [BREAKING] Remove support for document and thumbnail encryption [@stumpylog](https://github.com/stumpylog) ([#11850](https://github.com/paperless-ngx/paperless-ngx/pull/11850))
|
||||||
|
- [BREAKING] Feature: Simplify and improve the consumer [@stumpylog](https://github.com/stumpylog) ([#11753](https://github.com/paperless-ngx/paperless-ngx/pull/11753))
|
||||||
|
|
||||||
|
### Notable Changes
|
||||||
|
|
||||||
|
- Feature: Replace Whoosh with tantivy search backend [@stumpylog](https://github.com/stumpylog) ([#12471](https://github.com/paperless-ngx/paperless-ngx/pull/12471))
|
||||||
|
- Feature: document parser plugin framework [@stumpylog](https://github.com/stumpylog) ([#12294](https://github.com/paperless-ngx/paperless-ngx/pull/12294))
|
||||||
|
- Feature: document file versions [@shamoon](https://github.com/shamoon) ([#12061](https://github.com/paperless-ngx/paperless-ngx/pull/12061))
|
||||||
|
- Feature: sharelink bundles [@shamoon](https://github.com/shamoon) ([#11682](https://github.com/paperless-ngx/paperless-ngx/pull/11682))
|
||||||
|
- Feature: Paperless AI [@shamoon](https://github.com/shamoon) ([#10319](https://github.com/paperless-ngx/paperless-ngx/pull/10319))
|
||||||
|
- Feature: Remote OCR (Azure AI) [@shamoon](https://github.com/shamoon) ([#10320](https://github.com/paperless-ngx/paperless-ngx/pull/10320))
|
||||||
|
|
||||||
|
### Features / Enhancements
|
||||||
|
|
||||||
|
- [Beta] Paperless-ngx v3.0.0 Beta [@shamoon](https://github.com/shamoon) ([#12713](https://github.com/paperless-ngx/paperless-ngx/pull/12713))
|
||||||
|
- Tweakhancement (beta): allow some bulk operations with all [@shamoon](https://github.com/shamoon) ([#13193](https://github.com/paperless-ngx/paperless-ngx/pull/13193))
|
||||||
|
- Security (beta): enforce current permissions in autocomplete [@shamoon](https://github.com/shamoon) ([#13188](https://github.com/paperless-ngx/paperless-ngx/pull/13188))
|
||||||
|
- Security (beta): bound email linkification [@shamoon](https://github.com/shamoon) ([#13187](https://github.com/paperless-ngx/paperless-ngx/pull/13187))
|
||||||
|
- Change: update root modified timestamp on version changes [@shamoon](https://github.com/shamoon) ([#13170](https://github.com/paperless-ngx/paperless-ngx/pull/13170))
|
||||||
|
- Performance: add DB indexes for common query/sort patterns [@stumpylog](https://github.com/stumpylog) ([#13167](https://github.com/paperless-ngx/paperless-ngx/pull/13167))
|
||||||
|
- Performance: use NgOptimizedImage for thumbnail lazy loading [@shamoon](https://github.com/shamoon) ([#13169](https://github.com/paperless-ngx/paperless-ngx/pull/13169))
|
||||||
|
- Performance: Tantivy indexing optimization [@Merinorus](https://github.com/Merinorus) ([#13053](https://github.com/paperless-ngx/paperless-ngx/pull/13053))
|
||||||
|
- Chorehancement: update to Angular v22, 'zoneless' / 'reactive' [@shamoon](https://github.com/shamoon) ([#13114](https://github.com/paperless-ngx/paperless-ngx/pull/13114))
|
||||||
|
- Enhancement (beta): support LLM timeout config [@shamoon](https://github.com/shamoon) ([#13002](https://github.com/paperless-ngx/paperless-ngx/pull/13002))
|
||||||
|
- Enhancement: ignore diacritics, support multiple substring matching for UI filtering [@shamoon](https://github.com/shamoon) ([#13021](https://github.com/paperless-ngx/paperless-ngx/pull/13021))
|
||||||
|
- Enhancement(beta): replace LanceDB vector store with sqlite-vec [@stumpylog](https://github.com/stumpylog) ([#12990](https://github.com/paperless-ngx/paperless-ngx/pull/12990))
|
||||||
|
- Enhancement (beta): Switch the AI vector store to LanceDB [@stumpylog](https://github.com/stumpylog) ([#12944](https://github.com/paperless-ngx/paperless-ngx/pull/12944))
|
||||||
|
- Fixhancement (beta): tasks dismiss all [@shamoon](https://github.com/shamoon) ([#12949](https://github.com/paperless-ngx/paperless-ngx/pull/12949))
|
||||||
|
- Enhancement (beta): add direct LLM language setting [@shamoon](https://github.com/shamoon) ([#12906](https://github.com/paperless-ngx/paperless-ngx/pull/12906))
|
||||||
|
- Enhancement: try to respect language for AI suggestions [@shamoon](https://github.com/shamoon) ([#12894](https://github.com/paperless-ngx/paperless-ngx/pull/12894))
|
||||||
|
- Enhancement: AI LLM chunk size and context window config [@shamoon](https://github.com/shamoon) ([#12891](https://github.com/paperless-ngx/paperless-ngx/pull/12891))
|
||||||
|
- Enhancement: support ollama embeddings [@shamoon](https://github.com/shamoon) ([#12753](https://github.com/paperless-ngx/paperless-ngx/pull/12753))
|
||||||
|
- Tweakhancment: Include the last applied 'documents' migration in the log [@stumpylog](https://github.com/stumpylog) ([#12757](https://github.com/paperless-ngx/paperless-ngx/pull/12757))
|
||||||
|
- Enhancement: version-aware thumbnail etag [@shamoon](https://github.com/shamoon) ([#12754](https://github.com/paperless-ngx/paperless-ngx/pull/12754))
|
||||||
|
- Feature: Further reduce document importer memory usage [@stumpylog](https://github.com/stumpylog) ([#12707](https://github.com/paperless-ngx/paperless-ngx/pull/12707))
|
||||||
|
- Tweakhancement: use fixed position instead of display none for printing [@shamoon](https://github.com/shamoon) ([#12706](https://github.com/paperless-ngx/paperless-ngx/pull/12706))
|
||||||
|
- Enhancement: Paperless-ngx v3 Logo [@shamoon](https://github.com/shamoon) ([#12673](https://github.com/paperless-ngx/paperless-ngx/pull/12673))
|
||||||
|
- Tweakhancement: localize some more task result messages [@shamoon](https://github.com/shamoon) ([#12672](https://github.com/paperless-ngx/paperless-ngx/pull/12672))
|
||||||
|
- Enhancement: chat message document links [@shamoon](https://github.com/shamoon) ([#12670](https://github.com/paperless-ngx/paperless-ngx/pull/12670))
|
||||||
|
- Tweakhancement: make upload notification open an anchor link [@shamoon](https://github.com/shamoon) ([#12659](https://github.com/paperless-ngx/paperless-ngx/pull/12659))
|
||||||
|
- Tweakhancement: tweak tasks UI, make open doc an anchor [@shamoon](https://github.com/shamoon) ([#12658](https://github.com/paperless-ngx/paperless-ngx/pull/12658))
|
||||||
|
- Enhancement: show small task summary in system status [@shamoon](https://github.com/shamoon) ([#12634](https://github.com/paperless-ngx/paperless-ngx/pull/12634))
|
||||||
|
- Enhancement: new Tasks UI [@shamoon](https://github.com/shamoon) ([#12614](https://github.com/paperless-ngx/paperless-ngx/pull/12614))
|
||||||
|
- Feature: Allow monitoring access to tasks summary [@stumpylog](https://github.com/stumpylog) ([#12624](https://github.com/paperless-ngx/paperless-ngx/pull/12624))
|
||||||
|
- Tweak: remove stale index warning [@shamoon](https://github.com/shamoon) ([#12616](https://github.com/paperless-ngx/paperless-ngx/pull/12616))
|
||||||
|
- Enhancement: add highlighting to title + content searches [@shamoon](https://github.com/shamoon) ([#12593](https://github.com/paperless-ngx/paperless-ngx/pull/12593))
|
||||||
|
- Feature: Redesign the task system [@stumpylog](https://github.com/stumpylog) ([#12584](https://github.com/paperless-ngx/paperless-ngx/pull/12584))
|
||||||
|
- Feature: Tune SQLite default settings for increased speed [@stumpylog](https://github.com/stumpylog) ([#12580](https://github.com/paperless-ngx/paperless-ngx/pull/12580))
|
||||||
|
- Feature: Document fuzzy match improvements [@stumpylog](https://github.com/stumpylog) ([#12579](https://github.com/paperless-ngx/paperless-ngx/pull/12579))
|
||||||
|
- Enhancement: validate and sanitize uploaded logos [@shamoon](https://github.com/shamoon) ([#12551](https://github.com/paperless-ngx/paperless-ngx/pull/12551))
|
||||||
|
- Enhancement: add view\_global\_statistics and view\_system\_status permissions [@shamoon](https://github.com/shamoon) ([#12530](https://github.com/paperless-ngx/paperless-ngx/pull/12530))
|
||||||
|
- Fixhancement: include sharelinks + bundles in export/import [@shamoon](https://github.com/shamoon) ([#12479](https://github.com/paperless-ngx/paperless-ngx/pull/12479))
|
||||||
|
- Enhancement: unify text search to use tantivy [@shamoon](https://github.com/shamoon) ([#12485](https://github.com/paperless-ngx/paperless-ngx/pull/12485))
|
||||||
|
- Feature: Consumer logging correlation [@stumpylog](https://github.com/stumpylog) ([#12510](https://github.com/paperless-ngx/paperless-ngx/pull/12510))
|
||||||
|
- Security enhancement: allow opt-in blocking internal mail hosts [@shamoon](https://github.com/shamoon) ([#12502](https://github.com/paperless-ngx/paperless-ngx/pull/12502))
|
||||||
|
- Fixhancement: include trashed documents in document exporter/importer [@JanKleine](https://github.com/JanKleine) ([#12425](https://github.com/paperless-ngx/paperless-ngx/pull/12425))
|
||||||
|
- Tweakhancement: show file extension in StoragePath test [@shamoon](https://github.com/shamoon) ([#12452](https://github.com/paperless-ngx/paperless-ngx/pull/12452))
|
||||||
|
- Feature: Transition all checksums to use SHA256 [@stumpylog](https://github.com/stumpylog) ([#12432](https://github.com/paperless-ngx/paperless-ngx/pull/12432))
|
||||||
|
- Enhancement: auto-hide the search bar on mobile [@shamoon](https://github.com/shamoon) ([#12404](https://github.com/paperless-ngx/paperless-ngx/pull/12404))
|
||||||
|
- Change: sort custom fields alphabetically by default [@shamoon](https://github.com/shamoon) ([#12358](https://github.com/paperless-ngx/paperless-ngx/pull/12358))
|
||||||
|
- Feature: Add progress information to the classifier training for a better ux [@stumpylog](https://github.com/stumpylog) ([#12331](https://github.com/paperless-ngx/paperless-ngx/pull/12331))
|
||||||
|
- Enhancement: Make the StatusConsumer truly async [@stumpylog](https://github.com/stumpylog) ([#12298](https://github.com/paperless-ngx/paperless-ngx/pull/12298))
|
||||||
|
- Feature: Migrate import/export to rich progress [@stumpylog](https://github.com/stumpylog) ([#12260](https://github.com/paperless-ngx/paperless-ngx/pull/12260))
|
||||||
|
- Enhancement: Show more document details in merge dialog [@svenstaro](https://github.com/svenstaro) ([#12271](https://github.com/paperless-ngx/paperless-ngx/pull/12271))
|
||||||
|
- Enhancement: saved view sharing [@shamoon](https://github.com/shamoon) ([#12142](https://github.com/paperless-ngx/paperless-ngx/pull/12142))
|
||||||
|
- Enhancement: “live” document updates [@shamoon](https://github.com/shamoon) ([#12141](https://github.com/paperless-ngx/paperless-ngx/pull/12141))
|
||||||
|
- Enhancement: Improve the retagger output using rich [@stumpylog](https://github.com/stumpylog) ([#12194](https://github.com/paperless-ngx/paperless-ngx/pull/12194))
|
||||||
|
- Enhancement: Transition sanity check to rich and improve output [@stumpylog](https://github.com/stumpylog) ([#12182](https://github.com/paperless-ngx/paperless-ngx/pull/12182))
|
||||||
|
- Enhancement: Switch all indexing to use rich [@stumpylog](https://github.com/stumpylog) ([#12193](https://github.com/paperless-ngx/paperless-ngx/pull/12193))
|
||||||
|
- Feature: Switch progress bar library to rich [@stumpylog](https://github.com/stumpylog) ([#12169](https://github.com/paperless-ngx/paperless-ngx/pull/12169))
|
||||||
|
- Enhancement: Formatted filename for single document downloads [@JanKleine](https://github.com/JanKleine) ([#12095](https://github.com/paperless-ngx/paperless-ngx/pull/12095))
|
||||||
|
- Enhancement: prevent duplicate mail processing across rules [@shamoon](https://github.com/shamoon) ([#12159](https://github.com/paperless-ngx/paperless-ngx/pull/12159))
|
||||||
|
- Feature: move to trash action for workflows [@JanKleine](https://github.com/JanKleine) ([#11176](https://github.com/paperless-ngx/paperless-ngx/pull/11176))
|
||||||
|
- Tweakhancement: reset to page 1 on reset filters [@shamoon](https://github.com/shamoon) ([#12143](https://github.com/paperless-ngx/paperless-ngx/pull/12143))
|
||||||
|
- Enhancement: consolidate management lists into document attributes section [@shamoon](https://github.com/shamoon) ([#12045](https://github.com/paperless-ngx/paperless-ngx/pull/12045))
|
||||||
|
- Enhancement: option to stop processing further mail rules [@shamoon](https://github.com/shamoon) ([#12053](https://github.com/paperless-ngx/paperless-ngx/pull/12053))
|
||||||
|
- Tweak: improve 2-digit year parsing [@shamoon](https://github.com/shamoon) ([#12044](https://github.com/paperless-ngx/paperless-ngx/pull/12044))
|
||||||
|
- Enhancement: pngx pdf viewer [@shamoon](https://github.com/shamoon) ([#12043](https://github.com/paperless-ngx/paperless-ngx/pull/12043))
|
||||||
|
- Fixhancement: improve ASN handling with PDF operations [@shamoon](https://github.com/shamoon) ([#11689](https://github.com/paperless-ngx/paperless-ngx/pull/11689))
|
||||||
|
- Feature: Enable users to customize date parsing via plugins [@stumpylog](https://github.com/stumpylog) ([#11931](https://github.com/paperless-ngx/paperless-ngx/pull/11931))
|
||||||
|
- Feature: password removal workflow action [@shamoon](https://github.com/shamoon) ([#11665](https://github.com/paperless-ngx/paperless-ngx/pull/11665))
|
||||||
|
- Enhancement: per-type object page sizing [@shamoon](https://github.com/shamoon) ([#11977](https://github.com/paperless-ngx/paperless-ngx/pull/11977))
|
||||||
|
- Enhancement: improve filter drop-down performance with virtual scrolling [@shamoon](https://github.com/shamoon) ([#11973](https://github.com/paperless-ngx/paperless-ngx/pull/11973))
|
||||||
|
- Tweakhancement: tweak bulk delete text [@shamoon](https://github.com/shamoon) ([#11967](https://github.com/paperless-ngx/paperless-ngx/pull/11967))
|
||||||
|
- Feature: support split documents based on tag barcodes @schober-ch ([#11645](https://github.com/paperless-ngx/paperless-ngx/pull/11645))
|
||||||
|
- Enhancement: Add setting for default PDF Editor mode [@JanKleine](https://github.com/JanKleine) ([#11927](https://github.com/paperless-ngx/paperless-ngx/pull/11927))
|
||||||
|
- Fixhancement: auto-queue llm index if needed [@shamoon](https://github.com/shamoon) ([#11891](https://github.com/paperless-ngx/paperless-ngx/pull/11891))
|
||||||
|
- Enhancement: user control of doc details fields [@shamoon](https://github.com/shamoon) ([#11906](https://github.com/paperless-ngx/paperless-ngx/pull/11906))
|
||||||
|
- Enhancement: improve relative dates in date filter [@JanKleine](https://github.com/JanKleine) ([#11899](https://github.com/paperless-ngx/paperless-ngx/pull/11899))
|
||||||
|
- Enhancement: allow duplicates with warnings, UI for discovery [@shamoon](https://github.com/shamoon) ([#11815](https://github.com/paperless-ngx/paperless-ngx/pull/11815))
|
||||||
|
- Enhancement: configurable SSO groups claim [@Gabgobie](https://github.com/Gabgobie) ([#11841](https://github.com/paperless-ngx/paperless-ngx/pull/11841))
|
||||||
|
- Enhancement: support select all for management lists [@shamoon](https://github.com/shamoon) ([#11889](https://github.com/paperless-ngx/paperless-ngx/pull/11889))
|
||||||
|
- Tweakhancement: display document id, with copy [@shamoon](https://github.com/shamoon) ([#11896](https://github.com/paperless-ngx/paperless-ngx/pull/11896))
|
||||||
|
- Enhancement: Add support for app oidc [@paulgessinger](https://github.com/paulgessinger) ([#11756](https://github.com/paperless-ngx/paperless-ngx/pull/11756))
|
||||||
|
- Enhancement: Add 'any of' workflow trigger filters [@shamoon](https://github.com/shamoon) ([#11683](https://github.com/paperless-ngx/paperless-ngx/pull/11683))
|
||||||
|
- Enhancement: support doc\_id placeholder in workflow templates [@shamoon](https://github.com/shamoon) ([#11847](https://github.com/paperless-ngx/paperless-ngx/pull/11847))
|
||||||
|
- Performance: improve treenode inefficiencies [@shamoon](https://github.com/shamoon) ([#11606](https://github.com/paperless-ngx/paperless-ngx/pull/11606))
|
||||||
|
- Feature: password removal action [@shamoon](https://github.com/shamoon) ([#11656](https://github.com/paperless-ngx/paperless-ngx/pull/11656))
|
||||||
|
- Tweakhancement: use anchor element for management list quick filter buttons [@shamoon](https://github.com/shamoon) ([#11692](https://github.com/paperless-ngx/paperless-ngx/pull/11692))
|
||||||
|
- Feature: Indonesian translation [@shamoon](https://github.com/shamoon) ([#11641](https://github.com/paperless-ngx/paperless-ngx/pull/11641))
|
||||||
|
|
||||||
|
### Bug Fixes
|
||||||
|
|
||||||
|
- Fix (beta): prevent pdfjs offsetParent warning [@shamoon](https://github.com/shamoon) ([#13197](https://github.com/paperless-ngx/paperless-ngx/pull/13197))
|
||||||
|
- Fix (beta): dont include hidden buttons in virtual scroll container height [@shamoon](https://github.com/shamoon) ([#13194](https://github.com/paperless-ngx/paperless-ngx/pull/13194))
|
||||||
|
- Fix (beta): fix custom field bulk editing for 'all' [@shamoon](https://github.com/shamoon) ([#13191](https://github.com/paperless-ngx/paperless-ngx/pull/13191))
|
||||||
|
- Security (beta): two small permission check fixes [@shamoon](https://github.com/shamoon) ([#13186](https://github.com/paperless-ngx/paperless-ngx/pull/13186))
|
||||||
|
- Fix (beta): short-circuit ObjectOwnedOrGrantedPermissionsFilter for superusers [@stumpylog](https://github.com/stumpylog) ([#13183](https://github.com/paperless-ngx/paperless-ngx/pull/13183))
|
||||||
|
- Fix (beta): compute num\_notes via a subquery instead of Count()+distinct [@stumpylog](https://github.com/stumpylog) ([#13182](https://github.com/paperless-ngx/paperless-ngx/pull/13182))
|
||||||
|
- Fix: prevent tag assignment from reverting other pending workflow assignments [@stumpylog](https://github.com/stumpylog) ([#13178](https://github.com/paperless-ngx/paperless-ngx/pull/13178))
|
||||||
|
- Fix(beta): cache per-request effective-document resolution for thumb/metadata/preview [@stumpylog](https://github.com/stumpylog) ([#13166](https://github.com/paperless-ngx/paperless-ngx/pull/13166))
|
||||||
|
- Fix: fallback from archive version for share links [@shamoon](https://github.com/shamoon) ([#13163](https://github.com/paperless-ngx/paperless-ngx/pull/13163))
|
||||||
|
- Fix: better handle saved view errors in dashboard widgets [@shamoon](https://github.com/shamoon) ([#13164](https://github.com/paperless-ngx/paperless-ngx/pull/13164))
|
||||||
|
- Fix (beta): convert chat component to signal-backed Angular [@shamoon](https://github.com/shamoon) ([#13152](https://github.com/paperless-ngx/paperless-ngx/pull/13152))
|
||||||
|
- Fix: HTTP 500 filtering documents by tags + custom field query on MariaDB [@stumpylog](https://github.com/stumpylog) ([#13143](https://github.com/paperless-ngx/paperless-ngx/pull/13143))
|
||||||
|
- Fix: scope ProcessedMail UID deduplication by IMAP UIDVALIDITY [@chouquette](https://github.com/chouquette) ([#12839](https://github.com/paperless-ngx/paperless-ngx/pull/12839))
|
||||||
|
- Fix: allow setting any UID and GID when running rootless [@Tahvok](https://github.com/Tahvok) ([#13090](https://github.com/paperless-ngx/paperless-ngx/pull/13090))
|
||||||
|
- Fix (beta): fix sidebar document close button [@shamoon](https://github.com/shamoon) ([#13136](https://github.com/paperless-ngx/paperless-ngx/pull/13136))
|
||||||
|
- Fix: include wasm assets in pdf.js-based viewer [@shamoon](https://github.com/shamoon) ([#13122](https://github.com/paperless-ngx/paperless-ngx/pull/13122))
|
||||||
|
- Fix: fix ui labels from md5 to sha256 [@shamoon](https://github.com/shamoon) ([#13119](https://github.com/paperless-ngx/paperless-ngx/pull/13119))
|
||||||
|
- Fixhancement: dont assign empty title in workflow with broken template [@shamoon](https://github.com/shamoon) ([#13112](https://github.com/paperless-ngx/paperless-ngx/pull/13112))
|
||||||
|
- Fix: clamp mailrule max age before migration [@shamoon](https://github.com/shamoon) ([#13093](https://github.com/paperless-ngx/paperless-ngx/pull/13093))
|
||||||
|
- Fix: use latest document version for bulk download [@shamoon](https://github.com/shamoon) ([#13103](https://github.com/paperless-ngx/paperless-ngx/pull/13103))
|
||||||
|
- Fix (beta): handle relative date with comma query [@shamoon](https://github.com/shamoon) ([#13087](https://github.com/paperless-ngx/paperless-ngx/pull/13087))
|
||||||
|
- Fix (beta): fix sanity checker affected document counts [@NgoQuocViet2001](https://github.com/NgoQuocViet2001) ([#13052](https://github.com/paperless-ngx/paperless-ngx/pull/13052))
|
||||||
|
- Fix (beta): preserve Unicode in localization prompt [@shamoon](https://github.com/shamoon) ([#13055](https://github.com/paperless-ngx/paperless-ngx/pull/13055))
|
||||||
|
- Fix: change tantivy token limit to title field max length [@shamoon](https://github.com/shamoon) ([#13046](https://github.com/paperless-ngx/paperless-ngx/pull/13046))
|
||||||
|
- Fix: include last-modified in doc etag [@shamoon](https://github.com/shamoon) ([#13044](https://github.com/paperless-ngx/paperless-ngx/pull/13044))
|
||||||
|
- Fix (beta): Stream chunks during compaction to prevent oom on smaller installs [@stumpylog](https://github.com/stumpylog) ([#13014](https://github.com/paperless-ngx/paperless-ngx/pull/13014))
|
||||||
|
- Fixhancement (beta): truncate embedding queries for small chunk size [@shamoon](https://github.com/shamoon) ([#13028](https://github.com/paperless-ngx/paperless-ngx/pull/13028))
|
||||||
|
- Fix(beta): Workaround a Tantivy panic in more like this searching [@stumpylog](https://github.com/stumpylog) ([#13026](https://github.com/paperless-ngx/paperless-ngx/pull/13026))
|
||||||
|
- Fix (beta): Catch consumer files created during watcher re-creations [@stumpylog](https://github.com/stumpylog) ([#13013](https://github.com/paperless-ngx/paperless-ngx/pull/13013))
|
||||||
|
- Fix (beta): restore v2 (Whoosh) advanced-search query compatibility [@stumpylog](https://github.com/stumpylog) ([#13010](https://github.com/paperless-ngx/paperless-ngx/pull/13010))
|
||||||
|
- Fix (beta): retry celery ping and report warning on no response [@shamoon](https://github.com/shamoon) ([#13012](https://github.com/paperless-ngx/paperless-ngx/pull/13012))
|
||||||
|
- Fix (beta): don't send chat message on Enter while composing with IME (CJK) [@mzyy94](https://github.com/mzyy94) ([#12999](https://github.com/paperless-ngx/paperless-ngx/pull/12999))
|
||||||
|
- Fix: Apply unicode normalization to all paths and path components [@stumpylog](https://github.com/stumpylog) ([#12993](https://github.com/paperless-ngx/paperless-ngx/pull/12993))
|
||||||
|
- Fix: Handle the UTF 16 and BOM text files better [@stumpylog](https://github.com/stumpylog) ([#12994](https://github.com/paperless-ngx/paperless-ngx/pull/12994))
|
||||||
|
- Fix (beta): avoid DRF update calling `save` on all fields [@shamoon](https://github.com/shamoon) ([#12992](https://github.com/paperless-ngx/paperless-ngx/pull/12992))
|
||||||
|
- Fix: release pooled DB connection during AI LLM/embedding calls [@stumpylog](https://github.com/stumpylog) ([#12983](https://github.com/paperless-ngx/paperless-ngx/pull/12983))
|
||||||
|
- Fix: health-check pooled DB connections and close the pool on worker shutdown [@stumpylog](https://github.com/stumpylog) ([#12977](https://github.com/paperless-ngx/paperless-ngx/pull/12977))
|
||||||
|
- Fix: wrap long titles in delete confirm dialog [@shamoon](https://github.com/shamoon) ([#12973](https://github.com/paperless-ngx/paperless-ngx/pull/12973))
|
||||||
|
- Fix (beta): fix re-ordering in merge dialog [@shamoon](https://github.com/shamoon) ([#12967](https://github.com/paperless-ngx/paperless-ngx/pull/12967))
|
||||||
|
- Fix (beta): move task filtering to backend fully [@shamoon](https://github.com/shamoon) ([#12956](https://github.com/paperless-ngx/paperless-ngx/pull/12956))
|
||||||
|
- Fix (beta): avoid unnecessary creating new PDF with pw removal workflow [@shamoon](https://github.com/shamoon) ([#12948](https://github.com/paperless-ngx/paperless-ngx/pull/12948))
|
||||||
|
- Fix (beta): correct chat message bg color [@shamoon](https://github.com/shamoon) ([#12955](https://github.com/paperless-ngx/paperless-ngx/pull/12955))
|
||||||
|
- Fix (beta): respect disable state for suggest endpoint, require change perms [@shamoon](https://github.com/shamoon) ([#12942](https://github.com/paperless-ngx/paperless-ngx/pull/12942))
|
||||||
|
- Fix: Ensure checksum comparison is using SHA256 [@stumpylog](https://github.com/stumpylog) ([#12939](https://github.com/paperless-ngx/paperless-ngx/pull/12939))
|
||||||
|
- Fix: Restrict the AI backend at settings configure time [@stumpylog](https://github.com/stumpylog) ([#12903](https://github.com/paperless-ngx/paperless-ngx/pull/12903))
|
||||||
|
- Fix (beta): correctly apply i18n in suggestions dropdown [@shamoon](https://github.com/shamoon) ([#12905](https://github.com/paperless-ngx/paperless-ngx/pull/12905))
|
||||||
|
- Fix: Lock AI index during reading and don't index documents many times during a bulk update [@stumpylog](https://github.com/stumpylog) ([#12899](https://github.com/paperless-ngx/paperless-ngx/pull/12899))
|
||||||
|
- Fix: Minor fixes for the AI indexing [@stumpylog](https://github.com/stumpylog) ([#12893](https://github.com/paperless-ngx/paperless-ngx/pull/12893))
|
||||||
|
- Fix (beta): dont use tool calling with ollama [@shamoon](https://github.com/shamoon) ([#12896](https://github.com/paperless-ngx/paperless-ngx/pull/12896))
|
||||||
|
- Fix: Improvements for security around the AI [@stumpylog](https://github.com/stumpylog) ([#12895](https://github.com/paperless-ngx/paperless-ngx/pull/12895))
|
||||||
|
- Fix: correctly show timestamp tooltip on history item [@nathanaelhoun](https://github.com/nathanaelhoun) ([#12879](https://github.com/paperless-ngx/paperless-ngx/pull/12879))
|
||||||
|
- Fix: Fold query and autocomplete terms with Tantivy's ascii\_fold so special letters match [@stumpylog](https://github.com/stumpylog) ([#12868](https://github.com/paperless-ngx/paperless-ngx/pull/12868))
|
||||||
|
- Fix: Don't store autocomplete\_word, only index it [@stumpylog](https://github.com/stumpylog) ([#12867](https://github.com/paperless-ngx/paperless-ngx/pull/12867))
|
||||||
|
- Fix: Preserve Whoosh date range swapping in Tantviy [@stumpylog](https://github.com/stumpylog) ([#12866](https://github.com/paperless-ngx/paperless-ngx/pull/12866))
|
||||||
|
- Fix: Always release search index writer [@stumpylog](https://github.com/stumpylog) ([#12865](https://github.com/paperless-ngx/paperless-ngx/pull/12865))
|
||||||
|
- Fix: Handle CJK title, content and metadata searching [@stumpylog](https://github.com/stumpylog) ([#12862](https://github.com/paperless-ngx/paperless-ngx/pull/12862))
|
||||||
|
- Fix: Restrict date query rewrites to date or datetime fields only [@stumpylog](https://github.com/stumpylog) ([#12864](https://github.com/paperless-ngx/paperless-ngx/pull/12864))
|
||||||
|
- Fix: Missing call to tanvity wait\_merging\_threads [@stumpylog](https://github.com/stumpylog) ([#12863](https://github.com/paperless-ngx/paperless-ngx/pull/12863))
|
||||||
|
- Fix: Handle tanvity index lock contention [@stumpylog](https://github.com/stumpylog) ([#12856](https://github.com/paperless-ngx/paperless-ngx/pull/12856))
|
||||||
|
- Fix (beta): better catch chat errors [@shamoon](https://github.com/shamoon) ([#12854](https://github.com/paperless-ngx/paperless-ngx/pull/12854))
|
||||||
|
- Security (beta): fixes for v3 beta [@shamoon](https://github.com/shamoon) ([#12838](https://github.com/paperless-ngx/paperless-ngx/pull/12838))
|
||||||
|
- Fix (beta): normalize long punctuation chunks to improve embedding [@shamoon](https://github.com/shamoon) ([#12848](https://github.com/paperless-ngx/paperless-ngx/pull/12848))
|
||||||
|
- Fix: use chord.on\_error before apply\_async [@shamoon](https://github.com/shamoon) ([#12842](https://github.com/paperless-ngx/paperless-ngx/pull/12842))
|
||||||
|
- Fix: Don't use smaller integer fields for some workflow fields [@stumpylog](https://github.com/stumpylog) ([#12834](https://github.com/paperless-ngx/paperless-ngx/pull/12834))
|
||||||
|
- Fix: Password removal source file location [@stumpylog](https://github.com/stumpylog) ([#12830](https://github.com/paperless-ngx/paperless-ngx/pull/12830))
|
||||||
|
- Fix (beta): use correct html button type for custom field buttons [@shamoon](https://github.com/shamoon) ([#12819](https://github.com/paperless-ngx/paperless-ngx/pull/12819))
|
||||||
|
- Fix: Defer password removal workflow action until the file is in place [@stumpylog](https://github.com/stumpylog) ([#12814](https://github.com/paperless-ngx/paperless-ngx/pull/12814))
|
||||||
|
- Fix: Allow setting allauth rate limit configuration settings [@stumpylog](https://github.com/stumpylog) ([#12798](https://github.com/paperless-ngx/paperless-ngx/pull/12798))
|
||||||
|
- Fix: Don't embed the metadata which is already embedded into the context [@stumpylog](https://github.com/stumpylog) ([#12795](https://github.com/paperless-ngx/paperless-ngx/pull/12795))
|
||||||
|
- Fix: Sanitize dash or plus from the text search path [@stumpylog](https://github.com/stumpylog) ([#12789](https://github.com/paperless-ngx/paperless-ngx/pull/12789))
|
||||||
|
- Fix: improve new tasks ui layout across screen sizes [@shamoon](https://github.com/shamoon) ([#12784](https://github.com/paperless-ngx/paperless-ngx/pull/12784))
|
||||||
|
- Fix: Update parser contract to require empty strings, not None [@shamoon](https://github.com/shamoon) ([#12775](https://github.com/paperless-ngx/paperless-ngx/pull/12775))
|
||||||
|
- Fix: Use a persistent, writeable location for hugging face models [@stumpylog](https://github.com/stumpylog) ([#12771](https://github.com/paperless-ngx/paperless-ngx/pull/12771))
|
||||||
|
- Fix: create LLM\_INDEX\_DIR before writing meta.json on first run [@pReya](https://github.com/pReya) ([#12759](https://github.com/paperless-ngx/paperless-ngx/pull/12759))
|
||||||
|
- Fix: use response synthesizer for RAG doc chat [@shamoon](https://github.com/shamoon) ([#12751](https://github.com/paperless-ngx/paperless-ngx/pull/12751))
|
||||||
|
- Fix: only update modified field in notes actions [@shamoon](https://github.com/shamoon) ([#12750](https://github.com/paperless-ngx/paperless-ngx/pull/12750))
|
||||||
|
- Fix: Handle dash or plus operators in search queries [@stumpylog](https://github.com/stumpylog) ([#12734](https://github.com/paperless-ngx/paperless-ngx/pull/12734))
|
||||||
|
- Fixes trash preview when a document has deleted versions [@stumpylog](https://github.com/stumpylog) ([#12742](https://github.com/paperless-ngx/paperless-ngx/pull/12742))
|
||||||
|
- Fix: exclude versions from stats count [@shamoon](https://github.com/shamoon) ([#12738](https://github.com/paperless-ngx/paperless-ngx/pull/12738))
|
||||||
|
- Fixes celery chords by using Redis as our result backend [@stumpylog](https://github.com/stumpylog) ([#12741](https://github.com/paperless-ngx/paperless-ngx/pull/12741))
|
||||||
|
- Fix: bump version.py to 3.0.0 also [@shamoon](https://github.com/shamoon) ([#12736](https://github.com/paperless-ngx/paperless-ngx/pull/12736))
|
||||||
|
- Fix: makes the font cache folder writeable to all users [@stumpylog](https://github.com/stumpylog) ([#12726](https://github.com/paperless-ngx/paperless-ngx/pull/12726))
|
||||||
|
- Fixes Whoosh year only queries to be rewritten to Tantivy date syntax [@stumpylog](https://github.com/stumpylog) ([#12725](https://github.com/paperless-ngx/paperless-ngx/pull/12725))
|
||||||
|
- Fix: pass allow parallel tool calls in LLM client [@shamoon](https://github.com/shamoon) ([#12718](https://github.com/paperless-ngx/paperless-ngx/pull/12718))
|
||||||
|
- Fix: Handle passwords for removal action as lists [@stumpylog](https://github.com/stumpylog) ([#12716](https://github.com/paperless-ngx/paperless-ngx/pull/12716))
|
||||||
|
- Bumps all our versions to 3.0.0 [@stumpylog](https://github.com/stumpylog) ([#12715](https://github.com/paperless-ngx/paperless-ngx/pull/12715))
|
||||||
|
- Fix: avoid unnecessary close\_old\_connections in Celery task dispatch [@stumpylog](https://github.com/stumpylog) ([#12701](https://github.com/paperless-ngx/paperless-ngx/pull/12701))
|
||||||
|
- Fix: apply tag changes directly to document in db [@shamoon](https://github.com/shamoon) ([#12664](https://github.com/paperless-ngx/paperless-ngx/pull/12664))
|
||||||
|
- Fix: Use FileResponse for file API responses [@stumpylog](https://github.com/stumpylog) ([#12638](https://github.com/paperless-ngx/paperless-ngx/pull/12638))
|
||||||
|
- Fix: Changes bare metal webserver to use uvloop [@stumpylog](https://github.com/stumpylog) ([#12626](https://github.com/paperless-ngx/paperless-ngx/pull/12626))
|
||||||
|
- Fixhancement: Exact custom field monetary exact searching [@stumpylog](https://github.com/stumpylog) ([#12592](https://github.com/paperless-ngx/paperless-ngx/pull/12592))
|
||||||
|
- Fix: relative added date check visibility [@shamoon](https://github.com/shamoon) ([#12600](https://github.com/paperless-ngx/paperless-ngx/pull/12600))
|
||||||
|
- Fix: prevent intermediate change event when CustomFieldQueryAtom operator changes type [@ggouzi](https://github.com/ggouzi) ([#12597](https://github.com/paperless-ngx/paperless-ngx/pull/12597))
|
||||||
|
- Fix: reject invalid requests to API notes endpoint [@ggouzi](https://github.com/ggouzi) ([#12582](https://github.com/paperless-ngx/paperless-ngx/pull/12582))
|
||||||
|
- Fix: Respect workflow change\_groups during consumption [@stumpylog](https://github.com/stumpylog) ([#12431](https://github.com/paperless-ngx/paperless-ngx/pull/12431))
|
||||||
|
- Fix: Don't try to update the user when running as non-root [@stumpylog](https://github.com/stumpylog) ([#12365](https://github.com/paperless-ngx/paperless-ngx/pull/12365))
|
||||||
|
- Fix: use maxsplit=1 in Redis URL parsing to handle URLs with multiple colons [@cryptomilk](https://github.com/cryptomilk) ([#12239](https://github.com/paperless-ngx/paperless-ngx/pull/12239))
|
||||||
|
- Fixhancement: show sequential + id version labels, fix padding [@shamoon](https://github.com/shamoon) ([#12196](https://github.com/paperless-ngx/paperless-ngx/pull/12196))
|
||||||
|
- Fix: correct user dropdown button icon styling [@shamoon](https://github.com/shamoon) ([#12092](https://github.com/paperless-ngx/paperless-ngx/pull/12092))
|
||||||
|
- Fix: Handle an OOM kill of celery workers in the task handler [@stumpylog](https://github.com/stumpylog) ([#12040](https://github.com/paperless-ngx/paperless-ngx/pull/12040))
|
||||||
|
- Fix: re-run ASN check after barcode detection [@shamoon](https://github.com/shamoon) ([#11681](https://github.com/paperless-ngx/paperless-ngx/pull/11681))
|
||||||
|
- Fix: prevent infinite loading crash in mail component [@shamoon](https://github.com/shamoon) ([#11978](https://github.com/paperless-ngx/paperless-ngx/pull/11978))
|
||||||
|
- Fix: fix broken docker create\_classifier command in 2.20.6 [@shamoon](https://github.com/shamoon) ([#11965](https://github.com/paperless-ngx/paperless-ngx/pull/11965))
|
||||||
|
- Fix: ensure css color-scheme for dark mode [@shamoon](https://github.com/shamoon) ([#11855](https://github.com/paperless-ngx/paperless-ngx/pull/11855))
|
||||||
|
- Fix: fix tag list horizontal scroll, again [@shamoon](https://github.com/shamoon) ([#11839](https://github.com/paperless-ngx/paperless-ngx/pull/11839))
|
||||||
|
- Fix: ensure horizontal scroll for long tag names in list, wrap tags without parent [@shamoon](https://github.com/shamoon) ([#11811](https://github.com/paperless-ngx/paperless-ngx/pull/11811))
|
||||||
|
- Fix: use explicit order field for workflow actions [@shamoon](https://github.com/shamoon) ([#11781](https://github.com/paperless-ngx/paperless-ngx/pull/11781))
|
||||||
|
- Fix: fix recurring workflow to respect latest run time [@shamoon](https://github.com/shamoon) ([#11735](https://github.com/paperless-ngx/paperless-ngx/pull/11735))
|
||||||
|
- Fixhancement: add error handling and retry when opening index [@shamoon](https://github.com/shamoon) ([#11731](https://github.com/paperless-ngx/paperless-ngx/pull/11731))
|
||||||
|
- Fix: validate cf integer values within PostgreSQL range [@shamoon](https://github.com/shamoon) ([#11666](https://github.com/paperless-ngx/paperless-ngx/pull/11666))
|
||||||
|
- Fix: support ordering by storage path name [@shamoon](https://github.com/shamoon) ([#11661](https://github.com/paperless-ngx/paperless-ngx/pull/11661))
|
||||||
|
- Fix: propagate metadata override created value [@shamoon](https://github.com/shamoon) ([#11659](https://github.com/paperless-ngx/paperless-ngx/pull/11659))
|
||||||
|
- Fix: prevent ASN collisions for merge operations [@shamoon](https://github.com/shamoon) ([#11634](https://github.com/paperless-ngx/paperless-ngx/pull/11634))
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
|
||||||
|
- Documentation (beta): Updates documentation for new v3 features [@stumpylog](https://github.com/stumpylog) ([#13033](https://github.com/paperless-ngx/paperless-ngx/pull/13033))
|
||||||
|
- Documentation: fix PAPERLESS\_DATE\_PARSER\_LANGUAGES formatting [@shamoon](https://github.com/shamoon) ([#12997](https://github.com/paperless-ngx/paperless-ngx/pull/12997))
|
||||||
|
- Documentation: remove duplicate words in three files [@mvanhorn](https://github.com/mvanhorn) ([#12852](https://github.com/paperless-ngx/paperless-ngx/pull/12852))
|
||||||
|
- Documentation: Update v3 migration docs [@shamoon](https://github.com/shamoon) ([#12752](https://github.com/paperless-ngx/paperless-ngx/pull/12752))
|
||||||
|
|
||||||
|
### Maintenance
|
||||||
|
|
||||||
|
- Chore: separate actions from bulk edit endpoint [@shamoon](https://github.com/shamoon) ([#12286](https://github.com/paperless-ngx/paperless-ngx/pull/12286))
|
||||||
|
- Chore: address more zizmor flags [@shamoon](https://github.com/shamoon) ([#12529](https://github.com/paperless-ngx/paperless-ngx/pull/12529))
|
||||||
|
- Chore: Resolves some zizmor reported code scan findings [@stumpylog](https://github.com/stumpylog) ([#12516](https://github.com/paperless-ngx/paperless-ngx/pull/12516))
|
||||||
|
- Chore: Don't upload digets file as a zip [@stumpylog](https://github.com/stumpylog) ([#12264](https://github.com/paperless-ngx/paperless-ngx/pull/12264))
|
||||||
|
- Chore: Switches to use prek in place of pre-commit [@stumpylog](https://github.com/stumpylog) ([#12002](https://github.com/paperless-ngx/paperless-ngx/pull/12002))
|
||||||
|
- Chore: Enable mypy checking in CI [@stumpylog](https://github.com/stumpylog) ([#11991](https://github.com/paperless-ngx/paperless-ngx/pull/11991))
|
||||||
|
- Chore: upgrade to node v24 [@shamoon](https://github.com/shamoon) ([#11747](https://github.com/paperless-ngx/paperless-ngx/pull/11747))
|
||||||
|
- Performance: pre-compress static files on ARM64 [@stumpylog](https://github.com/stumpylog) ([#11721](https://github.com/paperless-ngx/paperless-ngx/pull/11721))
|
||||||
|
|
||||||
|
### Dependencies
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>160 changes</summary>
|
||||||
|
|
||||||
|
- Chore(deps): Bump the pre-commit-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13137](https://github.com/paperless-ngx/paperless-ngx/pull/13137))
|
||||||
|
- Chore(deps-dev): Bump the development group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13068](https://github.com/paperless-ngx/paperless-ngx/pull/13068))
|
||||||
|
- docker-compose(deps): bump greenmail/standalone from 2.1.8 to 2.1.9 in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#13067](https://github.com/paperless-ngx/paperless-ngx/pull/13067))
|
||||||
|
- docker-compose(deps): bump gotenberg/gotenberg from 8.33 to 8.34 in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#13066](https://github.com/paperless-ngx/paperless-ngx/pull/13066))
|
||||||
|
- docker-compose(deps): bump nginx from 1.31.1-alpine to 1.31.2-alpine in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#13065](https://github.com/paperless-ngx/paperless-ngx/pull/13065))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.11.19-python3.12-trixie-slim to 0.11.28-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#13070](https://github.com/paperless-ngx/paperless-ngx/pull/13070))
|
||||||
|
- Chore(deps): Bump the actions group across 1 directory with 14 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13079](https://github.com/paperless-ngx/paperless-ngx/pull/13079))
|
||||||
|
- Chore(deps): Bump pdfjs-dist from 5.7.284 to 6.0.227 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13080](https://github.com/paperless-ngx/paperless-ngx/pull/13080))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.60.0 to 1.61.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13078](https://github.com/paperless-ngx/paperless-ngx/pull/13078))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13076](https://github.com/paperless-ngx/paperless-ngx/pull/13076))
|
||||||
|
- Chore(deps): Bump uuid from 14.0.0 to 14.0.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13077](https://github.com/paperless-ngx/paperless-ngx/pull/13077))
|
||||||
|
- Chore(deps-dev): Bump jest-preset-angular from 16.1.5 to 17.0.0 in /src-ui in the frontend-jest-dependencies group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#13075](https://github.com/paperless-ngx/paperless-ngx/pull/13075))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.9.1 to 26.0.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13081](https://github.com/paperless-ngx/paperless-ngx/pull/13081))
|
||||||
|
- Chore(deps): Bump the npm\_and\_yarn group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13016](https://github.com/paperless-ngx/paperless-ngx/pull/13016))
|
||||||
|
- Chore(deps): Bump the uv group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12995](https://github.com/paperless-ngx/paperless-ngx/pull/12995))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12931](https://github.com/paperless-ngx/paperless-ngx/pull/12931))
|
||||||
|
- Chore(deps-dev): Bump types-markdown from 3.10.2.20260211 to 3.10.2.20260518 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12927](https://github.com/paperless-ngx/paperless-ngx/pull/12927))
|
||||||
|
- Chore(deps): Bump aiohttp from 3.13.4 to 3.14.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12930](https://github.com/paperless-ngx/paperless-ngx/pull/12930))
|
||||||
|
- Chore(deps): Bump the pre-commit-dependencies group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12923](https://github.com/paperless-ngx/paperless-ngx/pull/12923))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.11.6-python3.12-trixie-slim to 0.11.19-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#12920](https://github.com/paperless-ngx/paperless-ngx/pull/12920))
|
||||||
|
- Chore(deps): Bump the actions group across 1 directory with 12 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12909](https://github.com/paperless-ngx/paperless-ngx/pull/12909))
|
||||||
|
- docker-compose(deps): bump apache/tika from 3.2.3.0 to 3.3.1.0 in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#12912](https://github.com/paperless-ngx/paperless-ngx/pull/12912))
|
||||||
|
- docker-compose(deps): bump gotenberg/gotenberg from 8.27 to 8.33 in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#12910](https://github.com/paperless-ngx/paperless-ngx/pull/12910))
|
||||||
|
- docker-compose(deps): Bump nginx from 1.29.5-alpine to 1.31.1-alpine in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#12911](https://github.com/paperless-ngx/paperless-ngx/pull/12911))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.59.1 to 1.60.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12919](https://github.com/paperless-ngx/paperless-ngx/pull/12919))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.6.0 to 25.9.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12915](https://github.com/paperless-ngx/paperless-ngx/pull/12915))
|
||||||
|
- Chore(deps): Bump pdfjs-dist from 5.6.205 to 5.7.284 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12918](https://github.com/paperless-ngx/paperless-ngx/pull/12918))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12913](https://github.com/paperless-ngx/paperless-ngx/pull/12913))
|
||||||
|
- Chore(deps-dev): Bump webpack from 5.106.2 to 5.107.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12917](https://github.com/paperless-ngx/paperless-ngx/pull/12917))
|
||||||
|
- Chore(deps): Bump zone.js from 0.16.1 to 0.16.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12916](https://github.com/paperless-ngx/paperless-ngx/pull/12916))
|
||||||
|
- Chore(deps-dev): Bump the frontend-jest-dependencies group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12908](https://github.com/paperless-ngx/paperless-ngx/pull/12908))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group in /src-ui with 20 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12907](https://github.com/paperless-ngx/paperless-ngx/pull/12907))
|
||||||
|
- Chore(deps): Bump hono from 4.12.16 to 4.12.18 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12767](https://github.com/paperless-ngx/paperless-ngx/pull/12767))
|
||||||
|
- Chore(deps): Bump @babel/plugin-transform-modules-systemjs from 7.29.0 to 7.29.4 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12764](https://github.com/paperless-ngx/paperless-ngx/pull/12764))
|
||||||
|
- Chore(deps): Bump fast-uri from 3.1.1 to 3.1.2 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12763](https://github.com/paperless-ngx/paperless-ngx/pull/12763))
|
||||||
|
- Chore: Targeted dependency upgrades [@stumpylog](https://github.com/stumpylog) ([#12731](https://github.com/paperless-ngx/paperless-ngx/pull/12731))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.59.0 to 1.59.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12692](https://github.com/paperless-ngx/paperless-ngx/pull/12692))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12689](https://github.com/paperless-ngx/paperless-ngx/pull/12689))
|
||||||
|
- Chore(deps-dev): Bump @codecov/webpack-plugin from 1.9.1 to 2.0.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12691](https://github.com/paperless-ngx/paperless-ngx/pull/12691))
|
||||||
|
- Chore(deps-dev): Bump the frontend-jest-dependencies group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12685](https://github.com/paperless-ngx/paperless-ngx/pull/12685))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.5.0 to 25.6.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12690](https://github.com/paperless-ngx/paperless-ngx/pull/12690))
|
||||||
|
- Chore(deps-dev): Bump webpack from 5.105.3 to 5.106.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12693](https://github.com/paperless-ngx/paperless-ngx/pull/12693))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group across 1 directory with 15 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12684](https://github.com/paperless-ngx/paperless-ngx/pull/12684))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 7 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12702](https://github.com/paperless-ngx/paperless-ngx/pull/12702))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 9 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12696](https://github.com/paperless-ngx/paperless-ngx/pull/12696))
|
||||||
|
- Chore(deps): Bump the actions group across 1 directory with 17 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12686](https://github.com/paperless-ngx/paperless-ngx/pull/12686))
|
||||||
|
- Chore(deps): Bump ocrmypdf from 17.4.0 to 17.4.2 in the document-processing group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12687](https://github.com/paperless-ngx/paperless-ngx/pull/12687))
|
||||||
|
- Chore(deps-dev): Bump the development group with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12683](https://github.com/paperless-ngx/paperless-ngx/pull/12683))
|
||||||
|
- Chore(deps): Bump the pre-commit-dependencies group with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12694](https://github.com/paperless-ngx/paperless-ngx/pull/12694))
|
||||||
|
- Chore(deps): Bump uuid from 13.0.0 to 14.0.0 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12627](https://github.com/paperless-ngx/paperless-ngx/pull/12627))
|
||||||
|
- Chore(deps): Bump lxml from 6.0.2 to 6.1.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12619](https://github.com/paperless-ngx/paperless-ngx/pull/12619))
|
||||||
|
- Chore(deps): Bump hono from 4.12.12 to 4.12.14 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12585](https://github.com/paperless-ngx/paperless-ngx/pull/12585))
|
||||||
|
- Chore(deps): Bump follow-redirects from 1.15.11 to 1.16.0 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12575](https://github.com/paperless-ngx/paperless-ngx/pull/12575))
|
||||||
|
- Chore(deps): Bump pillow from 12.1.1 to 12.2.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12574](https://github.com/paperless-ngx/paperless-ngx/pull/12574))
|
||||||
|
- Chore(deps-dev): Bump pytest from 9.0.2 to 9.0.3 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12568](https://github.com/paperless-ngx/paperless-ngx/pull/12568))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 19 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12540](https://github.com/paperless-ngx/paperless-ngx/pull/12540))
|
||||||
|
- Chore(deps): Bump cryptography from 46.0.6 to 46.0.7 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12546](https://github.com/paperless-ngx/paperless-ngx/pull/12546))
|
||||||
|
- Chore(deps-dev): Bump types-python-dateutil from 2.9.0.20260305 to 2.9.0.20260323 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12542](https://github.com/paperless-ngx/paperless-ngx/pull/12542))
|
||||||
|
- Chore(deps-dev): Bump types-pytz from 2025.2.0.20251108 to 2026.1.1.20260304 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12541](https://github.com/paperless-ngx/paperless-ngx/pull/12541))
|
||||||
|
- Chore(deps): Bump django-guardian from 3.3.0 to 3.3.1 in the utilities-patch group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12539](https://github.com/paperless-ngx/paperless-ngx/pull/12539))
|
||||||
|
- Chore(deps-dev): Bump zensical from 0.0.29 to 0.0.31 in the development group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12532](https://github.com/paperless-ngx/paperless-ngx/pull/12532))
|
||||||
|
- Chore(deps): Bump pdfjs-dist from 5.4.624 to 5.6.205 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12536](https://github.com/paperless-ngx/paperless-ngx/pull/12536))
|
||||||
|
- Chore(deps): Bump the actions group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12538](https://github.com/paperless-ngx/paperless-ngx/pull/12538))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12535](https://github.com/paperless-ngx/paperless-ngx/pull/12535))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group across 1 directory with 5 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12533](https://github.com/paperless-ngx/paperless-ngx/pull/12533))
|
||||||
|
- Chore(deps-dev): Bump jest-preset-angular from 16.1.1 to 16.1.2 in /src-ui in the frontend-jest-dependencies group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12534](https://github.com/paperless-ngx/paperless-ngx/pull/12534))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.58.2 to 1.59.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12537](https://github.com/paperless-ngx/paperless-ngx/pull/12537))
|
||||||
|
- Chore(deps): Bump the npm\_and\_yarn group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12531](https://github.com/paperless-ngx/paperless-ngx/pull/12531))
|
||||||
|
- Chore(deps): Bump djangorestframework from 3.16.1 to 3.17.1 in the django-ecosystem group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12488](https://github.com/paperless-ngx/paperless-ngx/pull/12488))
|
||||||
|
- Chore(deps): Bump the document-processing group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12489](https://github.com/paperless-ngx/paperless-ngx/pull/12489))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 5 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12499](https://github.com/paperless-ngx/paperless-ngx/pull/12499))
|
||||||
|
- Chore(deps): Bump aiohttp from 3.13.3 to 3.13.4 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12486](https://github.com/paperless-ngx/paperless-ngx/pull/12486))
|
||||||
|
- Chore(deps): Bump the pre-commit-dependencies group with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12495](https://github.com/paperless-ngx/paperless-ngx/pull/12495))
|
||||||
|
- Chore(deps): Bump lodash from 4.17.23 to 4.18.1 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12498](https://github.com/paperless-ngx/paperless-ngx/pull/12498))
|
||||||
|
- Chore(deps): Bump the actions group across 1 directory with 9 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12490](https://github.com/paperless-ngx/paperless-ngx/pull/12490))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group across 1 directory with 20 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12491](https://github.com/paperless-ngx/paperless-ngx/pull/12491))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.4.0 to 25.5.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12494](https://github.com/paperless-ngx/paperless-ngx/pull/12494))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group in /src-ui with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12493](https://github.com/paperless-ngx/paperless-ngx/pull/12493))
|
||||||
|
- Chore(deps): Bump cryptography from 46.0.5 to 46.0.6 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12458](https://github.com/paperless-ngx/paperless-ngx/pull/12458))
|
||||||
|
- Chore(deps): Bump requests from 2.32.5 to 2.33.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12441](https://github.com/paperless-ngx/paperless-ngx/pull/12441))
|
||||||
|
- Chore(deps): Bump cbor2 from 5.8.0 to 5.9.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12424](https://github.com/paperless-ngx/paperless-ngx/pull/12424))
|
||||||
|
- Chore(deps): Bump tinytag from 2.2.0 to 2.2.1 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12396](https://github.com/paperless-ngx/paperless-ngx/pull/12396))
|
||||||
|
- Chore(deps): Bump ujson from 5.11.0 to 5.12.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12387](https://github.com/paperless-ngx/paperless-ngx/pull/12387))
|
||||||
|
- Chore(deps): Bump pyasn1 from 0.6.2 to 0.6.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12370](https://github.com/paperless-ngx/paperless-ngx/pull/12370))
|
||||||
|
- Chore(deps): Bump https://github.com/astral-sh/ruff-pre-commit from v0.15.5 to 0.15.6 in the pre-commit-dependencies group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12371](https://github.com/paperless-ngx/paperless-ngx/pull/12371))
|
||||||
|
- Chore(deps): Bump the actions group with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12377](https://github.com/paperless-ngx/paperless-ngx/pull/12377))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12382](https://github.com/paperless-ngx/paperless-ngx/pull/12382))
|
||||||
|
- Chore(deps): Bump openai from 2.24.0 to 2.26.0 in the utilities-minor group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12379](https://github.com/paperless-ngx/paperless-ngx/pull/12379))
|
||||||
|
- Chore(deps-dev): Bump types-python-dateutil from 2.9.0.20260124 to 2.9.0.20260305 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12380](https://github.com/paperless-ngx/paperless-ngx/pull/12380))
|
||||||
|
- Chore(deps): Update django-allauth[mfa,socialaccount] requirement from ~=65.14.0 to ~=65.15.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12381](https://github.com/paperless-ngx/paperless-ngx/pull/12381))
|
||||||
|
- Chore(deps-dev): Bump the frontend-jest-dependencies group in /src-ui with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12374](https://github.com/paperless-ngx/paperless-ngx/pull/12374))
|
||||||
|
- Chore(deps): Bump @ng-select/ng-select from 21.4.1 to 21.5.2 in /src-ui in the frontend-angular-dependencies group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12373](https://github.com/paperless-ngx/paperless-ngx/pull/12373))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.3.3 to 25.4.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12376](https://github.com/paperless-ngx/paperless-ngx/pull/12376))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group in /src-ui with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12375](https://github.com/paperless-ngx/paperless-ngx/pull/12375))
|
||||||
|
- Chore(deps): Bump pyopenssl from 25.3.0 to 26.0.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12363](https://github.com/paperless-ngx/paperless-ngx/pull/12363))
|
||||||
|
- Chore(deps): Bump pyjwt from 2.10.1 to 2.12.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12335](https://github.com/paperless-ngx/paperless-ngx/pull/12335))
|
||||||
|
- Chore: bump Angular dependencies to 21.2.x [@shamoon](https://github.com/shamoon) ([#12338](https://github.com/paperless-ngx/paperless-ngx/pull/12338))
|
||||||
|
- Bump ocrmypdf from 16.13.0 to 17.3.0 in the document-processing group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12267](https://github.com/paperless-ngx/paperless-ngx/pull/12267))
|
||||||
|
- Chore(deps): Bump tornado from 6.5.4 to 6.5.5 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12327](https://github.com/paperless-ngx/paperless-ngx/pull/12327))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 5 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12324](https://github.com/paperless-ngx/paperless-ngx/pull/12324))
|
||||||
|
- Chore(deps): Bump the pre-commit-dependencies group with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12323](https://github.com/paperless-ngx/paperless-ngx/pull/12323))
|
||||||
|
- Chore(deps): Bump the actions group with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12322](https://github.com/paperless-ngx/paperless-ngx/pull/12322))
|
||||||
|
- docker(deps): bump astral-sh/uv from 0.10.7-python3.12-trixie-slim to 0.10.9-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#12265](https://github.com/paperless-ngx/paperless-ngx/pull/12265))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 6 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12291](https://github.com/paperless-ngx/paperless-ngx/pull/12291))
|
||||||
|
- Chore: update ESLint to v10 [@shamoon](https://github.com/shamoon) ([#12256](https://github.com/paperless-ngx/paperless-ngx/pull/12256))
|
||||||
|
- Bump django-allauth from 65.14.0 to 65.14.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12253](https://github.com/paperless-ngx/paperless-ngx/pull/12253))
|
||||||
|
- Bump django from 5.2.11 to 5.2.12 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12249](https://github.com/paperless-ngx/paperless-ngx/pull/12249))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.10.5-python3.12-trixie-slim to 0.10.7-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#12191](https://github.com/paperless-ngx/paperless-ngx/pull/12191))
|
||||||
|
- Chore(deps): Bump the actions group across 1 directory with 6 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12224](https://github.com/paperless-ngx/paperless-ngx/pull/12224))
|
||||||
|
- Chore: Updates actions to the most specific version released [@stumpylog](https://github.com/stumpylog) ([#12222](https://github.com/paperless-ngx/paperless-ngx/pull/12222))
|
||||||
|
- Chore(deps): Bump whitenoise from 6.11.0 to 6.12.0 in the django-ecosystem group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12192](https://github.com/paperless-ngx/paperless-ngx/pull/12192))
|
||||||
|
- docker-compose(deps): Bump gotenberg/gotenberg from 8.26 to 8.27 in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#12190](https://github.com/paperless-ngx/paperless-ngx/pull/12190))
|
||||||
|
- Chore: Updates s6-overlay to 3.2.2.0 [@stumpylog](https://github.com/stumpylog) ([#12189](https://github.com/paperless-ngx/paperless-ngx/pull/12189))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 11 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12179](https://github.com/paperless-ngx/paperless-ngx/pull/12179))
|
||||||
|
- Chore(deps): Bump nltk from 3.9.2 to 3.9.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12177](https://github.com/paperless-ngx/paperless-ngx/pull/12177))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 7 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12174](https://github.com/paperless-ngx/paperless-ngx/pull/12174))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.10.0-python3.12-trixie-slim to 0.10.4-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#12125](https://github.com/paperless-ngx/paperless-ngx/pull/12125))
|
||||||
|
- Chore(deps): Bump pillow from 12.1.0 to 12.1.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12064](https://github.com/paperless-ngx/paperless-ngx/pull/12064))
|
||||||
|
- Chore(deps): Bump cryptography from 46.0.3 to 46.0.5 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12060](https://github.com/paperless-ngx/paperless-ngx/pull/12060))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12051](https://github.com/paperless-ngx/paperless-ngx/pull/12051))
|
||||||
|
- Chore(deps): Bump j178/prek-action from 1.1.0 to 1.1.1 in the actions group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12033](https://github.com/paperless-ngx/paperless-ngx/pull/12033))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.9.29-python3.12-trixie-slim to 0.10.0-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#12019](https://github.com/paperless-ngx/paperless-ngx/pull/12019))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.58.1 to 1.58.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12032](https://github.com/paperless-ngx/paperless-ngx/pull/12032))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12020](https://github.com/paperless-ngx/paperless-ngx/pull/12020))
|
||||||
|
- docker-compose(deps): Bump nginx from 1.29-alpine to 1.29.5-alpine in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#12018](https://github.com/paperless-ngx/paperless-ngx/pull/12018))
|
||||||
|
- Upgrade: Bump @types/node from 25.2.0 to 25.2.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12008](https://github.com/paperless-ngx/paperless-ngx/pull/12008))
|
||||||
|
- Upgrade: Bump webpack from 5.103.0 to 5.105.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12007](https://github.com/paperless-ngx/paperless-ngx/pull/12007))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.9.26-python3.12-trixie-slim to 0.9.28-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#11980](https://github.com/paperless-ngx/paperless-ngx/pull/11980))
|
||||||
|
- docker-compose(deps): Bump gotenberg/gotenberg from 8.25 to 8.26 in /docker/compose @[dependabot[bot]](https://github.com/apps/dependabot) ([#11979](https://github.com/paperless-ngx/paperless-ngx/pull/11979))
|
||||||
|
- Chore(deps): Bump the utilities-patch group with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11981](https://github.com/paperless-ngx/paperless-ngx/pull/11981))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 6 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11993](https://github.com/paperless-ngx/paperless-ngx/pull/11993))
|
||||||
|
- Chore(deps): Bump drf-spectacular-sidecar from 2025.10.1 to 2026.1.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11985](https://github.com/paperless-ngx/paperless-ngx/pull/11985))
|
||||||
|
- Chore(deps): Update django-allauth[mfa,socialaccount] requirement from ~=65.13.1 to ~=65.14.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11984](https://github.com/paperless-ngx/paperless-ngx/pull/11984))
|
||||||
|
- Chore(deps): Update granian[uvloop] requirement from ~=2.6.0 to ~=2.7.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11983](https://github.com/paperless-ngx/paperless-ngx/pull/11983))
|
||||||
|
- Chore(deps): Bump django from 5.2.10 to 5.2.11 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11988](https://github.com/paperless-ngx/paperless-ngx/pull/11988))
|
||||||
|
- Chore(deps): Bump the actions group with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11966](https://github.com/paperless-ngx/paperless-ngx/pull/11966))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group in /src-ui with 22 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11968](https://github.com/paperless-ngx/paperless-ngx/pull/11968))
|
||||||
|
- Chore(deps): Bump zone.js from 0.15.1 to 0.16.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#11970](https://github.com/paperless-ngx/paperless-ngx/pull/11970))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group in /src-ui with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11969](https://github.com/paperless-ngx/paperless-ngx/pull/11969))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 24.10.1 to 25.2.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#11972](https://github.com/paperless-ngx/paperless-ngx/pull/11972))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.57.0 to 1.58.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#11971](https://github.com/paperless-ngx/paperless-ngx/pull/11971))
|
||||||
|
- Chore: Bulk backend updates [@stumpylog](https://github.com/stumpylog) ([#11543](https://github.com/paperless-ngx/paperless-ngx/pull/11543))
|
||||||
|
- Chore: Manually upgrades allauth to resolve a security issue with it [@stumpylog](https://github.com/stumpylog) ([#11853](https://github.com/paperless-ngx/paperless-ngx/pull/11853))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 7 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11793](https://github.com/paperless-ngx/paperless-ngx/pull/11793))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 10 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11799](https://github.com/paperless-ngx/paperless-ngx/pull/11799))
|
||||||
|
- Chore(deps): Bump pyasn1 from 0.6.1 to 0.6.2 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11801](https://github.com/paperless-ngx/paperless-ngx/pull/11801))
|
||||||
|
- Chore(deps): Bump torch from 2.7.1 to 2.8.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11800](https://github.com/paperless-ngx/paperless-ngx/pull/11800))
|
||||||
|
- docker(deps): Bump astral-sh/uv from 0.9.15-python3.12-trixie-slim to 0.9.24-python3.12-trixie-slim @[dependabot[bot]](https://github.com/apps/dependabot) ([#11762](https://github.com/paperless-ngx/paperless-ngx/pull/11762))
|
||||||
|
- Chore(deps): Bump brotli from 1.1.0 to 1.2.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11796](https://github.com/paperless-ngx/paperless-ngx/pull/11796))
|
||||||
|
- Chore(deps): Bump transformers from 4.51.3 to 4.53.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11797](https://github.com/paperless-ngx/paperless-ngx/pull/11797))
|
||||||
|
- Chore(deps): Bump django from 5.2.7 to 5.2.9 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11794](https://github.com/paperless-ngx/paperless-ngx/pull/11794))
|
||||||
|
- Chore(deps): Bump the llama-index group @[dependabot[bot]](https://github.com/apps/dependabot) ([#11798](https://github.com/paperless-ngx/paperless-ngx/pull/11798))
|
||||||
|
- Chore(deps): Bump marshmallow from 3.26.1 to 3.26.2 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11790](https://github.com/paperless-ngx/paperless-ngx/pull/11790))
|
||||||
|
- Chore(deps): Bump uv from 0.9.3 to 0.9.6 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11795](https://github.com/paperless-ngx/paperless-ngx/pull/11795))
|
||||||
|
- Chore(deps): Bump aiohttp from 3.11.18 to 3.13.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11789](https://github.com/paperless-ngx/paperless-ngx/pull/11789))
|
||||||
|
- Chore(deps): Bump urllib3 from 2.5.0 to 2.6.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11792](https://github.com/paperless-ngx/paperless-ngx/pull/11792))
|
||||||
|
- Chore(deps): Bump virtualenv from 20.34.0 to 20.36.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11774](https://github.com/paperless-ngx/paperless-ngx/pull/11774))
|
||||||
|
- Chore(deps): Bump azure-core from 1.33.0 to 1.38.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11776](https://github.com/paperless-ngx/paperless-ngx/pull/11776))
|
||||||
|
- Chore: upgrade to Angular v21 [@shamoon](https://github.com/shamoon) ([#11746](https://github.com/paperless-ngx/paperless-ngx/pull/11746))
|
||||||
|
- Chore(deps): Bump the actions group with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11695](https://github.com/paperless-ngx/paperless-ngx/pull/11695))
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
### All App Changes
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>360 changes</summary>
|
||||||
|
|
||||||
|
- [Beta] Paperless-ngx v3.0.0 Beta [@shamoon](https://github.com/shamoon) ([#12713](https://github.com/paperless-ngx/paperless-ngx/pull/12713))
|
||||||
|
- Chore (beta): mark first batch of thumbnails as priority for LCP warning [@shamoon](https://github.com/shamoon) ([#13196](https://github.com/paperless-ngx/paperless-ngx/pull/13196))
|
||||||
|
- Fix (beta): prevent pdfjs offsetParent warning [@shamoon](https://github.com/shamoon) ([#13197](https://github.com/paperless-ngx/paperless-ngx/pull/13197))
|
||||||
|
- Fix (beta): dont include hidden buttons in virtual scroll container height [@shamoon](https://github.com/shamoon) ([#13194](https://github.com/paperless-ngx/paperless-ngx/pull/13194))
|
||||||
|
- Tweakhancement (beta): allow some bulk operations with all [@shamoon](https://github.com/shamoon) ([#13193](https://github.com/paperless-ngx/paperless-ngx/pull/13193))
|
||||||
|
- Fix (beta): fix custom field bulk editing for 'all' [@shamoon](https://github.com/shamoon) ([#13191](https://github.com/paperless-ngx/paperless-ngx/pull/13191))
|
||||||
|
- Security (beta): enforce current permissions in autocomplete [@shamoon](https://github.com/shamoon) ([#13188](https://github.com/paperless-ngx/paperless-ngx/pull/13188))
|
||||||
|
- Security (beta): bound email linkification [@shamoon](https://github.com/shamoon) ([#13187](https://github.com/paperless-ngx/paperless-ngx/pull/13187))
|
||||||
|
- Security (beta): two small permission check fixes [@shamoon](https://github.com/shamoon) ([#13186](https://github.com/paperless-ngx/paperless-ngx/pull/13186))
|
||||||
|
- Fix (beta): short-circuit ObjectOwnedOrGrantedPermissionsFilter for superusers [@stumpylog](https://github.com/stumpylog) ([#13183](https://github.com/paperless-ngx/paperless-ngx/pull/13183))
|
||||||
|
- Fix (beta): compute num\_notes via a subquery instead of Count()+distinct [@stumpylog](https://github.com/stumpylog) ([#13182](https://github.com/paperless-ngx/paperless-ngx/pull/13182))
|
||||||
|
- Change: update root modified timestamp on version changes [@shamoon](https://github.com/shamoon) ([#13170](https://github.com/paperless-ngx/paperless-ngx/pull/13170))
|
||||||
|
- Fix: prevent tag assignment from reverting other pending workflow assignments [@stumpylog](https://github.com/stumpylog) ([#13178](https://github.com/paperless-ngx/paperless-ngx/pull/13178))
|
||||||
|
- Performance: add DB indexes for common query/sort patterns [@stumpylog](https://github.com/stumpylog) ([#13167](https://github.com/paperless-ngx/paperless-ngx/pull/13167))
|
||||||
|
- Performance: use NgOptimizedImage for thumbnail lazy loading [@shamoon](https://github.com/shamoon) ([#13169](https://github.com/paperless-ngx/paperless-ngx/pull/13169))
|
||||||
|
- Fix(beta): cache per-request effective-document resolution for thumb/metadata/preview [@stumpylog](https://github.com/stumpylog) ([#13166](https://github.com/paperless-ngx/paperless-ngx/pull/13166))
|
||||||
|
- Fix: fallback from archive version for share links [@shamoon](https://github.com/shamoon) ([#13163](https://github.com/paperless-ngx/paperless-ngx/pull/13163))
|
||||||
|
- Fix: better handle saved view errors in dashboard widgets [@shamoon](https://github.com/shamoon) ([#13164](https://github.com/paperless-ngx/paperless-ngx/pull/13164))
|
||||||
|
- Fix (beta): convert chat component to signal-backed Angular [@shamoon](https://github.com/shamoon) ([#13152](https://github.com/paperless-ngx/paperless-ngx/pull/13152))
|
||||||
|
- Fix: HTTP 500 filtering documents by tags + custom field query on MariaDB [@stumpylog](https://github.com/stumpylog) ([#13143](https://github.com/paperless-ngx/paperless-ngx/pull/13143))
|
||||||
|
- Performance: Tantivy indexing optimization [@Merinorus](https://github.com/Merinorus) ([#13053](https://github.com/paperless-ngx/paperless-ngx/pull/13053))
|
||||||
|
- Chore(deps-dev): Bump the development group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13068](https://github.com/paperless-ngx/paperless-ngx/pull/13068))
|
||||||
|
- Fix: scope ProcessedMail UID deduplication by IMAP UIDVALIDITY [@chouquette](https://github.com/chouquette) ([#12839](https://github.com/paperless-ngx/paperless-ngx/pull/12839))
|
||||||
|
- Fix (beta): fix sidebar document close button [@shamoon](https://github.com/shamoon) ([#13136](https://github.com/paperless-ngx/paperless-ngx/pull/13136))
|
||||||
|
- Fix: include wasm assets in pdf.js-based viewer [@shamoon](https://github.com/shamoon) ([#13122](https://github.com/paperless-ngx/paperless-ngx/pull/13122))
|
||||||
|
- Fix: fix ui labels from md5 to sha256 [@shamoon](https://github.com/shamoon) ([#13119](https://github.com/paperless-ngx/paperless-ngx/pull/13119))
|
||||||
|
- Chorehancement: update to Angular v22, 'zoneless' / 'reactive' [@shamoon](https://github.com/shamoon) ([#13114](https://github.com/paperless-ngx/paperless-ngx/pull/13114))
|
||||||
|
- Fixhancement: dont assign empty title in workflow with broken template [@shamoon](https://github.com/shamoon) ([#13112](https://github.com/paperless-ngx/paperless-ngx/pull/13112))
|
||||||
|
- Fix: clamp mailrule max age before migration [@shamoon](https://github.com/shamoon) ([#13093](https://github.com/paperless-ngx/paperless-ngx/pull/13093))
|
||||||
|
- Fix: use latest document version for bulk download [@shamoon](https://github.com/shamoon) ([#13103](https://github.com/paperless-ngx/paperless-ngx/pull/13103))
|
||||||
|
- Fix (beta): handle relative date with comma query [@shamoon](https://github.com/shamoon) ([#13087](https://github.com/paperless-ngx/paperless-ngx/pull/13087))
|
||||||
|
- Chore(deps): Bump pdfjs-dist from 5.7.284 to 6.0.227 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13080](https://github.com/paperless-ngx/paperless-ngx/pull/13080))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.60.0 to 1.61.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13078](https://github.com/paperless-ngx/paperless-ngx/pull/13078))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13076](https://github.com/paperless-ngx/paperless-ngx/pull/13076))
|
||||||
|
- Chore(deps): Bump uuid from 14.0.0 to 14.0.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13077](https://github.com/paperless-ngx/paperless-ngx/pull/13077))
|
||||||
|
- Chore(deps-dev): Bump jest-preset-angular from 16.1.5 to 17.0.0 in /src-ui in the frontend-jest-dependencies group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#13075](https://github.com/paperless-ngx/paperless-ngx/pull/13075))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.9.1 to 26.0.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#13081](https://github.com/paperless-ngx/paperless-ngx/pull/13081))
|
||||||
|
- Fix (beta): fix sanity checker affected document counts [@NgoQuocViet2001](https://github.com/NgoQuocViet2001) ([#13052](https://github.com/paperless-ngx/paperless-ngx/pull/13052))
|
||||||
|
- Fix (beta): preserve Unicode in localization prompt [@shamoon](https://github.com/shamoon) ([#13055](https://github.com/paperless-ngx/paperless-ngx/pull/13055))
|
||||||
|
- Fix: change tantivy token limit to title field max length [@shamoon](https://github.com/shamoon) ([#13046](https://github.com/paperless-ngx/paperless-ngx/pull/13046))
|
||||||
|
- Fix: include last-modified in doc etag [@shamoon](https://github.com/shamoon) ([#13044](https://github.com/paperless-ngx/paperless-ngx/pull/13044))
|
||||||
|
- Fix (beta): Stream chunks during compaction to prevent oom on smaller installs [@stumpylog](https://github.com/stumpylog) ([#13014](https://github.com/paperless-ngx/paperless-ngx/pull/13014))
|
||||||
|
- Fixhancement (beta): truncate embedding queries for small chunk size [@shamoon](https://github.com/shamoon) ([#13028](https://github.com/paperless-ngx/paperless-ngx/pull/13028))
|
||||||
|
- Enhancement (beta): support LLM timeout config [@shamoon](https://github.com/shamoon) ([#13002](https://github.com/paperless-ngx/paperless-ngx/pull/13002))
|
||||||
|
- Fix(beta): Workaround a Tantivy panic in more like this searching [@stumpylog](https://github.com/stumpylog) ([#13026](https://github.com/paperless-ngx/paperless-ngx/pull/13026))
|
||||||
|
- Chore (beta): set tool\_required to openai-like llm calls [@shamoon](https://github.com/shamoon) ([#13025](https://github.com/paperless-ngx/paperless-ngx/pull/13025))
|
||||||
|
- Enhancement: ignore diacritics, support multiple substring matching for UI filtering [@shamoon](https://github.com/shamoon) ([#13021](https://github.com/paperless-ngx/paperless-ngx/pull/13021))
|
||||||
|
- Fix (beta): Catch consumer files created during watcher re-creations [@stumpylog](https://github.com/stumpylog) ([#13013](https://github.com/paperless-ngx/paperless-ngx/pull/13013))
|
||||||
|
- Chore(deps): Bump the npm\_and\_yarn group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#13016](https://github.com/paperless-ngx/paperless-ngx/pull/13016))
|
||||||
|
- Fix (beta): restore v2 (Whoosh) advanced-search query compatibility [@stumpylog](https://github.com/stumpylog) ([#13010](https://github.com/paperless-ngx/paperless-ngx/pull/13010))
|
||||||
|
- Fix (beta): retry celery ping and report warning on no response [@shamoon](https://github.com/shamoon) ([#13012](https://github.com/paperless-ngx/paperless-ngx/pull/13012))
|
||||||
|
- Enhancement(beta): replace LanceDB vector store with sqlite-vec [@stumpylog](https://github.com/stumpylog) ([#12990](https://github.com/paperless-ngx/paperless-ngx/pull/12990))
|
||||||
|
- Chore(deps): Bump the uv group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12995](https://github.com/paperless-ngx/paperless-ngx/pull/12995))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12931](https://github.com/paperless-ngx/paperless-ngx/pull/12931))
|
||||||
|
- Fix (beta): don't send chat message on Enter while composing with IME (CJK) [@mzyy94](https://github.com/mzyy94) ([#12999](https://github.com/paperless-ngx/paperless-ngx/pull/12999))
|
||||||
|
- Fix: Apply unicode normalization to all paths and path components [@stumpylog](https://github.com/stumpylog) ([#12993](https://github.com/paperless-ngx/paperless-ngx/pull/12993))
|
||||||
|
- Fix: Handle the UTF 16 and BOM text files better [@stumpylog](https://github.com/stumpylog) ([#12994](https://github.com/paperless-ngx/paperless-ngx/pull/12994))
|
||||||
|
- Fix (beta): avoid DRF update calling `save` on all fields [@shamoon](https://github.com/shamoon) ([#12992](https://github.com/paperless-ngx/paperless-ngx/pull/12992))
|
||||||
|
- Fix: release pooled DB connection during AI LLM/embedding calls [@stumpylog](https://github.com/stumpylog) ([#12983](https://github.com/paperless-ngx/paperless-ngx/pull/12983))
|
||||||
|
- Fix: health-check pooled DB connections and close the pool on worker shutdown [@stumpylog](https://github.com/stumpylog) ([#12977](https://github.com/paperless-ngx/paperless-ngx/pull/12977))
|
||||||
|
- Fix: wrap long titles in delete confirm dialog [@shamoon](https://github.com/shamoon) ([#12973](https://github.com/paperless-ngx/paperless-ngx/pull/12973))
|
||||||
|
- Fix (beta): fix re-ordering in merge dialog [@shamoon](https://github.com/shamoon) ([#12967](https://github.com/paperless-ngx/paperless-ngx/pull/12967))
|
||||||
|
- Fix (beta): move task filtering to backend fully [@shamoon](https://github.com/shamoon) ([#12956](https://github.com/paperless-ngx/paperless-ngx/pull/12956))
|
||||||
|
- Chore(deps-dev): Bump types-markdown from 3.10.2.20260211 to 3.10.2.20260518 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12927](https://github.com/paperless-ngx/paperless-ngx/pull/12927))
|
||||||
|
- Chore(deps): Bump aiohttp from 3.13.4 to 3.14.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12930](https://github.com/paperless-ngx/paperless-ngx/pull/12930))
|
||||||
|
- Fix (beta): avoid unnecessary creating new PDF with pw removal workflow [@shamoon](https://github.com/shamoon) ([#12948](https://github.com/paperless-ngx/paperless-ngx/pull/12948))
|
||||||
|
- Enhancement (beta): Switch the AI vector store to LanceDB [@stumpylog](https://github.com/stumpylog) ([#12944](https://github.com/paperless-ngx/paperless-ngx/pull/12944))
|
||||||
|
- Fixhancement (beta): tasks dismiss all [@shamoon](https://github.com/shamoon) ([#12949](https://github.com/paperless-ngx/paperless-ngx/pull/12949))
|
||||||
|
- Fix (beta): correct chat message bg color [@shamoon](https://github.com/shamoon) ([#12955](https://github.com/paperless-ngx/paperless-ngx/pull/12955))
|
||||||
|
- Fix (beta): respect disable state for suggest endpoint, require change perms [@shamoon](https://github.com/shamoon) ([#12942](https://github.com/paperless-ngx/paperless-ngx/pull/12942))
|
||||||
|
- Fix: Ensure checksum comparison is using SHA256 [@stumpylog](https://github.com/stumpylog) ([#12939](https://github.com/paperless-ngx/paperless-ngx/pull/12939))
|
||||||
|
- Fix: Restrict the AI backend at settings configure time [@stumpylog](https://github.com/stumpylog) ([#12903](https://github.com/paperless-ngx/paperless-ngx/pull/12903))
|
||||||
|
- Enhancement (beta): add direct LLM language setting [@shamoon](https://github.com/shamoon) ([#12906](https://github.com/paperless-ngx/paperless-ngx/pull/12906))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.59.1 to 1.60.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12919](https://github.com/paperless-ngx/paperless-ngx/pull/12919))
|
||||||
|
- Fix (beta): correctly apply i18n in suggestions dropdown [@shamoon](https://github.com/shamoon) ([#12905](https://github.com/paperless-ngx/paperless-ngx/pull/12905))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.6.0 to 25.9.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12915](https://github.com/paperless-ngx/paperless-ngx/pull/12915))
|
||||||
|
- Chore(deps): Bump pdfjs-dist from 5.6.205 to 5.7.284 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12918](https://github.com/paperless-ngx/paperless-ngx/pull/12918))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12913](https://github.com/paperless-ngx/paperless-ngx/pull/12913))
|
||||||
|
- Chore(deps-dev): Bump webpack from 5.106.2 to 5.107.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12917](https://github.com/paperless-ngx/paperless-ngx/pull/12917))
|
||||||
|
- Chore(deps): Bump zone.js from 0.16.1 to 0.16.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12916](https://github.com/paperless-ngx/paperless-ngx/pull/12916))
|
||||||
|
- Chore(deps-dev): Bump the frontend-jest-dependencies group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12908](https://github.com/paperless-ngx/paperless-ngx/pull/12908))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group in /src-ui with 20 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12907](https://github.com/paperless-ngx/paperless-ngx/pull/12907))
|
||||||
|
- Fix: Lock AI index during reading and don't index documents many times during a bulk update [@stumpylog](https://github.com/stumpylog) ([#12899](https://github.com/paperless-ngx/paperless-ngx/pull/12899))
|
||||||
|
- Fix: Minor fixes for the AI indexing [@stumpylog](https://github.com/stumpylog) ([#12893](https://github.com/paperless-ngx/paperless-ngx/pull/12893))
|
||||||
|
- Fix (beta): dont use tool calling with ollama [@shamoon](https://github.com/shamoon) ([#12896](https://github.com/paperless-ngx/paperless-ngx/pull/12896))
|
||||||
|
- Enhancement: try to respect language for AI suggestions [@shamoon](https://github.com/shamoon) ([#12894](https://github.com/paperless-ngx/paperless-ngx/pull/12894))
|
||||||
|
- Enhancement: AI LLM chunk size and context window config [@shamoon](https://github.com/shamoon) ([#12891](https://github.com/paperless-ngx/paperless-ngx/pull/12891))
|
||||||
|
- Fix: Improvements for security around the AI [@stumpylog](https://github.com/stumpylog) ([#12895](https://github.com/paperless-ngx/paperless-ngx/pull/12895))
|
||||||
|
- Fix: correctly show timestamp tooltip on history item [@nathanaelhoun](https://github.com/nathanaelhoun) ([#12879](https://github.com/paperless-ngx/paperless-ngx/pull/12879))
|
||||||
|
- Fix: Fold query and autocomplete terms with Tantivy's ascii\_fold so special letters match [@stumpylog](https://github.com/stumpylog) ([#12868](https://github.com/paperless-ngx/paperless-ngx/pull/12868))
|
||||||
|
- Fix: Don't store autocomplete\_word, only index it [@stumpylog](https://github.com/stumpylog) ([#12867](https://github.com/paperless-ngx/paperless-ngx/pull/12867))
|
||||||
|
- Fix: Preserve Whoosh date range swapping in Tantviy [@stumpylog](https://github.com/stumpylog) ([#12866](https://github.com/paperless-ngx/paperless-ngx/pull/12866))
|
||||||
|
- Fix: Always release search index writer [@stumpylog](https://github.com/stumpylog) ([#12865](https://github.com/paperless-ngx/paperless-ngx/pull/12865))
|
||||||
|
- Fix: Handle CJK title, content and metadata searching [@stumpylog](https://github.com/stumpylog) ([#12862](https://github.com/paperless-ngx/paperless-ngx/pull/12862))
|
||||||
|
- Fix: Restrict date query rewrites to date or datetime fields only [@stumpylog](https://github.com/stumpylog) ([#12864](https://github.com/paperless-ngx/paperless-ngx/pull/12864))
|
||||||
|
- Fix: Missing call to tanvity wait\_merging\_threads [@stumpylog](https://github.com/stumpylog) ([#12863](https://github.com/paperless-ngx/paperless-ngx/pull/12863))
|
||||||
|
- Fix: Handle tanvity index lock contention [@stumpylog](https://github.com/stumpylog) ([#12856](https://github.com/paperless-ngx/paperless-ngx/pull/12856))
|
||||||
|
- Fix (beta): better catch chat errors [@shamoon](https://github.com/shamoon) ([#12854](https://github.com/paperless-ngx/paperless-ngx/pull/12854))
|
||||||
|
- Performance (beta): dont re-build vector index with each chat [@shamoon](https://github.com/shamoon) ([#12847](https://github.com/paperless-ngx/paperless-ngx/pull/12847))
|
||||||
|
- Security (beta): fixes for v3 beta [@shamoon](https://github.com/shamoon) ([#12838](https://github.com/paperless-ngx/paperless-ngx/pull/12838))
|
||||||
|
- Fix (beta): normalize long punctuation chunks to improve embedding [@shamoon](https://github.com/shamoon) ([#12848](https://github.com/paperless-ngx/paperless-ngx/pull/12848))
|
||||||
|
- Fix: use chord.on\_error before apply\_async [@shamoon](https://github.com/shamoon) ([#12842](https://github.com/paperless-ngx/paperless-ngx/pull/12842))
|
||||||
|
- Fix: Don't use smaller integer fields for some workflow fields [@stumpylog](https://github.com/stumpylog) ([#12834](https://github.com/paperless-ngx/paperless-ngx/pull/12834))
|
||||||
|
- Fix: Password removal source file location [@stumpylog](https://github.com/stumpylog) ([#12830](https://github.com/paperless-ngx/paperless-ngx/pull/12830))
|
||||||
|
- Fix (beta): use correct html button type for custom field buttons [@shamoon](https://github.com/shamoon) ([#12819](https://github.com/paperless-ngx/paperless-ngx/pull/12819))
|
||||||
|
- Fix: Defer password removal workflow action until the file is in place [@stumpylog](https://github.com/stumpylog) ([#12814](https://github.com/paperless-ngx/paperless-ngx/pull/12814))
|
||||||
|
- Fix: Allow setting allauth rate limit configuration settings [@stumpylog](https://github.com/stumpylog) ([#12798](https://github.com/paperless-ngx/paperless-ngx/pull/12798))
|
||||||
|
- Fix: Don't embed the metadata which is already embedded into the context [@stumpylog](https://github.com/stumpylog) ([#12795](https://github.com/paperless-ngx/paperless-ngx/pull/12795))
|
||||||
|
- Fix: Sanitize dash or plus from the text search path [@stumpylog](https://github.com/stumpylog) ([#12789](https://github.com/paperless-ngx/paperless-ngx/pull/12789))
|
||||||
|
- Fix: improve new tasks ui layout across screen sizes [@shamoon](https://github.com/shamoon) ([#12784](https://github.com/paperless-ngx/paperless-ngx/pull/12784))
|
||||||
|
- Fix: Update parser contract to require empty strings, not None [@shamoon](https://github.com/shamoon) ([#12775](https://github.com/paperless-ngx/paperless-ngx/pull/12775))
|
||||||
|
- Chore: Further dependency security updates [@stumpylog](https://github.com/stumpylog) ([#12780](https://github.com/paperless-ngx/paperless-ngx/pull/12780))
|
||||||
|
- Fix: Use a persistent, writeable location for hugging face models [@stumpylog](https://github.com/stumpylog) ([#12771](https://github.com/paperless-ngx/paperless-ngx/pull/12771))
|
||||||
|
- Chore(deps): Bump hono from 4.12.16 to 4.12.18 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12767](https://github.com/paperless-ngx/paperless-ngx/pull/12767))
|
||||||
|
- Enhancement: support ollama embeddings [@shamoon](https://github.com/shamoon) ([#12753](https://github.com/paperless-ngx/paperless-ngx/pull/12753))
|
||||||
|
- Fix: create LLM\_INDEX\_DIR before writing meta.json on first run [@pReya](https://github.com/pReya) ([#12759](https://github.com/paperless-ngx/paperless-ngx/pull/12759))
|
||||||
|
- Chore(deps): Bump @babel/plugin-transform-modules-systemjs from 7.29.0 to 7.29.4 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12764](https://github.com/paperless-ngx/paperless-ngx/pull/12764))
|
||||||
|
- Chore(deps): Bump fast-uri from 3.1.1 to 3.1.2 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12763](https://github.com/paperless-ngx/paperless-ngx/pull/12763))
|
||||||
|
- Tweakhancment: Include the last applied 'documents' migration in the log [@stumpylog](https://github.com/stumpylog) ([#12757](https://github.com/paperless-ngx/paperless-ngx/pull/12757))
|
||||||
|
- Fix: use response synthesizer for RAG doc chat [@shamoon](https://github.com/shamoon) ([#12751](https://github.com/paperless-ngx/paperless-ngx/pull/12751))
|
||||||
|
- Fix: only update modified field in notes actions [@shamoon](https://github.com/shamoon) ([#12750](https://github.com/paperless-ngx/paperless-ngx/pull/12750))
|
||||||
|
- Enhancement: version-aware thumbnail etag [@shamoon](https://github.com/shamoon) ([#12754](https://github.com/paperless-ngx/paperless-ngx/pull/12754))
|
||||||
|
- Documentation: Update v3 migration docs [@shamoon](https://github.com/shamoon) ([#12752](https://github.com/paperless-ngx/paperless-ngx/pull/12752))
|
||||||
|
- Fix: Handle dash or plus operators in search queries [@stumpylog](https://github.com/stumpylog) ([#12734](https://github.com/paperless-ngx/paperless-ngx/pull/12734))
|
||||||
|
- Fixes trash preview when a document has deleted versions [@stumpylog](https://github.com/stumpylog) ([#12742](https://github.com/paperless-ngx/paperless-ngx/pull/12742))
|
||||||
|
- Fix: exclude versions from stats count [@shamoon](https://github.com/shamoon) ([#12738](https://github.com/paperless-ngx/paperless-ngx/pull/12738))
|
||||||
|
- Fixes celery chords by using Redis as our result backend [@stumpylog](https://github.com/stumpylog) ([#12741](https://github.com/paperless-ngx/paperless-ngx/pull/12741))
|
||||||
|
- Fix: bump version.py to 3.0.0 also [@shamoon](https://github.com/shamoon) ([#12736](https://github.com/paperless-ngx/paperless-ngx/pull/12736))
|
||||||
|
- Chore: Targeted dependency upgrades [@stumpylog](https://github.com/stumpylog) ([#12731](https://github.com/paperless-ngx/paperless-ngx/pull/12731))
|
||||||
|
- Fixes Whoosh year only queries to be rewritten to Tantivy date syntax [@stumpylog](https://github.com/stumpylog) ([#12725](https://github.com/paperless-ngx/paperless-ngx/pull/12725))
|
||||||
|
- Fix: pass allow parallel tool calls in LLM client [@shamoon](https://github.com/shamoon) ([#12718](https://github.com/paperless-ngx/paperless-ngx/pull/12718))
|
||||||
|
- Fix: Handle passwords for removal action as lists [@stumpylog](https://github.com/stumpylog) ([#12716](https://github.com/paperless-ngx/paperless-ngx/pull/12716))
|
||||||
|
- Bumps all our versions to 3.0.0 [@stumpylog](https://github.com/stumpylog) ([#12715](https://github.com/paperless-ngx/paperless-ngx/pull/12715))
|
||||||
|
- Feature: Further reduce document importer memory usage [@stumpylog](https://github.com/stumpylog) ([#12707](https://github.com/paperless-ngx/paperless-ngx/pull/12707))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.59.0 to 1.59.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12692](https://github.com/paperless-ngx/paperless-ngx/pull/12692))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12689](https://github.com/paperless-ngx/paperless-ngx/pull/12689))
|
||||||
|
- Chore(deps-dev): Bump @codecov/webpack-plugin from 1.9.1 to 2.0.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12691](https://github.com/paperless-ngx/paperless-ngx/pull/12691))
|
||||||
|
- Chore(deps-dev): Bump the frontend-jest-dependencies group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12685](https://github.com/paperless-ngx/paperless-ngx/pull/12685))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.5.0 to 25.6.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12690](https://github.com/paperless-ngx/paperless-ngx/pull/12690))
|
||||||
|
- Chore(deps-dev): Bump webpack from 5.105.3 to 5.106.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12693](https://github.com/paperless-ngx/paperless-ngx/pull/12693))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group across 1 directory with 15 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12684](https://github.com/paperless-ngx/paperless-ngx/pull/12684))
|
||||||
|
- Tweakhancement: use fixed position instead of display none for printing [@shamoon](https://github.com/shamoon) ([#12706](https://github.com/paperless-ngx/paperless-ngx/pull/12706))
|
||||||
|
- Fix: avoid unnecessary close\_old\_connections in Celery task dispatch [@stumpylog](https://github.com/stumpylog) ([#12701](https://github.com/paperless-ngx/paperless-ngx/pull/12701))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 7 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12702](https://github.com/paperless-ngx/paperless-ngx/pull/12702))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 9 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12696](https://github.com/paperless-ngx/paperless-ngx/pull/12696))
|
||||||
|
- Chore(deps): Bump ocrmypdf from 17.4.0 to 17.4.2 in the document-processing group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12687](https://github.com/paperless-ngx/paperless-ngx/pull/12687))
|
||||||
|
- Chore(deps-dev): Bump the development group with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12683](https://github.com/paperless-ngx/paperless-ngx/pull/12683))
|
||||||
|
- Enhancement: Paperless-ngx v3 Logo [@shamoon](https://github.com/shamoon) ([#12673](https://github.com/paperless-ngx/paperless-ngx/pull/12673))
|
||||||
|
- Tweakhancement: localize some more task result messages [@shamoon](https://github.com/shamoon) ([#12672](https://github.com/paperless-ngx/paperless-ngx/pull/12672))
|
||||||
|
- Enhancement: chat message document links [@shamoon](https://github.com/shamoon) ([#12670](https://github.com/paperless-ngx/paperless-ngx/pull/12670))
|
||||||
|
- Fix: apply tag changes directly to document in db [@shamoon](https://github.com/shamoon) ([#12664](https://github.com/paperless-ngx/paperless-ngx/pull/12664))
|
||||||
|
- Tweakhancement: make upload notification open an anchor link [@shamoon](https://github.com/shamoon) ([#12659](https://github.com/paperless-ngx/paperless-ngx/pull/12659))
|
||||||
|
- Tweakhancement: tweak tasks UI, make open doc an anchor [@shamoon](https://github.com/shamoon) ([#12658](https://github.com/paperless-ngx/paperless-ngx/pull/12658))
|
||||||
|
- Fix: Use FileResponse for file API responses [@stumpylog](https://github.com/stumpylog) ([#12638](https://github.com/paperless-ngx/paperless-ngx/pull/12638))
|
||||||
|
- Chore: Paginate the task listing [@stumpylog](https://github.com/stumpylog) ([#12633](https://github.com/paperless-ngx/paperless-ngx/pull/12633))
|
||||||
|
- Enhancement: show small task summary in system status [@shamoon](https://github.com/shamoon) ([#12634](https://github.com/paperless-ngx/paperless-ngx/pull/12634))
|
||||||
|
- Security: Don't allow the example secret key as a secret key [@stumpylog](https://github.com/stumpylog) ([#12630](https://github.com/paperless-ngx/paperless-ngx/pull/12630))
|
||||||
|
- Chore(deps): Bump uuid from 13.0.0 to 14.0.0 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12627](https://github.com/paperless-ngx/paperless-ngx/pull/12627))
|
||||||
|
- Enhancement: new Tasks UI [@shamoon](https://github.com/shamoon) ([#12614](https://github.com/paperless-ngx/paperless-ngx/pull/12614))
|
||||||
|
- Feature: Allow monitoring access to tasks summary [@stumpylog](https://github.com/stumpylog) ([#12624](https://github.com/paperless-ngx/paperless-ngx/pull/12624))
|
||||||
|
- Chore(deps): Bump lxml from 6.0.2 to 6.1.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12619](https://github.com/paperless-ngx/paperless-ngx/pull/12619))
|
||||||
|
- Performance: Increases workflow related M2M prefetching [@stumpylog](https://github.com/stumpylog) ([#12618](https://github.com/paperless-ngx/paperless-ngx/pull/12618))
|
||||||
|
- Tweak: remove stale index warning [@shamoon](https://github.com/shamoon) ([#12616](https://github.com/paperless-ngx/paperless-ngx/pull/12616))
|
||||||
|
- Fixhancement: Exact custom field monetary exact searching [@stumpylog](https://github.com/stumpylog) ([#12592](https://github.com/paperless-ngx/paperless-ngx/pull/12592))
|
||||||
|
- Performance: Resolves another N+1 query related to custom fields [@stumpylog](https://github.com/stumpylog) ([#12617](https://github.com/paperless-ngx/paperless-ngx/pull/12617))
|
||||||
|
- Enhancement: add highlighting to title + content searches [@shamoon](https://github.com/shamoon) ([#12593](https://github.com/paperless-ngx/paperless-ngx/pull/12593))
|
||||||
|
- Feature: Redesign the task system [@stumpylog](https://github.com/stumpylog) ([#12584](https://github.com/paperless-ngx/paperless-ngx/pull/12584))
|
||||||
|
- Fix: relative added date check visibility [@shamoon](https://github.com/shamoon) ([#12600](https://github.com/paperless-ngx/paperless-ngx/pull/12600))
|
||||||
|
- Fix: prevent intermediate change event when CustomFieldQueryAtom operator changes type [@ggouzi](https://github.com/ggouzi) ([#12597](https://github.com/paperless-ngx/paperless-ngx/pull/12597))
|
||||||
|
- Chore(deps): Bump hono from 4.12.12 to 4.12.14 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12585](https://github.com/paperless-ngx/paperless-ngx/pull/12585))
|
||||||
|
- Feature: Tune SQLite default settings for increased speed [@stumpylog](https://github.com/stumpylog) ([#12580](https://github.com/paperless-ngx/paperless-ngx/pull/12580))
|
||||||
|
- Feature: Document fuzzy match improvements [@stumpylog](https://github.com/stumpylog) ([#12579](https://github.com/paperless-ngx/paperless-ngx/pull/12579))
|
||||||
|
- Fix: reject invalid requests to API notes endpoint [@ggouzi](https://github.com/ggouzi) ([#12582](https://github.com/paperless-ngx/paperless-ngx/pull/12582))
|
||||||
|
- [BREAKING] Remove the positional arguments from the pre/post consume scripts [@stumpylog](https://github.com/stumpylog) ([#12573](https://github.com/paperless-ngx/paperless-ngx/pull/12573))
|
||||||
|
- Chore(deps): Bump follow-redirects from 1.15.11 to 1.16.0 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12575](https://github.com/paperless-ngx/paperless-ngx/pull/12575))
|
||||||
|
- Chore(deps): Bump pillow from 12.1.1 to 12.2.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12574](https://github.com/paperless-ngx/paperless-ngx/pull/12574))
|
||||||
|
- Chore(deps-dev): Bump pytest from 9.0.2 to 9.0.3 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12568](https://github.com/paperless-ngx/paperless-ngx/pull/12568))
|
||||||
|
- Enhancement: validate and sanitize uploaded logos [@shamoon](https://github.com/shamoon) ([#12551](https://github.com/paperless-ngx/paperless-ngx/pull/12551))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 19 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12540](https://github.com/paperless-ngx/paperless-ngx/pull/12540))
|
||||||
|
- Chore(deps): Bump cryptography from 46.0.6 to 46.0.7 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12546](https://github.com/paperless-ngx/paperless-ngx/pull/12546))
|
||||||
|
- Chore(deps-dev): Bump types-python-dateutil from 2.9.0.20260305 to 2.9.0.20260323 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12542](https://github.com/paperless-ngx/paperless-ngx/pull/12542))
|
||||||
|
- Chore(deps-dev): Bump types-pytz from 2025.2.0.20251108 to 2026.1.1.20260304 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12541](https://github.com/paperless-ngx/paperless-ngx/pull/12541))
|
||||||
|
- Chore(deps): Bump django-guardian from 3.3.0 to 3.3.1 in the utilities-patch group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12539](https://github.com/paperless-ngx/paperless-ngx/pull/12539))
|
||||||
|
- Chore(deps-dev): Bump zensical from 0.0.29 to 0.0.31 in the development group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12532](https://github.com/paperless-ngx/paperless-ngx/pull/12532))
|
||||||
|
- Chore(deps): Bump pdfjs-dist from 5.4.624 to 5.6.205 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12536](https://github.com/paperless-ngx/paperless-ngx/pull/12536))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12535](https://github.com/paperless-ngx/paperless-ngx/pull/12535))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group across 1 directory with 5 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12533](https://github.com/paperless-ngx/paperless-ngx/pull/12533))
|
||||||
|
- Chore(deps-dev): Bump jest-preset-angular from 16.1.1 to 16.1.2 in /src-ui in the frontend-jest-dependencies group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12534](https://github.com/paperless-ngx/paperless-ngx/pull/12534))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.58.2 to 1.59.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12537](https://github.com/paperless-ngx/paperless-ngx/pull/12537))
|
||||||
|
- Enhancement: add view\_global\_statistics and view\_system\_status permissions [@shamoon](https://github.com/shamoon) ([#12530](https://github.com/paperless-ngx/paperless-ngx/pull/12530))
|
||||||
|
- Chore(deps): Bump the npm\_and\_yarn group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12531](https://github.com/paperless-ngx/paperless-ngx/pull/12531))
|
||||||
|
- [BREAKING] Decouple OCR control from archive file control [@stumpylog](https://github.com/stumpylog) ([#12448](https://github.com/paperless-ngx/paperless-ngx/pull/12448))
|
||||||
|
- Fixhancement: include sharelinks + bundles in export/import [@shamoon](https://github.com/shamoon) ([#12479](https://github.com/paperless-ngx/paperless-ngx/pull/12479))
|
||||||
|
- Chore: Better typed status manager messages [@stumpylog](https://github.com/stumpylog) ([#12509](https://github.com/paperless-ngx/paperless-ngx/pull/12509))
|
||||||
|
- Enhancement: unify text search to use tantivy [@shamoon](https://github.com/shamoon) ([#12485](https://github.com/paperless-ngx/paperless-ngx/pull/12485))
|
||||||
|
- Feature: Consumer logging correlation [@stumpylog](https://github.com/stumpylog) ([#12510](https://github.com/paperless-ngx/paperless-ngx/pull/12510))
|
||||||
|
- Chore(deps): Bump djangorestframework from 3.16.1 to 3.17.1 in the django-ecosystem group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12488](https://github.com/paperless-ngx/paperless-ngx/pull/12488))
|
||||||
|
- Chore(deps): Bump the document-processing group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12489](https://github.com/paperless-ngx/paperless-ngx/pull/12489))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 5 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12499](https://github.com/paperless-ngx/paperless-ngx/pull/12499))
|
||||||
|
- Security: Sign the Celery pickled tasks [@stumpylog](https://github.com/stumpylog) ([#12504](https://github.com/paperless-ngx/paperless-ngx/pull/12504))
|
||||||
|
- Security enhancement: allow opt-in blocking internal mail hosts [@shamoon](https://github.com/shamoon) ([#12502](https://github.com/paperless-ngx/paperless-ngx/pull/12502))
|
||||||
|
- Feature: Replace Whoosh with tantivy search backend [@stumpylog](https://github.com/stumpylog) ([#12471](https://github.com/paperless-ngx/paperless-ngx/pull/12471))
|
||||||
|
- Chore(deps): Bump aiohttp from 3.13.3 to 3.13.4 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12486](https://github.com/paperless-ngx/paperless-ngx/pull/12486))
|
||||||
|
- Chore(deps): Bump lodash from 4.17.23 to 4.18.1 in /src-ui in the npm\_and\_yarn group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12498](https://github.com/paperless-ngx/paperless-ngx/pull/12498))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group across 1 directory with 20 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12491](https://github.com/paperless-ngx/paperless-ngx/pull/12491))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.4.0 to 25.5.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12494](https://github.com/paperless-ngx/paperless-ngx/pull/12494))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group in /src-ui with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12493](https://github.com/paperless-ngx/paperless-ngx/pull/12493))
|
||||||
|
- Performance: support bulk edit without id lists [@shamoon](https://github.com/shamoon) ([#12355](https://github.com/paperless-ngx/paperless-ngx/pull/12355))
|
||||||
|
- Performance: deprecate and remove usage of `all` in API results [@shamoon](https://github.com/shamoon) ([#12309](https://github.com/paperless-ngx/paperless-ngx/pull/12309))
|
||||||
|
- Performance: support passing selection data with filtered document requests [@shamoon](https://github.com/shamoon) ([#12300](https://github.com/paperless-ngx/paperless-ngx/pull/12300))
|
||||||
|
- Chore(deps): Bump cryptography from 46.0.5 to 46.0.6 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12458](https://github.com/paperless-ngx/paperless-ngx/pull/12458))
|
||||||
|
- Tweakhancement: show file extension in StoragePath test [@shamoon](https://github.com/shamoon) ([#12452](https://github.com/paperless-ngx/paperless-ngx/pull/12452))
|
||||||
|
- Feature: Transition all checksums to use SHA256 [@stumpylog](https://github.com/stumpylog) ([#12432](https://github.com/paperless-ngx/paperless-ngx/pull/12432))
|
||||||
|
- Chore(deps): Bump requests from 2.32.5 to 2.33.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12441](https://github.com/paperless-ngx/paperless-ngx/pull/12441))
|
||||||
|
- Enhancement: auto-hide the search bar on mobile [@shamoon](https://github.com/shamoon) ([#12404](https://github.com/paperless-ngx/paperless-ngx/pull/12404))
|
||||||
|
- Chore: logger, response and template sanitization cleanup [@shamoon](https://github.com/shamoon) ([#12439](https://github.com/paperless-ngx/paperless-ngx/pull/12439))
|
||||||
|
- Fix: Respect workflow change\_groups during consumption [@stumpylog](https://github.com/stumpylog) ([#12431](https://github.com/paperless-ngx/paperless-ngx/pull/12431))
|
||||||
|
- Security: prevent prototype pollution in frontend settings and list view [@shamoon](https://github.com/shamoon) ([#12438](https://github.com/paperless-ngx/paperless-ngx/pull/12438))
|
||||||
|
- Chore(deps): Bump cbor2 from 5.8.0 to 5.9.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12424](https://github.com/paperless-ngx/paperless-ngx/pull/12424))
|
||||||
|
- Chore(deps): Bump tinytag from 2.2.0 to 2.2.1 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#12396](https://github.com/paperless-ngx/paperless-ngx/pull/12396))
|
||||||
|
- Chore(deps): Bump ujson from 5.11.0 to 5.12.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12387](https://github.com/paperless-ngx/paperless-ngx/pull/12387))
|
||||||
|
- Chore(deps): Bump pyasn1 from 0.6.2 to 0.6.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12370](https://github.com/paperless-ngx/paperless-ngx/pull/12370))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12382](https://github.com/paperless-ngx/paperless-ngx/pull/12382))
|
||||||
|
- Chore(deps): Bump openai from 2.24.0 to 2.26.0 in the utilities-minor group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12379](https://github.com/paperless-ngx/paperless-ngx/pull/12379))
|
||||||
|
- Chore(deps-dev): Bump types-python-dateutil from 2.9.0.20260124 to 2.9.0.20260305 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12380](https://github.com/paperless-ngx/paperless-ngx/pull/12380))
|
||||||
|
- Chore(deps): Update django-allauth[mfa,socialaccount] requirement from ~=65.14.0 to ~=65.15.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12381](https://github.com/paperless-ngx/paperless-ngx/pull/12381))
|
||||||
|
- Chore(deps-dev): Bump the frontend-jest-dependencies group in /src-ui with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12374](https://github.com/paperless-ngx/paperless-ngx/pull/12374))
|
||||||
|
- Chore(deps): Bump @ng-select/ng-select from 21.4.1 to 21.5.2 in /src-ui in the frontend-angular-dependencies group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12373](https://github.com/paperless-ngx/paperless-ngx/pull/12373))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 25.3.3 to 25.4.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12376](https://github.com/paperless-ngx/paperless-ngx/pull/12376))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group in /src-ui with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12375](https://github.com/paperless-ngx/paperless-ngx/pull/12375))
|
||||||
|
- Chore: Reduce system status test times [@stumpylog](https://github.com/stumpylog) ([#12354](https://github.com/paperless-ngx/paperless-ngx/pull/12354))
|
||||||
|
- Chore(deps): Bump pyopenssl from 25.3.0 to 26.0.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12363](https://github.com/paperless-ngx/paperless-ngx/pull/12363))
|
||||||
|
- Chore(deps): Bump pyjwt from 2.10.1 to 2.12.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12335](https://github.com/paperless-ngx/paperless-ngx/pull/12335))
|
||||||
|
- Change: sort custom fields alphabetically by default [@shamoon](https://github.com/shamoon) ([#12358](https://github.com/paperless-ngx/paperless-ngx/pull/12358))
|
||||||
|
- Feature: Add progress information to the classifier training for a better ux [@stumpylog](https://github.com/stumpylog) ([#12331](https://github.com/paperless-ngx/paperless-ngx/pull/12331))
|
||||||
|
- Chore: bump Angular dependencies to 21.2.x [@shamoon](https://github.com/shamoon) ([#12338](https://github.com/paperless-ngx/paperless-ngx/pull/12338))
|
||||||
|
- Bump ocrmypdf from 16.13.0 to 17.3.0 in the document-processing group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12267](https://github.com/paperless-ngx/paperless-ngx/pull/12267))
|
||||||
|
- Feature: document parser plugin framework [@stumpylog](https://github.com/stumpylog) ([#12294](https://github.com/paperless-ngx/paperless-ngx/pull/12294))
|
||||||
|
- Chore(deps): Bump tornado from 6.5.4 to 6.5.5 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12327](https://github.com/paperless-ngx/paperless-ngx/pull/12327))
|
||||||
|
- Enhancement: Make the StatusConsumer truly async [@stumpylog](https://github.com/stumpylog) ([#12298](https://github.com/paperless-ngx/paperless-ngx/pull/12298))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 5 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12324](https://github.com/paperless-ngx/paperless-ngx/pull/12324))
|
||||||
|
- Chore: mark document detail email action as deprecated [@shamoon](https://github.com/shamoon) ([#12308](https://github.com/paperless-ngx/paperless-ngx/pull/12308))
|
||||||
|
- Chore: separate actions from bulk edit endpoint [@shamoon](https://github.com/shamoon) ([#12286](https://github.com/paperless-ngx/paperless-ngx/pull/12286))
|
||||||
|
- [BREAKING] Chore: drop support for api versions \< 9 [@shamoon](https://github.com/shamoon) ([#12284](https://github.com/paperless-ngx/paperless-ngx/pull/12284))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 6 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12291](https://github.com/paperless-ngx/paperless-ngx/pull/12291))
|
||||||
|
- Performance: Stream JSON during import for memory improvements [@stumpylog](https://github.com/stumpylog) ([#12276](https://github.com/paperless-ngx/paperless-ngx/pull/12276))
|
||||||
|
- Feature: Migrate import/export to rich progress [@stumpylog](https://github.com/stumpylog) ([#12260](https://github.com/paperless-ngx/paperless-ngx/pull/12260))
|
||||||
|
- Enhancement: Show more document details in merge dialog [@svenstaro](https://github.com/svenstaro) ([#12271](https://github.com/paperless-ngx/paperless-ngx/pull/12271))
|
||||||
|
- Performance: Further export memory improvements [@stumpylog](https://github.com/stumpylog) ([#12273](https://github.com/paperless-ngx/paperless-ngx/pull/12273))
|
||||||
|
- Chore: pytest style paperless tests [@stumpylog](https://github.com/stumpylog) ([#12254](https://github.com/paperless-ngx/paperless-ngx/pull/12254))
|
||||||
|
- Chore: update ESLint to v10 [@shamoon](https://github.com/shamoon) ([#12256](https://github.com/paperless-ngx/paperless-ngx/pull/12256))
|
||||||
|
- Bump django-allauth from 65.14.0 to 65.14.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12253](https://github.com/paperless-ngx/paperless-ngx/pull/12253))
|
||||||
|
- Bump django from 5.2.11 to 5.2.12 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12249](https://github.com/paperless-ngx/paperless-ngx/pull/12249))
|
||||||
|
- [BREAKING] Chore: Drop support for Python 3.10 [@stumpylog](https://github.com/stumpylog) ([#12234](https://github.com/paperless-ngx/paperless-ngx/pull/12234))
|
||||||
|
- Performance: Improve exporter memory efficiency [@stumpylog](https://github.com/stumpylog) ([#12236](https://github.com/paperless-ngx/paperless-ngx/pull/12236))
|
||||||
|
- Enhancement: saved view sharing [@shamoon](https://github.com/shamoon) ([#12142](https://github.com/paperless-ngx/paperless-ngx/pull/12142))
|
||||||
|
- Fix: use maxsplit=1 in Redis URL parsing to handle URLs with multiple colons [@cryptomilk](https://github.com/cryptomilk) ([#12239](https://github.com/paperless-ngx/paperless-ngx/pull/12239))
|
||||||
|
- Enhancement: “live” document updates [@shamoon](https://github.com/shamoon) ([#12141](https://github.com/paperless-ngx/paperless-ngx/pull/12141))
|
||||||
|
- Enhancement: Improve the retagger output using rich [@stumpylog](https://github.com/stumpylog) ([#12194](https://github.com/paperless-ngx/paperless-ngx/pull/12194))
|
||||||
|
- Enhancement: Transition sanity check to rich and improve output [@stumpylog](https://github.com/stumpylog) ([#12182](https://github.com/paperless-ngx/paperless-ngx/pull/12182))
|
||||||
|
- Enhancement: Switch all indexing to use rich [@stumpylog](https://github.com/stumpylog) ([#12193](https://github.com/paperless-ngx/paperless-ngx/pull/12193))
|
||||||
|
- Chore(deps): Bump whitenoise from 6.11.0 to 6.12.0 in the django-ecosystem group @[dependabot[bot]](https://github.com/apps/dependabot) ([#12192](https://github.com/paperless-ngx/paperless-ngx/pull/12192))
|
||||||
|
- Fixhancement: show sequential + id version labels, fix padding [@shamoon](https://github.com/shamoon) ([#12196](https://github.com/paperless-ngx/paperless-ngx/pull/12196))
|
||||||
|
- [BREAKING] Chore: Refactor advanced database settings to allow more user configuration [@stumpylog](https://github.com/stumpylog) ([#12165](https://github.com/paperless-ngx/paperless-ngx/pull/12165))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 11 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12179](https://github.com/paperless-ngx/paperless-ngx/pull/12179))
|
||||||
|
- Feature: Switch progress bar library to rich [@stumpylog](https://github.com/stumpylog) ([#12169](https://github.com/paperless-ngx/paperless-ngx/pull/12169))
|
||||||
|
- Chore(deps): Bump nltk from 3.9.2 to 3.9.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12177](https://github.com/paperless-ngx/paperless-ngx/pull/12177))
|
||||||
|
- Enhancement: Formatted filename for single document downloads [@JanKleine](https://github.com/JanKleine) ([#12095](https://github.com/paperless-ngx/paperless-ngx/pull/12095))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 7 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12174](https://github.com/paperless-ngx/paperless-ngx/pull/12174))
|
||||||
|
- Feature: document file versions [@shamoon](https://github.com/shamoon) ([#12061](https://github.com/paperless-ngx/paperless-ngx/pull/12061))
|
||||||
|
- [BREAKING] Remove API v1 compatibility [@stumpylog](https://github.com/stumpylog) ([#12166](https://github.com/paperless-ngx/paperless-ngx/pull/12166))
|
||||||
|
- Enhancement: prevent duplicate mail processing across rules [@shamoon](https://github.com/shamoon) ([#12159](https://github.com/paperless-ngx/paperless-ngx/pull/12159))
|
||||||
|
- Feature: move to trash action for workflows [@JanKleine](https://github.com/JanKleine) ([#11176](https://github.com/paperless-ngx/paperless-ngx/pull/11176))
|
||||||
|
- Tweakhancement: reset to page 1 on reset filters [@shamoon](https://github.com/shamoon) ([#12143](https://github.com/paperless-ngx/paperless-ngx/pull/12143))
|
||||||
|
- Fix: correct user dropdown button icon styling [@shamoon](https://github.com/shamoon) ([#12092](https://github.com/paperless-ngx/paperless-ngx/pull/12092))
|
||||||
|
- Enhancement: consolidate management lists into document attributes section [@shamoon](https://github.com/shamoon) ([#12045](https://github.com/paperless-ngx/paperless-ngx/pull/12045))
|
||||||
|
- Enhancement: option to stop processing further mail rules [@shamoon](https://github.com/shamoon) ([#12053](https://github.com/paperless-ngx/paperless-ngx/pull/12053))
|
||||||
|
- [BREAKING] Remove pybzar as a barcode reader [@stumpylog](https://github.com/stumpylog) ([#12065](https://github.com/paperless-ngx/paperless-ngx/pull/12065))
|
||||||
|
- Chore(deps): Bump pillow from 12.1.0 to 12.1.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12064](https://github.com/paperless-ngx/paperless-ngx/pull/12064))
|
||||||
|
- Chore(deps): Bump cryptography from 46.0.3 to 46.0.5 @[dependabot[bot]](https://github.com/apps/dependabot) ([#12060](https://github.com/paperless-ngx/paperless-ngx/pull/12060))
|
||||||
|
- Chore: Optimizes the integer fields for choice types [@stumpylog](https://github.com/stumpylog) ([#12057](https://github.com/paperless-ngx/paperless-ngx/pull/12057))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12051](https://github.com/paperless-ngx/paperless-ngx/pull/12051))
|
||||||
|
- Tweak: improve 2-digit year parsing [@shamoon](https://github.com/shamoon) ([#12044](https://github.com/paperless-ngx/paperless-ngx/pull/12044))
|
||||||
|
- Enhancement: pngx pdf viewer [@shamoon](https://github.com/shamoon) ([#12043](https://github.com/paperless-ngx/paperless-ngx/pull/12043))
|
||||||
|
- Fix: Handle an OOM kill of celery workers in the task handler [@stumpylog](https://github.com/stumpylog) ([#12040](https://github.com/paperless-ngx/paperless-ngx/pull/12040))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.58.1 to 1.58.2 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12032](https://github.com/paperless-ngx/paperless-ngx/pull/12032))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#12020](https://github.com/paperless-ngx/paperless-ngx/pull/12020))
|
||||||
|
- Fixhancement: improve ASN handling with PDF operations [@shamoon](https://github.com/shamoon) ([#11689](https://github.com/paperless-ngx/paperless-ngx/pull/11689))
|
||||||
|
- Upgrade: Bump @types/node from 25.2.0 to 25.2.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12008](https://github.com/paperless-ngx/paperless-ngx/pull/12008))
|
||||||
|
- Upgrade: Bump webpack from 5.103.0 to 5.105.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#12007](https://github.com/paperless-ngx/paperless-ngx/pull/12007))
|
||||||
|
- Chore: bump Angular to 21.1.3, ngx-ui-tour-ng-bootstrap to v18 [@shamoon](https://github.com/shamoon) ([#12015](https://github.com/paperless-ngx/paperless-ngx/pull/12015))
|
||||||
|
- Chore: Switches to use prek in place of pre-commit [@stumpylog](https://github.com/stumpylog) ([#12002](https://github.com/paperless-ngx/paperless-ngx/pull/12002))
|
||||||
|
- Chore(deps): Bump the utilities-patch group with 3 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11981](https://github.com/paperless-ngx/paperless-ngx/pull/11981))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 6 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11993](https://github.com/paperless-ngx/paperless-ngx/pull/11993))
|
||||||
|
- Chore(deps): Bump drf-spectacular-sidecar from 2025.10.1 to 2026.1.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11985](https://github.com/paperless-ngx/paperless-ngx/pull/11985))
|
||||||
|
- Chore(deps): Update django-allauth[mfa,socialaccount] requirement from ~=65.13.1 to ~=65.14.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11984](https://github.com/paperless-ngx/paperless-ngx/pull/11984))
|
||||||
|
- Chore(deps): Update granian[uvloop] requirement from ~=2.6.0 to ~=2.7.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11983](https://github.com/paperless-ngx/paperless-ngx/pull/11983))
|
||||||
|
- Chore(deps): Bump django from 5.2.10 to 5.2.11 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11988](https://github.com/paperless-ngx/paperless-ngx/pull/11988))
|
||||||
|
- Feature: Enable users to customize date parsing via plugins [@stumpylog](https://github.com/stumpylog) ([#11931](https://github.com/paperless-ngx/paperless-ngx/pull/11931))
|
||||||
|
- Chore: Fix further test flakiness [@stumpylog](https://github.com/stumpylog) ([#11987](https://github.com/paperless-ngx/paperless-ngx/pull/11987))
|
||||||
|
- Feature: password removal workflow action [@shamoon](https://github.com/shamoon) ([#11665](https://github.com/paperless-ngx/paperless-ngx/pull/11665))
|
||||||
|
- Fix: re-run ASN check after barcode detection [@shamoon](https://github.com/shamoon) ([#11681](https://github.com/paperless-ngx/paperless-ngx/pull/11681))
|
||||||
|
- Fix: prevent infinite loading crash in mail component [@shamoon](https://github.com/shamoon) ([#11978](https://github.com/paperless-ngx/paperless-ngx/pull/11978))
|
||||||
|
- Enhancement: per-type object page sizing [@shamoon](https://github.com/shamoon) ([#11977](https://github.com/paperless-ngx/paperless-ngx/pull/11977))
|
||||||
|
- Chore(deps): Bump the frontend-angular-dependencies group in /src-ui with 22 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11968](https://github.com/paperless-ngx/paperless-ngx/pull/11968))
|
||||||
|
- Chore(deps): Bump zone.js from 0.15.1 to 0.16.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#11970](https://github.com/paperless-ngx/paperless-ngx/pull/11970))
|
||||||
|
- Chore(deps-dev): Bump the frontend-eslint-dependencies group in /src-ui with 4 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11969](https://github.com/paperless-ngx/paperless-ngx/pull/11969))
|
||||||
|
- Chore(deps-dev): Bump @types/node from 24.10.1 to 25.2.0 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#11972](https://github.com/paperless-ngx/paperless-ngx/pull/11972))
|
||||||
|
- Chore(deps-dev): Bump @playwright/test from 1.57.0 to 1.58.1 in /src-ui @[dependabot[bot]](https://github.com/apps/dependabot) ([#11971](https://github.com/paperless-ngx/paperless-ngx/pull/11971))
|
||||||
|
- Enhancement: improve filter drop-down performance with virtual scrolling [@shamoon](https://github.com/shamoon) ([#11973](https://github.com/paperless-ngx/paperless-ngx/pull/11973))
|
||||||
|
- Tweakhancement: tweak bulk delete text [@shamoon](https://github.com/shamoon) ([#11967](https://github.com/paperless-ngx/paperless-ngx/pull/11967))
|
||||||
|
- Feature: support split documents based on tag barcodes @schober-ch ([#11645](https://github.com/paperless-ngx/paperless-ngx/pull/11645))
|
||||||
|
- Chore: Bulk backend updates [@stumpylog](https://github.com/stumpylog) ([#11543](https://github.com/paperless-ngx/paperless-ngx/pull/11543))
|
||||||
|
- Tweak: increase minimum screen width before inserting padding [@JanKleine](https://github.com/JanKleine) ([#11926](https://github.com/paperless-ngx/paperless-ngx/pull/11926))
|
||||||
|
- Enhancement: Add setting for default PDF Editor mode [@JanKleine](https://github.com/JanKleine) ([#11927](https://github.com/paperless-ngx/paperless-ngx/pull/11927))
|
||||||
|
- Fixhancement: auto-queue llm index if needed [@shamoon](https://github.com/shamoon) ([#11891](https://github.com/paperless-ngx/paperless-ngx/pull/11891))
|
||||||
|
- Feature: sharelink bundles [@shamoon](https://github.com/shamoon) ([#11682](https://github.com/paperless-ngx/paperless-ngx/pull/11682))
|
||||||
|
- Enhancement: user control of doc details fields [@shamoon](https://github.com/shamoon) ([#11906](https://github.com/paperless-ngx/paperless-ngx/pull/11906))
|
||||||
|
- Enhancement: improve relative dates in date filter [@JanKleine](https://github.com/JanKleine) ([#11899](https://github.com/paperless-ngx/paperless-ngx/pull/11899))
|
||||||
|
- Performance: faster statistics panel on dashboard [@Merinorus](https://github.com/Merinorus) ([#11760](https://github.com/paperless-ngx/paperless-ngx/pull/11760))
|
||||||
|
- Enhancement: allow duplicates with warnings, UI for discovery [@shamoon](https://github.com/shamoon) ([#11815](https://github.com/paperless-ngx/paperless-ngx/pull/11815))
|
||||||
|
- Enhancement: configurable SSO groups claim [@Gabgobie](https://github.com/Gabgobie) ([#11841](https://github.com/paperless-ngx/paperless-ngx/pull/11841))
|
||||||
|
- Enhancement: support select all for management lists [@shamoon](https://github.com/shamoon) ([#11889](https://github.com/paperless-ngx/paperless-ngx/pull/11889))
|
||||||
|
- Tweakhancement: display document id, with copy [@shamoon](https://github.com/shamoon) ([#11896](https://github.com/paperless-ngx/paperless-ngx/pull/11896))
|
||||||
|
- Enhancement: Add support for app oidc [@paulgessinger](https://github.com/paulgessinger) ([#11756](https://github.com/paperless-ngx/paperless-ngx/pull/11756))
|
||||||
|
- Enhancement: Add 'any of' workflow trigger filters [@shamoon](https://github.com/shamoon) ([#11683](https://github.com/paperless-ngx/paperless-ngx/pull/11683))
|
||||||
|
- [BREAKING] Remove support for document and thumbnail encryption [@stumpylog](https://github.com/stumpylog) ([#11850](https://github.com/paperless-ngx/paperless-ngx/pull/11850))
|
||||||
|
- Fix: ensure css color-scheme for dark mode [@shamoon](https://github.com/shamoon) ([#11855](https://github.com/paperless-ngx/paperless-ngx/pull/11855))
|
||||||
|
- Enhancement: support doc\_id placeholder in workflow templates [@shamoon](https://github.com/shamoon) ([#11847](https://github.com/paperless-ngx/paperless-ngx/pull/11847))
|
||||||
|
- [BREAKING] Feature: Simplify and improve the consumer [@stumpylog](https://github.com/stumpylog) ([#11753](https://github.com/paperless-ngx/paperless-ngx/pull/11753))
|
||||||
|
- Fix: fix tag list horizontal scroll, again [@shamoon](https://github.com/shamoon) ([#11839](https://github.com/paperless-ngx/paperless-ngx/pull/11839))
|
||||||
|
- Fix: ensure horizontal scroll for long tag names in list, wrap tags without parent [@shamoon](https://github.com/shamoon) ([#11811](https://github.com/paperless-ngx/paperless-ngx/pull/11811))
|
||||||
|
- Chore(deps): Bump the utilities-patch group across 1 directory with 7 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11793](https://github.com/paperless-ngx/paperless-ngx/pull/11793))
|
||||||
|
- Chore(deps): Bump the utilities-minor group across 1 directory with 10 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#11799](https://github.com/paperless-ngx/paperless-ngx/pull/11799))
|
||||||
|
- Chore(deps): Bump pyasn1 from 0.6.1 to 0.6.2 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11801](https://github.com/paperless-ngx/paperless-ngx/pull/11801))
|
||||||
|
- Chore(deps): Bump torch from 2.7.1 to 2.8.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11800](https://github.com/paperless-ngx/paperless-ngx/pull/11800))
|
||||||
|
- Chore(deps): Bump brotli from 1.1.0 to 1.2.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11796](https://github.com/paperless-ngx/paperless-ngx/pull/11796))
|
||||||
|
- Chore(deps): Bump transformers from 4.51.3 to 4.53.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11797](https://github.com/paperless-ngx/paperless-ngx/pull/11797))
|
||||||
|
- Chore(deps): Bump django from 5.2.7 to 5.2.9 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11794](https://github.com/paperless-ngx/paperless-ngx/pull/11794))
|
||||||
|
- Chore(deps): Bump the llama-index group @[dependabot[bot]](https://github.com/apps/dependabot) ([#11798](https://github.com/paperless-ngx/paperless-ngx/pull/11798))
|
||||||
|
- Chore(deps): Bump marshmallow from 3.26.1 to 3.26.2 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11790](https://github.com/paperless-ngx/paperless-ngx/pull/11790))
|
||||||
|
- Chore(deps): Bump uv from 0.9.3 to 0.9.6 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11795](https://github.com/paperless-ngx/paperless-ngx/pull/11795))
|
||||||
|
- Chore(deps): Bump aiohttp from 3.11.18 to 3.13.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11789](https://github.com/paperless-ngx/paperless-ngx/pull/11789))
|
||||||
|
- Chore(deps): Bump urllib3 from 2.5.0 to 2.6.3 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11792](https://github.com/paperless-ngx/paperless-ngx/pull/11792))
|
||||||
|
- Chore(deps): Bump virtualenv from 20.34.0 to 20.36.1 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11774](https://github.com/paperless-ngx/paperless-ngx/pull/11774))
|
||||||
|
- Fix: use explicit order field for workflow actions [@shamoon](https://github.com/shamoon) ([#11781](https://github.com/paperless-ngx/paperless-ngx/pull/11781))
|
||||||
|
- Chore(deps): Bump azure-core from 1.33.0 to 1.38.0 @[dependabot[bot]](https://github.com/apps/dependabot) ([#11776](https://github.com/paperless-ngx/paperless-ngx/pull/11776))
|
||||||
|
- Feature: Paperless AI [@shamoon](https://github.com/shamoon) ([#10319](https://github.com/paperless-ngx/paperless-ngx/pull/10319))
|
||||||
|
- Performance: improve treenode inefficiencies [@shamoon](https://github.com/shamoon) ([#11606](https://github.com/paperless-ngx/paperless-ngx/pull/11606))
|
||||||
|
- Chore: upgrade to node v24 [@shamoon](https://github.com/shamoon) ([#11747](https://github.com/paperless-ngx/paperless-ngx/pull/11747))
|
||||||
|
- Chore: upgrade to Angular v21 [@shamoon](https://github.com/shamoon) ([#11746](https://github.com/paperless-ngx/paperless-ngx/pull/11746))
|
||||||
|
- Feature: Remote OCR (Azure AI) [@shamoon](https://github.com/shamoon) ([#10320](https://github.com/paperless-ngx/paperless-ngx/pull/10320))
|
||||||
|
- Feature: password removal action [@shamoon](https://github.com/shamoon) ([#11656](https://github.com/paperless-ngx/paperless-ngx/pull/11656))
|
||||||
|
- Fix: fix recurring workflow to respect latest run time [@shamoon](https://github.com/shamoon) ([#11735](https://github.com/paperless-ngx/paperless-ngx/pull/11735))
|
||||||
|
- Fixhancement: add error handling and retry when opening index [@shamoon](https://github.com/shamoon) ([#11731](https://github.com/paperless-ngx/paperless-ngx/pull/11731))
|
||||||
|
- Tweakhancement: use anchor element for management list quick filter buttons [@shamoon](https://github.com/shamoon) ([#11692](https://github.com/paperless-ngx/paperless-ngx/pull/11692))
|
||||||
|
- Fix: validate cf integer values within PostgreSQL range [@shamoon](https://github.com/shamoon) ([#11666](https://github.com/paperless-ngx/paperless-ngx/pull/11666))
|
||||||
|
- Fix: support ordering by storage path name [@shamoon](https://github.com/shamoon) ([#11661](https://github.com/paperless-ngx/paperless-ngx/pull/11661))
|
||||||
|
- Fix: propagate metadata override created value [@shamoon](https://github.com/shamoon) ([#11659](https://github.com/paperless-ngx/paperless-ngx/pull/11659))
|
||||||
|
- Fix: prevent ASN collisions for merge operations [@shamoon](https://github.com/shamoon) ([#11634](https://github.com/paperless-ngx/paperless-ngx/pull/11634))
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
## paperless-ngx 2.20.15
|
## paperless-ngx 2.20.15
|
||||||
|
|
||||||
### Security
|
### Security
|
||||||
@@ -7045,7 +7827,7 @@ primarily.
|
|||||||
|
|
||||||
### paperless-ng 0.9.0
|
### paperless-ng 0.9.0
|
||||||
|
|
||||||
- **Deprecated:** GnuPG. [See this note on the state of GnuPG in paperless-ng.](administration.md#encryption)
|
- **Deprecated:** GnuPG. [See this note on the state of GnuPG in paperless-ng.](#paperless-ng-093)
|
||||||
This features will most likely be removed in future versions.
|
This features will most likely be removed in future versions.
|
||||||
- **Added:** New frontend. Features:
|
- **Added:** New frontend. Features:
|
||||||
- Single page application: It's much more responsive than the
|
- Single page application: It's much more responsive than the
|
||||||
|
|||||||
@@ -192,7 +192,7 @@ searchable PDF and stores it as the archive copy. `ARCHIVE_FILE_GENERATION=never
|
|||||||
has no effect for documents handled by the remote parser — the archive is produced
|
has no effect for documents handled by the remote parser — the archive is produced
|
||||||
unconditionally by the remote engine.
|
unconditionally by the remote engine.
|
||||||
|
|
||||||
# Search Index (Whoosh -> Tantivy)
|
## Search Index (Whoosh -> Tantivy)
|
||||||
|
|
||||||
The full-text search backend has been replaced with [Tantivy](https://github.com/quickwit-oss/tantivy).
|
The full-text search backend has been replaced with [Tantivy](https://github.com/quickwit-oss/tantivy).
|
||||||
The index format is incompatible with Whoosh, so **the search index is automatically rebuilt from
|
The index format is incompatible with Whoosh, so **the search index is automatically rebuilt from
|
||||||
|
|||||||
+2
-2
@@ -157,8 +157,8 @@ Do not combine this with `USERMAP_UID` or `USERMAP_GID`, which are intended for
|
|||||||
Some file systems, such as NFS network shares, don't support file system
|
Some file systems, such as NFS network shares, don't support file system
|
||||||
notifications with `inotify`. When the consumption directory is on such a
|
notifications with `inotify`. When the consumption directory is on such a
|
||||||
file system, Paperless-ngx will not pick up new files with the default
|
file system, Paperless-ngx will not pick up new files with the default
|
||||||
configuration. Use [`PAPERLESS_CONSUMER_POLLING`](configuration.md#PAPERLESS_CONSUMER_POLLING)
|
configuration. Set [`PAPERLESS_CONSUMER_POLLING_INTERVAL`](configuration.md#PAPERLESS_CONSUMER_POLLING_INTERVAL)
|
||||||
to enable polling and disable inotify. See [here](configuration.md#polling).
|
to a positive number to enable polling and disable native filesystem notifications.
|
||||||
|
|
||||||
## Bare Metal Install {#bare_metal}
|
## Bare Metal Install {#bare_metal}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,839 @@
|
|||||||
|
# Export Zip Compression Control Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Add `--zip-compression {stored,deflated,bzip2,lzma,zstd}` and `--zip-compression-level N` flags to `document_exporter`, threaded into `ZipExportSink`, with import-side safety for codecs the running Python can't read.
|
||||||
|
|
||||||
|
**Architecture:** A new pure-data module `documents/export/compression.py` owns the method↔constant map, per-method level bounds, the runtime availability probe, and a compress-type readability check. `ZipExportSink` gains `compression`/`compresslevel` constructor params. The command validates flags up front (fail-fast `CommandError`) and constructs the sink; the importer pre-checks entry compress types before extracting.
|
||||||
|
|
||||||
|
**Tech Stack:** Python ≥3.11 (zstd only on 3.14+), `zipfile`, `compression.zstd` (PEP 784), pytest + pytest-mock + factory-boy. Backend tests run on the Linux VM (Python 3.11 — zstd positive tests are `skipif`-guarded); `ruff` runs locally.
|
||||||
|
|
||||||
|
**Spec:** `docs/superpowers/specs/2026-06-16-export-zip-compression-design.md`
|
||||||
|
|
||||||
|
**PREREQUISITE:** The base refactor `docs/superpowers/plans/2026-06-16-export-sink-architecture.md` MUST be merged first. This plan assumes `src/documents/export/sinks.py` exists with `ZipExportSink(target, zip_name, *, delete=False)` opening its `ZipFile` in `_open()`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Verified facts (CPython 3.14.3, via `uv run --python 3.14 --no-project`)
|
||||||
|
|
||||||
|
- Constants: `ZIP_STORED=0`, `ZIP_DEFLATED=8`, `ZIP_BZIP2=12`, `ZIP_LZMA=14`, `ZIP_ZSTANDARD=93` (zstd added 3.14; absent on < 3.14).
|
||||||
|
- `ZipFile(file, "w", compression=…, compresslevel=…)` applies both as the default for every `write`/`writestr` — no per-entry args needed (verified).
|
||||||
|
- Level bounds: `deflated` 0–9, `bzip2` 1–9, `lzma`/`stored` ignore level, `zstd` -131072…22 (`compression.zstd.CompressionParameter.compression_level.bounds() == (-131072, 22)`).
|
||||||
|
- An invalid level fails at the **first write** (`ValueError: Invalid initialization option` / `compresslevel must be between 1 and 9`), plus GC-time `AttributeError` noise on close — hence up-front validation.
|
||||||
|
- zstd is backed by `compression.zstd`; `zipfile` raises `RuntimeError` if it's unavailable.
|
||||||
|
|
||||||
|
## Conventions for every task
|
||||||
|
|
||||||
|
- **Run backend tests on the VM:** `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "<targets>"` (never locally).
|
||||||
|
- **Lint locally:** `ruff check <paths> && ruff format <paths>` (global ruff, not `uv run`).
|
||||||
|
- **Tests are pytest-style:** classes, `@pytest.mark.django_db` on the class only where DB is needed (the `compression.py` and sink tests need no DB), factory-boy, `mocker`, `parametrize`, full type annotations.
|
||||||
|
- The VM runs Python 3.11, so **zstd positive tests must be `@pytest.mark.skipif(...)`-guarded**; they will simply not run there. zstd _rejection_ tests (the < 3.14 path) DO run on the VM.
|
||||||
|
|
||||||
|
## File structure
|
||||||
|
|
||||||
|
- **Create** `src/documents/export/compression.py` — method map, CLI choices, level bounds, `compression_available()`, `level_error()`, `compress_type_readable()`, `unreadable_method_names()`. Pure, no Django.
|
||||||
|
- **Create** `src/documents/tests/export/test_compression.py` — unit tests for the above.
|
||||||
|
- **Modify** `src/documents/export/sinks.py` — `ZipExportSink.__init__` gains `compression`/`compresslevel`; `_open()` passes them to `ZipFile`.
|
||||||
|
- **Modify** `src/documents/tests/export/test_sinks.py` — assert the chosen `compress_type` is applied.
|
||||||
|
- **Modify** `src/documents/management/commands/document_exporter.py` — add the two CLI flags, up-front validation, and pass resolved values to `ZipExportSink`.
|
||||||
|
- **Modify** `src/documents/tests/test_management_exporter.py` — flag validation + default-unchanged tests.
|
||||||
|
- **Modify** `src/documents/management/commands/document_importer.py` — pre-extract compress-type check.
|
||||||
|
- **Modify** `src/documents/tests/test_management_importer.py` — unsupported-codec → `CommandError`.
|
||||||
|
- **Modify** `docs/administration.md` — document both flags + zstd portability caveat.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: `documents/export/compression.py` (pure compression policy)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Create: `src/documents/export/compression.py`
|
||||||
|
- Test: `src/documents/tests/export/test_compression.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing tests**
|
||||||
|
|
||||||
|
Create `src/documents/tests/export/test_compression.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from documents.export import compression
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompressionMethods:
|
||||||
|
def test_choices_always_include_zstd(self) -> None:
|
||||||
|
# zstd is offered regardless of runtime; availability is checked separately
|
||||||
|
assert compression.COMPRESSION_CHOICES == (
|
||||||
|
"stored",
|
||||||
|
"deflated",
|
||||||
|
"bzip2",
|
||||||
|
"lzma",
|
||||||
|
"zstd",
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("name", "constant"),
|
||||||
|
[
|
||||||
|
("stored", zipfile.ZIP_STORED),
|
||||||
|
("deflated", zipfile.ZIP_DEFLATED),
|
||||||
|
("bzip2", zipfile.ZIP_BZIP2),
|
||||||
|
("lzma", zipfile.ZIP_LZMA),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_method_maps_to_zipfile_constant(self, name: str, constant: int) -> None:
|
||||||
|
assert compression.COMPRESSION_METHODS[name] == constant
|
||||||
|
|
||||||
|
def test_stored_and_deflated_always_available(self) -> None:
|
||||||
|
assert compression.compression_available("stored")
|
||||||
|
assert compression.compression_available("deflated")
|
||||||
|
|
||||||
|
def test_zstd_availability_tracks_runtime(self) -> None:
|
||||||
|
expected: bool = sys.version_info >= (3, 14)
|
||||||
|
assert compression.compression_available("zstd") == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestLevelError:
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "level"),
|
||||||
|
[
|
||||||
|
("deflated", 0),
|
||||||
|
("deflated", 9),
|
||||||
|
("bzip2", 1),
|
||||||
|
("bzip2", 9),
|
||||||
|
("deflated", None),
|
||||||
|
("stored", None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_valid_levels_return_none(self, method: str, level: int | None) -> None:
|
||||||
|
assert compression.level_error(method, level) is None
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "level"),
|
||||||
|
[
|
||||||
|
("deflated", 10),
|
||||||
|
("deflated", -1),
|
||||||
|
("bzip2", 0),
|
||||||
|
("bzip2", 10),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_out_of_range_levels_return_message(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
level: int,
|
||||||
|
) -> None:
|
||||||
|
msg: str | None = compression.level_error(method, level)
|
||||||
|
assert msg is not None
|
||||||
|
assert "between" in msg
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("method", ["stored", "lzma"])
|
||||||
|
def test_level_on_levelless_method_is_rejected(self, method: str) -> None:
|
||||||
|
msg: str | None = compression.level_error(method, 5)
|
||||||
|
assert msg is not None
|
||||||
|
assert "no effect" in msg
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompressTypeReadable:
|
||||||
|
@pytest.mark.parametrize("ct", [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED])
|
||||||
|
def test_stored_and_deflated_always_readable(self, ct: int) -> None:
|
||||||
|
assert compression.compress_type_readable(ct)
|
||||||
|
|
||||||
|
def test_zstd_compress_type_readability_tracks_runtime(self) -> None:
|
||||||
|
# 93 = ZIP_ZSTANDARD; 20 = legacy zstd method id (read-only)
|
||||||
|
expected: bool = sys.version_info >= (3, 14)
|
||||||
|
assert compression.compress_type_readable(93) == expected
|
||||||
|
assert compression.compress_type_readable(20) == expected
|
||||||
|
|
||||||
|
def test_unknown_compress_type_is_unreadable(self) -> None:
|
||||||
|
assert not compression.compress_type_readable(9999)
|
||||||
|
|
||||||
|
def test_unreadable_method_names_lists_methods(self) -> None:
|
||||||
|
# An unknown method id maps to no name and is reported generically.
|
||||||
|
names: set[str] = compression.unreadable_method_names({9999})
|
||||||
|
assert names == {"method 9999"}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run to verify it fails**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/export/test_compression.py -v"`
|
||||||
|
Expected: FAIL with `ModuleNotFoundError: No module named 'documents.export.compression'`.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Implement `compression.py`**
|
||||||
|
|
||||||
|
Create `src/documents/export/compression.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
# ZIP_ZSTANDARD exists only on Python 3.14+ (PEP 784). None elsewhere.
|
||||||
|
ZSTD: int | None = getattr(zipfile, "ZIP_ZSTANDARD", None)
|
||||||
|
|
||||||
|
# CLI choices are fixed across runtimes so argparse never hides zstd; runtime
|
||||||
|
# availability is enforced separately in compression_available().
|
||||||
|
COMPRESSION_CHOICES: tuple[str, ...] = (
|
||||||
|
"stored",
|
||||||
|
"deflated",
|
||||||
|
"bzip2",
|
||||||
|
"lzma",
|
||||||
|
"zstd",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Method name -> zipfile compression constant (zstd only when supported).
|
||||||
|
COMPRESSION_METHODS: dict[str, int] = {
|
||||||
|
"stored": zipfile.ZIP_STORED,
|
||||||
|
"deflated": zipfile.ZIP_DEFLATED,
|
||||||
|
"bzip2": zipfile.ZIP_BZIP2,
|
||||||
|
"lzma": zipfile.ZIP_LZMA,
|
||||||
|
}
|
||||||
|
if ZSTD is not None:
|
||||||
|
COMPRESSION_METHODS["zstd"] = ZSTD
|
||||||
|
|
||||||
|
# Inclusive (min, max) level bounds per method; None => level not applicable.
|
||||||
|
# Verified on CPython 3.14.3.
|
||||||
|
LEVEL_BOUNDS: dict[str, tuple[int, int] | None] = {
|
||||||
|
"stored": None,
|
||||||
|
"deflated": (0, 9),
|
||||||
|
"bzip2": (1, 9),
|
||||||
|
"lzma": None,
|
||||||
|
"zstd": (-131072, 22),
|
||||||
|
}
|
||||||
|
|
||||||
|
# zipfile compress_type id -> method name. 93 = current zstd id, 20 = legacy
|
||||||
|
# zstd id that zipfile can still read.
|
||||||
|
_COMPRESS_TYPE_TO_METHOD: dict[int, str] = {
|
||||||
|
zipfile.ZIP_STORED: "stored",
|
||||||
|
zipfile.ZIP_DEFLATED: "deflated",
|
||||||
|
zipfile.ZIP_BZIP2: "bzip2",
|
||||||
|
zipfile.ZIP_LZMA: "lzma",
|
||||||
|
93: "zstd",
|
||||||
|
20: "zstd",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def compression_available(method: str) -> bool:
|
||||||
|
"""Whether the running interpreter can actually use the given method."""
|
||||||
|
if method in ("stored", "deflated"):
|
||||||
|
# zlib is a hard CPython dependency; stored needs nothing.
|
||||||
|
return True
|
||||||
|
if method == "bzip2":
|
||||||
|
return _module_importable("bz2")
|
||||||
|
if method == "lzma":
|
||||||
|
return _module_importable("lzma")
|
||||||
|
if method == "zstd":
|
||||||
|
return ZSTD is not None and _module_importable("compression.zstd")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _module_importable(name: str) -> bool:
|
||||||
|
try:
|
||||||
|
importlib.import_module(name)
|
||||||
|
except ImportError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def level_error(method: str, level: int | None) -> str | None:
|
||||||
|
"""Return a human message if (method, level) is invalid, else None."""
|
||||||
|
if level is None:
|
||||||
|
return None
|
||||||
|
bounds = LEVEL_BOUNDS[method]
|
||||||
|
if bounds is None:
|
||||||
|
return f"--zip-compression-level has no effect for '{method}'"
|
||||||
|
low, high = bounds
|
||||||
|
if not (low <= level <= high):
|
||||||
|
return (
|
||||||
|
f"--zip-compression-level for '{method}' must be between "
|
||||||
|
f"{low} and {high}"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def compress_type_readable(compress_type: int) -> bool:
|
||||||
|
"""Whether this interpreter can decompress an entry of the given type."""
|
||||||
|
method = _COMPRESS_TYPE_TO_METHOD.get(compress_type)
|
||||||
|
if method is None:
|
||||||
|
return False
|
||||||
|
return compression_available(method)
|
||||||
|
|
||||||
|
|
||||||
|
def unreadable_method_names(compress_types: set[int]) -> set[str]:
|
||||||
|
"""Map a set of compress_type ids to human method names for error messages."""
|
||||||
|
names: set[str] = set()
|
||||||
|
for ct in compress_types:
|
||||||
|
names.add(_COMPRESS_TYPE_TO_METHOD.get(ct, f"method {ct}"))
|
||||||
|
return names
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run to verify it passes**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/export/test_compression.py -v"`
|
||||||
|
Expected: PASS (on the 3.11 VM, `test_zstd_availability_tracks_runtime` and `test_zstd_compress_type_readability_tracks_runtime` assert `False`).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Lint**
|
||||||
|
|
||||||
|
Run: `ruff check src/documents/export/compression.py src/documents/tests/export/test_compression.py && ruff format src/documents/export/compression.py src/documents/tests/export/test_compression.py`
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/documents/export/compression.py src/documents/tests/export/test_compression.py
|
||||||
|
git commit -m "Feature: add export compression policy module"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: `ZipExportSink` accepts compression method + level
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Modify: `src/documents/export/sinks.py`
|
||||||
|
- Test: `src/documents/tests/export/test_sinks.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test**
|
||||||
|
|
||||||
|
Append to `src/documents/tests/export/test_sinks.py` (the top-of-file block already imports `zipfile`, `Path`, `pytest`, `ZipExportSink`, `StreamingManifestWriter` from the base-refactor plan):
|
||||||
|
|
||||||
|
```python
|
||||||
|
class TestZipExportSinkCompression:
|
||||||
|
@pytest.fixture()
|
||||||
|
def source_file(self, tmp_path: Path) -> Path:
|
||||||
|
src: Path = tmp_path / "src" / "doc.pdf"
|
||||||
|
src.parent.mkdir(parents=True)
|
||||||
|
src.write_bytes(b"PDF-CONTENT" * 100)
|
||||||
|
return src
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "constant"),
|
||||||
|
[
|
||||||
|
("stored", zipfile.ZIP_STORED),
|
||||||
|
("deflated", zipfile.ZIP_DEFLATED),
|
||||||
|
("bzip2", zipfile.ZIP_BZIP2),
|
||||||
|
("lzma", zipfile.ZIP_LZMA),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_compression_method_is_applied_to_file_entries(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
method: str,
|
||||||
|
constant: int,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with ZipExportSink(
|
||||||
|
target,
|
||||||
|
"export",
|
||||||
|
delete=False,
|
||||||
|
compression=constant,
|
||||||
|
) as sink:
|
||||||
|
sink.add_file(source_file, "doc.pdf")
|
||||||
|
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||||
|
info = zf.getinfo("doc.pdf")
|
||||||
|
assert info.compress_type == constant
|
||||||
|
|
||||||
|
def test_compressing_method_beats_stored(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
# Robust size invariant: a compressing method must be <= stored on
|
||||||
|
# compressible content (avoids flaky level-9-vs-level-1 comparisons).
|
||||||
|
sizes: dict[str, int] = {}
|
||||||
|
for name, constant in (("stored", zipfile.ZIP_STORED), ("deflated", zipfile.ZIP_DEFLATED)):
|
||||||
|
target: Path = tmp_path / name
|
||||||
|
target.mkdir()
|
||||||
|
with ZipExportSink(target, "export", delete=False, compression=constant) as sink:
|
||||||
|
sink.add_file(source_file, "doc.pdf")
|
||||||
|
sizes[name] = (target / "export.zip").stat().st_size
|
||||||
|
assert sizes["deflated"] <= sizes["stored"]
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run to verify it fails**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/export/test_sinks.py::TestZipExportSinkCompression -v"`
|
||||||
|
Expected: FAIL with `TypeError: __init__() got an unexpected keyword argument 'compression'`.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Add the params to `ZipExportSink`**
|
||||||
|
|
||||||
|
In `src/documents/export/sinks.py`, change `ZipExportSink.__init__` to accept the new keyword-only params and store them, and pass them in `_open()`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
target: Path,
|
||||||
|
zip_name: str,
|
||||||
|
*,
|
||||||
|
delete: bool = False,
|
||||||
|
compression: int = zipfile.ZIP_DEFLATED,
|
||||||
|
compresslevel: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._target = target.resolve()
|
||||||
|
self._zip_path = (self._target / zip_name).with_suffix(".zip")
|
||||||
|
self._tmp_path = self._zip_path.with_name(self._zip_path.name + ".tmp")
|
||||||
|
self._delete = delete
|
||||||
|
self._compression = compression
|
||||||
|
self._compresslevel = compresslevel
|
||||||
|
self._zip: zipfile.ZipFile | None = None
|
||||||
|
self._dirs: set[str] = set()
|
||||||
|
self._pending_manifest: tuple[Path, str] | None = None
|
||||||
|
self._stream_open = False
|
||||||
|
```
|
||||||
|
|
||||||
|
And in `_open()`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def _open(self) -> None:
|
||||||
|
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._zip = zipfile.ZipFile(
|
||||||
|
self._tmp_path,
|
||||||
|
"w",
|
||||||
|
compression=self._compression,
|
||||||
|
compresslevel=self._compresslevel,
|
||||||
|
allowZip64=True,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run to verify it passes**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/export/test_sinks.py -v"`
|
||||||
|
Expected: PASS (all sink tests, including the four method params and the size invariant). `bzip2`/`lzma` are present on the VM's CPython, so those params pass.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Lint**
|
||||||
|
|
||||||
|
Run: `ruff check src/documents/export/sinks.py && ruff format src/documents/export/sinks.py`
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/documents/export/sinks.py src/documents/tests/export/test_sinks.py
|
||||||
|
git commit -m "Feature: ZipExportSink accepts compression method and level"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: Wire CLI flags + validation into `document_exporter`
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Modify: `src/documents/management/commands/document_exporter.py`
|
||||||
|
- Test: `src/documents/tests/test_management_exporter.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add the argparse flags**
|
||||||
|
|
||||||
|
In `document_exporter.py`, add the import near the other `documents.export` import:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from documents.export.compression import COMPRESSION_CHOICES
|
||||||
|
from documents.export.compression import COMPRESSION_METHODS
|
||||||
|
from documents.export.compression import compression_available
|
||||||
|
from documents.export.compression import level_error
|
||||||
|
from documents.export.compression import ZSTD
|
||||||
|
```
|
||||||
|
|
||||||
|
In `add_arguments`, after the `--zip-name` argument, add:
|
||||||
|
|
||||||
|
```python
|
||||||
|
parser.add_argument(
|
||||||
|
"--zip-compression",
|
||||||
|
choices=COMPRESSION_CHOICES,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Compression method for the export zip (requires --zip). "
|
||||||
|
"Default: deflated. 'zstd' requires Python 3.14+ on both the "
|
||||||
|
"exporting and importing machine."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--zip-compression-level",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Compression level for the export zip (requires --zip). "
|
||||||
|
"deflated: 0-9, bzip2: 1-9, zstd: -131072..22; ignored for "
|
||||||
|
"stored/lzma."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Read + validate the flags in `handle()`**
|
||||||
|
|
||||||
|
In `handle()`, after the existing `--compare-*` + `--zip` guard, add the compression flag handling. Insert before the sink construction:
|
||||||
|
|
||||||
|
```python
|
||||||
|
zip_compression: str | None = options["zip_compression"]
|
||||||
|
zip_compression_level: int | None = options["zip_compression_level"]
|
||||||
|
|
||||||
|
if not self.zip_export and (
|
||||||
|
zip_compression is not None or zip_compression_level is not None
|
||||||
|
):
|
||||||
|
raise CommandError(
|
||||||
|
"--zip-compression and --zip-compression-level require --zip",
|
||||||
|
)
|
||||||
|
|
||||||
|
compression_method = zip_compression or "deflated"
|
||||||
|
if self.zip_export:
|
||||||
|
if not compression_available(compression_method):
|
||||||
|
if compression_method == "zstd" and ZSTD is None:
|
||||||
|
raise CommandError(
|
||||||
|
"zstd compression requires Python 3.14 or newer",
|
||||||
|
)
|
||||||
|
raise CommandError(
|
||||||
|
f"Compression method '{compression_method}' is not "
|
||||||
|
f"available on this Python runtime",
|
||||||
|
)
|
||||||
|
level_msg = level_error(compression_method, zip_compression_level)
|
||||||
|
if level_msg is not None:
|
||||||
|
raise CommandError(level_msg)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Pass the resolved values into `ZipExportSink`**
|
||||||
|
|
||||||
|
Change the `ZipExportSink(...)` construction in `handle()` to:
|
||||||
|
|
||||||
|
```python
|
||||||
|
if self.zip_export:
|
||||||
|
sink = ZipExportSink(
|
||||||
|
self.target,
|
||||||
|
options["zip_name"],
|
||||||
|
delete=self.delete,
|
||||||
|
compression=COMPRESSION_METHODS[compression_method],
|
||||||
|
compresslevel=zip_compression_level,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sink = DirectoryExportSink(
|
||||||
|
self.target,
|
||||||
|
compare_checksums=self.compare_checksums,
|
||||||
|
compare_json=self.compare_json,
|
||||||
|
delete=self.delete,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Write the command-level tests**
|
||||||
|
|
||||||
|
Add to the `TestExportImport` class in `src/documents/tests/test_management_exporter.py` (imports `call_command`, `CommandError`, `ZipFile`, `timezone` already present):
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_compression_flags_require_zip(self) -> None:
|
||||||
|
for args in (
|
||||||
|
["--zip-compression", "lzma"],
|
||||||
|
["--zip-compression-level", "5"],
|
||||||
|
):
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
*args,
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_compression_level_out_of_range_raises(self) -> None:
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"deflated",
|
||||||
|
"--zip-compression-level",
|
||||||
|
"99",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_compression_level_rejected_for_stored(self) -> None:
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"stored",
|
||||||
|
"--zip-compression-level",
|
||||||
|
"5",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_lzma_compression_round_trips(self) -> None:
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"lzma",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
expected = str(
|
||||||
|
self.target / f"export-{timezone.localdate().isoformat()}.zip",
|
||||||
|
)
|
||||||
|
self.assertIsFile(expected)
|
||||||
|
with ZipFile(expected) as zip_file:
|
||||||
|
info = zip_file.getinfo("manifest.json")
|
||||||
|
# manifest.json carries the chosen method; deflated is the default
|
||||||
|
self.assertEqual(info.compress_type, 14) # ZIP_LZMA
|
||||||
|
|
||||||
|
def test_default_zip_uses_deflate(self) -> None:
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
expected = str(
|
||||||
|
self.target / f"export-{timezone.localdate().isoformat()}.zip",
|
||||||
|
)
|
||||||
|
with ZipFile(expected) as zip_file:
|
||||||
|
info = zip_file.getinfo("manifest.json")
|
||||||
|
self.assertEqual(info.compress_type, 8) # ZIP_DEFLATED
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Run the tests**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/test_management_exporter.py -v"`
|
||||||
|
Expected: PASS — the new tests plus all existing exporter tests stay green.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Lint**
|
||||||
|
|
||||||
|
Run: `ruff check src/documents/management/commands/document_exporter.py src/documents/tests/test_management_exporter.py && ruff format src/documents/management/commands/document_exporter.py src/documents/tests/test_management_exporter.py`
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 7: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/documents/management/commands/document_exporter.py src/documents/tests/test_management_exporter.py
|
||||||
|
git commit -m "Feature: add --zip-compression and --zip-compression-level flags"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: Importer pre-check for unreadable codecs
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Modify: `src/documents/management/commands/document_importer.py`
|
||||||
|
- Test: `src/documents/tests/test_management_importer.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test**
|
||||||
|
|
||||||
|
The importer test file `src/documents/tests/test_management_importer.py` is
|
||||||
|
`TestCase`-style (`class TestCommandImport(... TestCase)`, `self.assertRaises`,
|
||||||
|
`DirectoriesMixin` gives `self.dirs.scratch_dir`). Match that style. Add this
|
||||||
|
method to `TestCommandImport`. It builds a valid zip and patches the readability
|
||||||
|
probe so the check fires deterministically on any runtime:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_import_rejects_unreadable_compression(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A zip archive with an entry whose compression this Python can't read
|
||||||
|
WHEN:
|
||||||
|
- Import is attempted
|
||||||
|
THEN:
|
||||||
|
- A CommandError naming the issue is raised, before extraction
|
||||||
|
"""
|
||||||
|
import zipfile
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
archive = Path(self.dirs.scratch_dir) / "export.zip"
|
||||||
|
with zipfile.ZipFile(archive, "w") as zf:
|
||||||
|
zf.writestr("manifest.json", "[]")
|
||||||
|
|
||||||
|
with mock.patch(
|
||||||
|
"documents.management.commands.document_importer.compress_type_readable",
|
||||||
|
return_value=False,
|
||||||
|
):
|
||||||
|
with self.assertRaises(CommandError) as e:
|
||||||
|
call_command(
|
||||||
|
"document_importer",
|
||||||
|
str(archive),
|
||||||
|
"--no-progress-bar",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
self.assertIn("compression", str(e.exception))
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run to verify it fails**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/test_management_importer.py -k unreadable_compression -v"`
|
||||||
|
Expected: FAIL — no pre-check exists yet, so the import proceeds (or fails with a different error).
|
||||||
|
|
||||||
|
- [ ] **Step 3: Implement the pre-check**
|
||||||
|
|
||||||
|
In `document_importer.py`, add the import:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from documents.export.compression import compress_type_readable
|
||||||
|
from documents.export.compression import unreadable_method_names
|
||||||
|
```
|
||||||
|
|
||||||
|
Find the zip-handling block (around `document_importer.py:453`):
|
||||||
|
|
||||||
|
```python
|
||||||
|
with ZipFile(self.source) as zf:
|
||||||
|
zf.extractall(tmp_dir)
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace it with a pre-check before extraction:
|
||||||
|
|
||||||
|
```python
|
||||||
|
with ZipFile(self.source) as zf:
|
||||||
|
unsupported = {
|
||||||
|
info.compress_type
|
||||||
|
for info in zf.infolist()
|
||||||
|
if not compress_type_readable(info.compress_type)
|
||||||
|
}
|
||||||
|
if unsupported:
|
||||||
|
names = ", ".join(sorted(unreadable_method_names(unsupported)))
|
||||||
|
raise CommandError(
|
||||||
|
f"This archive uses compression this Python cannot "
|
||||||
|
f"read ({names}). zstd archives require Python 3.14+.",
|
||||||
|
)
|
||||||
|
zf.extractall(tmp_dir)
|
||||||
|
```
|
||||||
|
|
||||||
|
Confirm `CommandError` is imported in `document_importer.py` (it is used elsewhere; if not, add `from django.core.management.base import CommandError`).
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run to verify it passes**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/test_management_importer.py -v"`
|
||||||
|
Expected: PASS — the new test plus all existing importer tests (normal deflated/stored archives still import).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Lint**
|
||||||
|
|
||||||
|
Run: `ruff check src/documents/management/commands/document_importer.py src/documents/tests/test_management_importer.py && ruff format src/documents/management/commands/document_importer.py src/documents/tests/test_management_importer.py`
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/documents/management/commands/document_importer.py src/documents/tests/test_management_importer.py
|
||||||
|
git commit -m "Feature: importer rejects archives with unreadable compression"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: Document the flags
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Modify: `docs/administration.md`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add the flags to the option list**
|
||||||
|
|
||||||
|
In `docs/administration.md`, update the usage block (around line 257) to include the new flags:
|
||||||
|
|
||||||
|
```
|
||||||
|
document_exporter target [-c] [-d] [-f] [-na] [-nt] [-p] [-sm] [-z]
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-c, --compare-checksums
|
||||||
|
-cj, --compare-json
|
||||||
|
-d, --delete
|
||||||
|
-f, --use-filename-format
|
||||||
|
-na, --no-archive
|
||||||
|
-nt, --no-thumbnail
|
||||||
|
-p, --use-folder-prefix
|
||||||
|
-sm, --split-manifest
|
||||||
|
-z, --zip
|
||||||
|
-zn, --zip-name
|
||||||
|
--zip-compression
|
||||||
|
--zip-compression-level
|
||||||
|
--data-only
|
||||||
|
--no-progress-bar
|
||||||
|
--passphrase
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Add the prose**
|
||||||
|
|
||||||
|
After the `-z`/`--zip` paragraph (around line 330), add:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
The compression method for the zip can be set with `--zip-compression`
|
||||||
|
(`stored`, `deflated` (default), `bzip2`, `lzma`, or `zstd`) and tuned with
|
||||||
|
`--zip-compression-level` (deflated: 0–9, bzip2: 1–9, zstd: -131072–22; ignored
|
||||||
|
for `stored` and `lzma`). Both options require `--zip`.
|
||||||
|
|
||||||
|
!!! warning
|
||||||
|
|
||||||
|
`zstd` compression requires Python 3.14 or newer on **both** the machine
|
||||||
|
creating the export and any machine importing it. An archive compressed with
|
||||||
|
`zstd` (or `lzma`/`bzip2` where those modules are unavailable) cannot be
|
||||||
|
imported on a runtime that lacks the codec; the importer will refuse it with
|
||||||
|
a clear error. The default `deflated` is universally readable.
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify the docs build is not broken (lint markdown)**
|
||||||
|
|
||||||
|
Run: `ruff check docs/ 2>/dev/null; echo "docs are markdown; rely on prettier pre-commit"`
|
||||||
|
(No code to test. The prettier pre-commit hook will reformat on commit.)
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add docs/administration.md
|
||||||
|
git commit -m "Docs: document --zip-compression and --zip-compression-level"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 6: Final verification
|
||||||
|
|
||||||
|
**Files:** none (verification only).
|
||||||
|
|
||||||
|
- [ ] **Step 1: Full backend suites on the VM**
|
||||||
|
|
||||||
|
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/export/ src/documents/tests/test_management_exporter.py src/documents/tests/test_management_importer.py -v"`
|
||||||
|
Expected: PASS, no failures.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Spot-check the zstd happy path on Python 3.14 (cannot run under Django on the 3.11 VM)**
|
||||||
|
|
||||||
|
The zstd positive round-trip can't run in the 3.11 test env. Confirm the policy module behaves on a real 3.14 interpreter with a standalone check (no Django needed):
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run --python 3.14 --no-project python -c "import sys; sys.path.insert(0,'src'); import django; print('skip')" 2>/dev/null || \
|
||||||
|
uv run --python 3.14 --no-project python -c "
|
||||||
|
import zipfile, io
|
||||||
|
from compression.zstd import CompressionParameter as CP
|
||||||
|
print('zstd const', zipfile.ZIP_ZSTANDARD, 'bounds', CP.compression_level.bounds())
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(buf,'w',compression=zipfile.ZIP_ZSTANDARD,compresslevel=19) as zf:
|
||||||
|
zf.writestr('a.txt','x'*1000)
|
||||||
|
with zipfile.ZipFile(buf) as zf:
|
||||||
|
assert zf.getinfo('a.txt').compress_type == zipfile.ZIP_ZSTANDARD
|
||||||
|
assert zf.read('a.txt') == b'x'*1000
|
||||||
|
print('zstd round-trip OK')
|
||||||
|
"
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: prints `zstd const 93 bounds (-131072, 22)` and `zstd round-trip OK`. This validates the constant, bounds, and that a zstd archive round-trips — the parts the 3.11 CI cannot exercise.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Type-check on the VM (pyrefly)**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tar czf - src pyproject.toml uv.lock .pyrefly-baseline.json | ssh -o BatchMode=yes -p 2244 trenton@localhost 'tar xzf - -C ~/projects/paperless-ngx'
|
||||||
|
ssh -o BatchMode=yes -p 2244 trenton@localhost 'bash -lc "cd ~/projects/paperless-ngx && uv run pyrefly check"'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no new type errors beyond the baseline. (Note: `import compression.zstd` is guarded behind `importlib.import_module`, so it is never statically resolved on the 3.11 baseline.)
|
||||||
|
|
||||||
|
- [ ] **Step 4: Final lint**
|
||||||
|
|
||||||
|
Run: `ruff check src/documents/export/ src/documents/management/commands/document_exporter.py src/documents/management/commands/document_importer.py && ruff format --check src/documents/export/ src/documents/management/commands/document_exporter.py src/documents/management/commands/document_importer.py`
|
||||||
|
Expected: clean.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Notes for the implementer
|
||||||
|
|
||||||
|
- **Default behavior is unchanged:** with no flags, the sink is constructed with `compression=ZIP_DEFLATED, compresslevel=None` — byte-method-identical to today (`shutil.make_archive` used `ZIP_DEFLATED` with no level). `test_default_zip_uses_deflate` pins this.
|
||||||
|
- **zstd availability is gated three ways and never imported statically:** the constant via `getattr`, the codec via `importlib.import_module("compression.zstd")`, and the CLI value rejected with a friendly message on < 3.14. The choices list always contains `zstd` so argparse doesn't hide it.
|
||||||
|
- **The importer pre-check is the safety net** for portability foot-guns — without it an unreadable entry raises a bare `NotImplementedError` mid-`extractall`. The check runs on `infolist()` (metadata only) before any extraction.
|
||||||
|
- **Why `--zip-compression` defaults to `None`, not `"deflated"`:** so `handle()` can detect "user passed it without `--zip`" and fail fast. The effective default is resolved as `zip_compression or "deflated"`.
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
# Export Zip Compression Control — Design
|
||||||
|
|
||||||
|
**Date:** 2026-06-16
|
||||||
|
**Branch base:** `dev`
|
||||||
|
**Status:** Design complete (zstd facts verified on CPython 3.14.3) — **depends on**
|
||||||
|
`2026-06-16-export-sink-architecture-design.md` being implemented first.
|
||||||
|
|
||||||
|
## Prerequisite
|
||||||
|
|
||||||
|
This builds directly on the export sink refactor. It assumes `ZipExportSink`
|
||||||
|
already exists and is the single place that owns `zipfile.ZipFile` creation and
|
||||||
|
entry writes. Do not start this until that refactor has landed; without it, the
|
||||||
|
change would have to touch the command's zip branches again.
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Zip export is hardwired to `ZIP_DEFLATED` at the library default level. Users
|
||||||
|
have no way to trade speed against archive size — a fast `ZIP_STORED` pass for a
|
||||||
|
quick local copy, or a maximal `ZIP_LZMA` pass for the smallest off-site backup.
|
||||||
|
The sink refactor turns "which compression" into a single constructor argument,
|
||||||
|
so exposing it is now a small, isolated change.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Let the operator choose the zip compression method and level from the CLI, with
|
||||||
|
behavior identical to today when the flags are omitted. All knowledge of
|
||||||
|
compression stays inside `ZipExportSink`; the command only parses flags and maps
|
||||||
|
them to sink arguments.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
In scope:
|
||||||
|
|
||||||
|
- `ZipExportSink` gains `compression: int` and `compresslevel: int | None`
|
||||||
|
constructor parameters (default `ZIP_DEFLATED`, `None` → library default),
|
||||||
|
passed straight to `zipfile.ZipFile(...)`.
|
||||||
|
- New `document_exporter` flags: `--zip-compression` and
|
||||||
|
`--zip-compression-level`, valid only with `--zip`.
|
||||||
|
- Validation: method availability, level range per method, and the
|
||||||
|
requires-`--zip` guard.
|
||||||
|
- Import-side: a pre-extract support check in `document_importer` that turns an
|
||||||
|
unsupported codec into a clear `CommandError` (the importer otherwise decompresses
|
||||||
|
transparently via `ZipFile.extractall`).
|
||||||
|
- Docs: add both flags and the zstd-portability caveat to `docs/administration.md`
|
||||||
|
(the `document_exporter` option list, lines ~257-270 and the `-z`/`-zn` section,
|
||||||
|
lines ~328-330). New flags are long-form only (`--zip-compression`,
|
||||||
|
`--zip-compression-level`) — no short aliases, to avoid `-zc`/`-zl` collisions
|
||||||
|
with the existing `-z`/`-zn`.
|
||||||
|
|
||||||
|
Out of scope:
|
||||||
|
|
||||||
|
- Compression for any non-zip sink (folder has none; a future S3 sink would
|
||||||
|
handle its own object storage compression separately).
|
||||||
|
- Changing the default. Omitting the flags must produce a byte-compatible-method
|
||||||
|
archive to today's (`ZIP_DEFLATED`, default level).
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### `ZipExportSink` changes
|
||||||
|
|
||||||
|
The base sink's signature is `ZipExportSink(target, zip_name, *, delete)`; this
|
||||||
|
adds two keyword-only params after `delete`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
target: Path,
|
||||||
|
zip_name: str,
|
||||||
|
*,
|
||||||
|
delete: bool = False,
|
||||||
|
compression: int = zipfile.ZIP_DEFLATED,
|
||||||
|
compresslevel: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
...
|
||||||
|
# opened in __enter__:
|
||||||
|
self._zip = zipfile.ZipFile(
|
||||||
|
self._tmp_path,
|
||||||
|
"w",
|
||||||
|
compression=compression,
|
||||||
|
compresslevel=compresslevel,
|
||||||
|
allowZip64=True,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
`ZipFile` applies `compression`/`compresslevel` as the default for every
|
||||||
|
`write`/`writestr` (verified: a `ZipFile(..., compression=ZIP_BZIP2)` yields
|
||||||
|
entries with `compress_type == ZIP_BZIP2` without per-call args), so `add_file` /
|
||||||
|
`add_json` / the manifest entry need no changes. Directory marker entries are
|
||||||
|
empty so their compressed payload is zero, but they are still _tagged_ with the
|
||||||
|
chosen `compress_type` — harmless, but tests that read `infolist()` should filter
|
||||||
|
or account for marker entries (see Testing).
|
||||||
|
|
||||||
|
### CLI flags (`document_exporter`)
|
||||||
|
|
||||||
|
- `--zip-compression {stored,deflated,bzip2,lzma}` — and `zstd` **when the
|
||||||
|
runtime supports it** (see below). Maps to the matching `zipfile.ZIP_*`
|
||||||
|
constant. Default `deflated`.
|
||||||
|
- `--zip-compression-level N` — integer. Per-method accepted ranges (verified
|
||||||
|
against the [3.14 `zipfile` docs](https://docs.python.org/3.14/library/zipfile.html#zipfile.ZipFile)):
|
||||||
|
- `deflated`: **0–9** (`zlib` also accepts `-1` = "default", identical to
|
||||||
|
omitting the flag / `compresslevel=None`).
|
||||||
|
- `bzip2`: **1–9** (`0` is invalid for bzip2).
|
||||||
|
- `lzma`, `stored`: level has **no effect** — passing `--zip-compression-level`
|
||||||
|
with either is a `CommandError`, not a silent accept (consistent with the
|
||||||
|
base refactor's fail-fast posture).
|
||||||
|
- `zstd`: **-131072 … 22** (the documented commonly-accepted range; the
|
||||||
|
authoritative bounds are
|
||||||
|
`compression.zstd.CompressionParameter.compression_level.bounds()`).
|
||||||
|
|
||||||
|
Default: unset → library default (`compresslevel=None`).
|
||||||
|
|
||||||
|
Both flags require `--zip`; passing either without `--zip` raises a
|
||||||
|
`CommandError`, matching the incremental-flag rule from the base refactor.
|
||||||
|
|
||||||
|
**Why validate up front (not let `zipfile` raise) — verified on 3.14.3:** an
|
||||||
|
invalid level does _not_ fail at `ZipFile(...)` construction — it fails at the
|
||||||
|
**first `write`/`writestr` call**, with an opaque message
|
||||||
|
(`ValueError: Invalid initialization option` for deflated > 9, or
|
||||||
|
`ValueError: compresslevel must be between 1 and 9` for bzip2). Worse, on context
|
||||||
|
exit the half-initialized write handle emits a secondary
|
||||||
|
`AttributeError: '_ZipWriteFile' object has no attribute '_compressor'` during GC
|
||||||
|
finalization, so the user sees stack-trace noise unrelated to the real cause.
|
||||||
|
Up-front validation turns all of that into a single clean `CommandError`.
|
||||||
|
|
||||||
|
### Validation (in `handle()`, before constructing the sink)
|
||||||
|
|
||||||
|
1. **Requires `--zip`.** Either flag without `--zip` → `CommandError`.
|
||||||
|
2. **Method availability — via a named, patchable seam.** Expose a module-level
|
||||||
|
helper `compression_available(method: str) -> bool` that does
|
||||||
|
`try: import bz2 / import lzma / from compression import zstd except ImportError:
|
||||||
|
return False` — **not** `importlib.util.find_spec`, which can report a stdlib
|
||||||
|
C-extension as present when importing it actually fails. `stored`/`deflated`
|
||||||
|
are always available (`zlib` is a hard CPython dependency). For `zstd` the probe
|
||||||
|
must import `compression.zstd` (3.14+), not merely check that
|
||||||
|
`zipfile.ZIP_ZSTANDARD` exists. Making this a named function is also what lets
|
||||||
|
the test patch "method unavailable" with `mocker`. If the chosen method is
|
||||||
|
unavailable, raise a `CommandError` naming the missing capability — `zipfile`
|
||||||
|
itself would otherwise raise a bare `RuntimeError`
|
||||||
|
("Compression requires the (missing) … module").
|
||||||
|
3. **Level range.** Reject an out-of-range `--zip-compression-level` for the
|
||||||
|
chosen method with a clear `CommandError`; reject the flag entirely for
|
||||||
|
`stored`/`lzma` (see above).
|
||||||
|
|
||||||
|
### zstd (Python 3.14+)
|
||||||
|
|
||||||
|
**Verified empirically on CPython 3.14.3** (via `uv run --python 3.14 --no-project`)
|
||||||
|
and against [PEP 784](https://peps.python.org/pep-0784/) +
|
||||||
|
[the 3.14 `zipfile` docs](https://docs.python.org/3.14/library/zipfile.html):
|
||||||
|
|
||||||
|
- The compression-method constant is **`zipfile.ZIP_ZSTANDARD`** (added 3.14; its
|
||||||
|
numeric value is `93`). It does **not** exist on < 3.14.
|
||||||
|
- It is backed by the new **`compression.zstd`** stdlib module (PEP 784 added a
|
||||||
|
`compression` namespace package; legacy `bz2`/`lzma`/`zlib` imports are
|
||||||
|
unchanged). `zipfile` raises `RuntimeError` if `compression.zstd` is
|
||||||
|
unavailable when zstd is requested.
|
||||||
|
- Accepted `compresslevel` is **`-131072 … 22`**, confirmed at runtime via
|
||||||
|
`compression.zstd.CompressionParameter.compression_level.bounds() == (-131072, 22)`.
|
||||||
|
|
||||||
|
Gate everything zstd-related at runtime so nothing is imported or referenced on
|
||||||
|
< 3.14 (the project targets Python ≥ 3.11):
|
||||||
|
|
||||||
|
```python
|
||||||
|
_ZSTD: int | None = getattr(zipfile, "ZIP_ZSTANDARD", None) # None before 3.14
|
||||||
|
```
|
||||||
|
|
||||||
|
Presence of the _constant_ does not guarantee the _codec_ is usable, so the
|
||||||
|
availability probe (validation step 2) imports `compression.zstd`, not merely
|
||||||
|
checks the constant.
|
||||||
|
|
||||||
|
Keep `zstd` in the `--zip-compression` `choices` **always** (even on < 3.14), and
|
||||||
|
reject it in validation with a friendly "zstd requires Python 3.14+" message. If
|
||||||
|
it were dropped from `choices` on older runtimes, argparse would emit a generic
|
||||||
|
"invalid choice" that reads as though the option never existed — worse UX.
|
||||||
|
|
||||||
|
### Import-side compatibility
|
||||||
|
|
||||||
|
`document_importer` reads zips with `ZipFile(self.source).extractall(...)`
|
||||||
|
(`document_importer.py:453`), which decompresses each entry transparently using
|
||||||
|
whatever method it was stored with — **provided the matching module exists on the
|
||||||
|
importing machine.**
|
||||||
|
|
||||||
|
The failure mode when it doesn't is unfriendly and must be handled: a zstd (or
|
||||||
|
otherwise unsupported) entry raises a bare `NotImplementedError` **per-entry,
|
||||||
|
during `extractall`** — _not_ at `ZipFile(self.source)` open, and `is_zipfile()`
|
||||||
|
still returns true (a zstd archive is a valid zip container). So the importer
|
||||||
|
enters the zip branch, creates its temp dir, may partially extract other entries,
|
||||||
|
then blows up mid-extract with no context. **Mitigation (in scope here):** before
|
||||||
|
extracting, inspect `ZipFile(self.source).infolist()` compress types and, if any
|
||||||
|
is unsupported on this runtime, raise a `CommandError` naming the method and the
|
||||||
|
requirement (e.g. "this archive uses zstd, which needs Python 3.14+") instead of
|
||||||
|
letting `NotImplementedError` escape.
|
||||||
|
|
||||||
|
Per-method summary (document in help text + `administration.md`):
|
||||||
|
|
||||||
|
- `deflated`/`stored`: universally importable.
|
||||||
|
- `bzip2`/`lzma`: importable wherever the `bz2`/`lzma` modules are present
|
||||||
|
(essentially always).
|
||||||
|
- `zstd`: importable only on Python 3.14+. An archive compressed with `zstd` is
|
||||||
|
**not** importable on older runtimes.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
New cases in the sink tests and an export→import round-trip
|
||||||
|
(pytest classes, factory-boy, `mocker`, `parametrize`, typed; run on the VM):
|
||||||
|
|
||||||
|
- **Round-trip per method.** Parametrize over the available methods (skip `zstd`
|
||||||
|
below 3.14, skip `bzip2`/`lzma` if the module is somehow absent): export a
|
||||||
|
small library, import it back, assert documents/manifest match.
|
||||||
|
- **Method is applied.** Assert each written _file_ entry's `compress_type`
|
||||||
|
equals the requested method (read back via `ZipFile.infolist()`), filtering out
|
||||||
|
directory marker entries (which are tagged but empty).
|
||||||
|
- **Level affects size — robustly.** Do **not** compare deflate level 9 vs 1
|
||||||
|
(on small or incompressible fixtures level 9 can equal or slightly exceed level
|
||||||
|
1, causing flaky CI). Instead assert that a compressing method on a
|
||||||
|
moderately-compressible fixture yields a total smaller than `stored`
|
||||||
|
(`ZIP_STORED`), which is a stable invariant.
|
||||||
|
- **Validation.** Each flag without `--zip` → `CommandError`; out-of-range level
|
||||||
|
(`--zip-compression-level 99`) → a clean `CommandError` from validation
|
||||||
|
(asserting we never reach the `writestr` that would raise the masked
|
||||||
|
`ValueError`); `--zip-compression-level` with `stored`/`lzma` → `CommandError`;
|
||||||
|
unavailable method (patch the named availability seam with `mocker`) →
|
||||||
|
`CommandError`; on < 3.14, `--zip-compression zstd` → the friendly
|
||||||
|
"requires 3.14+" `CommandError`.
|
||||||
|
- **Import pre-check.** An archive containing an unsupported compress type
|
||||||
|
produces a `CommandError` from the importer naming the method, not a raw
|
||||||
|
`NotImplementedError` (simulate by patching the importer's support probe).
|
||||||
|
- **Default unchanged.** Omitting both flags yields file entries with
|
||||||
|
`compress_type == ZIP_DEFLATED`, identical to pre-feature behavior.
|
||||||
|
|
||||||
|
## Risks
|
||||||
|
|
||||||
|
- **Foot-gun archives.** A user could produce a `zstd`/`lzma` archive their
|
||||||
|
import target can't read. Mitigation: explicit help text and the import-side
|
||||||
|
notes above; the default stays the universally-readable `deflated`.
|
||||||
|
- **Optional-module assumptions.** Don't assume `bz2`/`lzma` are always compiled
|
||||||
|
in; probe and error clearly. Mitigation: the availability validation step.
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
# ZIP_ZSTANDARD exists only on Python 3.14+ (PEP 784). None elsewhere.
|
||||||
|
ZSTD: int | None = getattr(zipfile, "ZIP_ZSTANDARD", None)
|
||||||
|
|
||||||
|
# CLI choices are fixed across runtimes so argparse never hides zstd; runtime
|
||||||
|
# availability is enforced separately in compression_available().
|
||||||
|
COMPRESSION_CHOICES: tuple[str, ...] = (
|
||||||
|
"stored",
|
||||||
|
"deflated",
|
||||||
|
"bzip2",
|
||||||
|
"lzma",
|
||||||
|
"zstd",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Method name -> zipfile compression constant (zstd only when supported).
|
||||||
|
COMPRESSION_METHODS: dict[str, int] = {
|
||||||
|
"stored": zipfile.ZIP_STORED,
|
||||||
|
"deflated": zipfile.ZIP_DEFLATED,
|
||||||
|
"bzip2": zipfile.ZIP_BZIP2,
|
||||||
|
"lzma": zipfile.ZIP_LZMA,
|
||||||
|
}
|
||||||
|
if ZSTD is not None:
|
||||||
|
COMPRESSION_METHODS["zstd"] = ZSTD
|
||||||
|
|
||||||
|
# Inclusive (min, max) level bounds per method; None => level not applicable.
|
||||||
|
# Verified on CPython 3.14.3.
|
||||||
|
#
|
||||||
|
# zstd's raw library bounds are (-131072, 22)
|
||||||
|
# (compression.zstd.CompressionParameter.compression_level.bounds()) — the
|
||||||
|
# minimum is an internal implementation constant (-ZSTD_TARGETLENGTH_MAX),
|
||||||
|
# not a meaningful distinct "level"; deeper negative values than -22 buy
|
||||||
|
# nothing over -22 in practice. We expose the conventional zstd CLI range
|
||||||
|
# instead of the raw library bounds.
|
||||||
|
LEVEL_BOUNDS: dict[str, tuple[int, int] | None] = {
|
||||||
|
"stored": None,
|
||||||
|
"deflated": (0, 9),
|
||||||
|
"bzip2": (1, 9),
|
||||||
|
"lzma": None,
|
||||||
|
"zstd": (-22, 22),
|
||||||
|
}
|
||||||
|
|
||||||
|
# zipfile compress_type id -> method name. 93 = current zstd id, 20 = legacy
|
||||||
|
# zstd id that zipfile can still read.
|
||||||
|
_COMPRESS_TYPE_TO_METHOD: dict[int, str] = {
|
||||||
|
zipfile.ZIP_STORED: "stored",
|
||||||
|
zipfile.ZIP_DEFLATED: "deflated",
|
||||||
|
zipfile.ZIP_BZIP2: "bzip2",
|
||||||
|
zipfile.ZIP_LZMA: "lzma",
|
||||||
|
93: "zstd",
|
||||||
|
20: "zstd",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def compression_available(method: str) -> bool:
|
||||||
|
"""Whether the running interpreter can actually use the given method."""
|
||||||
|
if method in ("stored", "deflated"):
|
||||||
|
# zlib is a hard CPython dependency; stored needs nothing.
|
||||||
|
return True
|
||||||
|
if method == "bzip2":
|
||||||
|
return _module_importable("bz2")
|
||||||
|
if method == "lzma":
|
||||||
|
return _module_importable("lzma")
|
||||||
|
if method == "zstd":
|
||||||
|
return ZSTD is not None and _module_importable("compression.zstd")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _module_importable(name: str) -> bool:
|
||||||
|
try:
|
||||||
|
importlib.import_module(name)
|
||||||
|
except ImportError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def level_error(method: str, level: int | None) -> str | None:
|
||||||
|
"""Return a human message if (method, level) is invalid, else None."""
|
||||||
|
if level is None:
|
||||||
|
return None
|
||||||
|
bounds = LEVEL_BOUNDS[method]
|
||||||
|
if bounds is None:
|
||||||
|
return f"--zip-compression-level has no effect for '{method}'"
|
||||||
|
low, high = bounds
|
||||||
|
if not (low <= level <= high):
|
||||||
|
return (
|
||||||
|
f"--zip-compression-level for '{method}' must be between {low} and {high}"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def compress_type_readable(compress_type: int) -> bool:
|
||||||
|
"""Whether this interpreter can decompress an entry of the given type."""
|
||||||
|
method = _COMPRESS_TYPE_TO_METHOD.get(compress_type)
|
||||||
|
if method is None:
|
||||||
|
return False
|
||||||
|
return compression_available(method)
|
||||||
|
|
||||||
|
|
||||||
|
def unreadable_method_names(compress_types: set[int]) -> set[str]:
|
||||||
|
"""Map a set of compress_type ids to human method names for error messages."""
|
||||||
|
names: set[str] = set()
|
||||||
|
for ct in compress_types:
|
||||||
|
names.add(_COMPRESS_TYPE_TO_METHOD.get(ct, f"method {ct}"))
|
||||||
|
return names
|
||||||
@@ -0,0 +1,353 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import abc
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from contextlib import AbstractContextManager
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from pathlib import PurePosixPath
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.core.serializers.json import DjangoJSONEncoder
|
||||||
|
|
||||||
|
from documents.file_handling import delete_empty_directories
|
||||||
|
from documents.utils import compute_checksum
|
||||||
|
from documents.utils import copy_file_with_basic_stats
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from typing import TextIO
|
||||||
|
|
||||||
|
|
||||||
|
def _dumps(content: list | dict) -> str:
|
||||||
|
"""Serialize export JSON consistently across all sinks."""
|
||||||
|
return json.dumps(content, cls=DjangoJSONEncoder, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
class StreamingManifestWriter:
|
||||||
|
"""Incrementally writes a JSON array to a text handle, one record at a time.
|
||||||
|
|
||||||
|
Knows nothing about folders or zips: it writes the array framing and records
|
||||||
|
to whatever handle the sink's ``stream()`` yields. The sink owns the handle's
|
||||||
|
lifecycle (atomic rename, compare, spooling).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, handle: TextIO) -> None:
|
||||||
|
self._file = handle
|
||||||
|
self._first = True
|
||||||
|
self._file.write("[")
|
||||||
|
|
||||||
|
def write_record(self, record: dict) -> None:
|
||||||
|
if not self._first:
|
||||||
|
self._file.write(",\n")
|
||||||
|
else:
|
||||||
|
self._first = False
|
||||||
|
self._file.write(_dumps(record))
|
||||||
|
|
||||||
|
def write_batch(self, records: list[dict]) -> None:
|
||||||
|
for record in records:
|
||||||
|
self.write_record(record)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Write the closing bracket. Does NOT close the handle (the sink owns it)."""
|
||||||
|
self._file.write("\n]")
|
||||||
|
|
||||||
|
|
||||||
|
class ExportSink(AbstractContextManager, abc.ABC):
|
||||||
|
"""Destination for a document export.
|
||||||
|
|
||||||
|
The command declares export contents via three verbs; the sink decides how to
|
||||||
|
persist each. ``arcname`` is always a relative POSIX path
|
||||||
|
(e.g. ``"manifest.json"``, ``"originals/foo.pdf"``).
|
||||||
|
|
||||||
|
Contract:
|
||||||
|
* At most one ``stream()`` open at a time (it is the manifest);
|
||||||
|
``add_file``/``add_json`` may be called while it is open.
|
||||||
|
* Context-manager: normal exit finalizes, an exception aborts. No partial or
|
||||||
|
failed run leaves a complete-looking artifact.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def add_file(
|
||||||
|
self,
|
||||||
|
source: Path,
|
||||||
|
arcname: str,
|
||||||
|
*,
|
||||||
|
checksum: str | None = None,
|
||||||
|
) -> None: ...
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def add_json(self, content: list | dict, arcname: str) -> None: ...
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def stream(self, arcname: str) -> AbstractContextManager[TextIO]: ...
|
||||||
|
|
||||||
|
def _open(self) -> None:
|
||||||
|
"""Hook called on context entry. Override as needed."""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def _finalize(self) -> None:
|
||||||
|
"""Commit on clean exit."""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def _abort(self) -> None:
|
||||||
|
"""Roll back on exception."""
|
||||||
|
|
||||||
|
def __enter__(self) -> ExportSink:
|
||||||
|
self._open()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||||
|
if exc_type is not None:
|
||||||
|
self._abort()
|
||||||
|
else:
|
||||||
|
self._finalize()
|
||||||
|
|
||||||
|
|
||||||
|
class DirectoryExportSink(ExportSink):
|
||||||
|
"""Writes loose files into a target directory, with incremental sync.
|
||||||
|
|
||||||
|
Owns the snapshot/skip/compare/prune machinery that used to live in the
|
||||||
|
command (``files_in_export_dir``, ``check_and_copy``, ``check_and_write_json``,
|
||||||
|
and the ``--delete`` pass).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
target: Path,
|
||||||
|
*,
|
||||||
|
compare_checksums: bool,
|
||||||
|
compare_json: bool,
|
||||||
|
delete: bool,
|
||||||
|
) -> None:
|
||||||
|
self._target = target.resolve()
|
||||||
|
self._compare_checksums = compare_checksums
|
||||||
|
self._compare_json = compare_json
|
||||||
|
self._delete = delete
|
||||||
|
self._snapshot: set[Path] = set()
|
||||||
|
self._stream_open = False
|
||||||
|
|
||||||
|
def _open(self) -> None:
|
||||||
|
for x in self._target.glob("**/*"):
|
||||||
|
if x.is_file():
|
||||||
|
self._snapshot.add(x.resolve())
|
||||||
|
|
||||||
|
def add_file(
|
||||||
|
self,
|
||||||
|
source: Path,
|
||||||
|
arcname: str,
|
||||||
|
*,
|
||||||
|
checksum: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
target = (self._target / arcname).resolve()
|
||||||
|
self._snapshot.discard(target)
|
||||||
|
perform_copy = False
|
||||||
|
if target.exists():
|
||||||
|
source_stat = source.stat()
|
||||||
|
target_stat = target.stat()
|
||||||
|
if self._compare_checksums and checksum:
|
||||||
|
perform_copy = compute_checksum(target) != checksum
|
||||||
|
elif (
|
||||||
|
source_stat.st_mtime != target_stat.st_mtime
|
||||||
|
or source_stat.st_size != target_stat.st_size
|
||||||
|
):
|
||||||
|
perform_copy = True
|
||||||
|
else:
|
||||||
|
perform_copy = True
|
||||||
|
if perform_copy:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
copy_file_with_basic_stats(source, target)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _content_unchanged(target: Path, new_bytes: bytes) -> bool:
|
||||||
|
"""True if ``target`` already holds byte-identical content (BLAKE2b)."""
|
||||||
|
return (
|
||||||
|
hashlib.blake2b(target.read_bytes()).hexdigest()
|
||||||
|
== hashlib.blake2b(new_bytes).hexdigest()
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_json(self, content: list | dict, arcname: str) -> None:
|
||||||
|
target = (self._target / arcname).resolve()
|
||||||
|
json_str = _dumps(content)
|
||||||
|
perform_write = True
|
||||||
|
if target in self._snapshot:
|
||||||
|
self._snapshot.discard(target)
|
||||||
|
if self._compare_json and self._content_unchanged(
|
||||||
|
target,
|
||||||
|
json_str.encode("utf-8"),
|
||||||
|
):
|
||||||
|
perform_write = False
|
||||||
|
if perform_write:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_text(json_str, encoding="utf-8")
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def stream(self, arcname: str) -> Iterator[TextIO]:
|
||||||
|
if self._stream_open:
|
||||||
|
raise RuntimeError("A stream is already open on this sink")
|
||||||
|
target = (self._target / arcname).resolve()
|
||||||
|
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
handle = tmp.open("w", encoding="utf-8")
|
||||||
|
self._stream_open = True
|
||||||
|
try:
|
||||||
|
yield handle
|
||||||
|
except BaseException:
|
||||||
|
handle.close()
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
handle.close()
|
||||||
|
self._commit_streamed_file(target, tmp)
|
||||||
|
finally:
|
||||||
|
self._stream_open = False
|
||||||
|
|
||||||
|
def _commit_streamed_file(self, target: Path, tmp: Path) -> None:
|
||||||
|
if target in self._snapshot:
|
||||||
|
self._snapshot.discard(target)
|
||||||
|
if self._compare_json and self._content_unchanged(
|
||||||
|
target,
|
||||||
|
tmp.read_bytes(),
|
||||||
|
):
|
||||||
|
tmp.unlink()
|
||||||
|
return
|
||||||
|
tmp.rename(target)
|
||||||
|
|
||||||
|
def _finalize(self) -> None:
|
||||||
|
if self._delete:
|
||||||
|
for f in self._snapshot:
|
||||||
|
f.unlink()
|
||||||
|
delete_empty_directories(f.parent, self._target)
|
||||||
|
|
||||||
|
def _abort(self) -> None:
|
||||||
|
# Folder mode is in-place/incremental: streamed .tmp files are already
|
||||||
|
# cleaned in stream(); leave everything else intact and skip the prune.
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class ZipExportSink(ExportSink):
|
||||||
|
"""Writes a single zip archive, produced atomically only on success.
|
||||||
|
|
||||||
|
Builds into ``<target>/<zip_name>.zip.tmp`` and renames to ``.zip`` on clean
|
||||||
|
finalize. The manifest stream is spooled to a temp file in SCRATCH_DIR and
|
||||||
|
added as an entry at finalize (a zip entry cannot be interleaved with others).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
target: Path,
|
||||||
|
zip_name: str,
|
||||||
|
*,
|
||||||
|
delete: bool = False,
|
||||||
|
compression: int = zipfile.ZIP_DEFLATED,
|
||||||
|
compresslevel: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._target = target.resolve()
|
||||||
|
self._zip_path = (self._target / zip_name).with_suffix(".zip")
|
||||||
|
self._tmp_path = self._zip_path.with_name(self._zip_path.name + ".tmp")
|
||||||
|
self._delete = delete
|
||||||
|
self._compression = compression
|
||||||
|
self._compresslevel = compresslevel
|
||||||
|
self._zip: zipfile.ZipFile | None = None
|
||||||
|
self._dirs: set[str] = set()
|
||||||
|
self._pending_manifest: tuple[Path, str] | None = None
|
||||||
|
self._stream_open = False
|
||||||
|
|
||||||
|
def _open(self) -> None:
|
||||||
|
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._zip = zipfile.ZipFile(
|
||||||
|
self._tmp_path,
|
||||||
|
"w",
|
||||||
|
compression=self._compression,
|
||||||
|
compresslevel=self._compresslevel,
|
||||||
|
allowZip64=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _ensure_dirs(self, arcname: str) -> None:
|
||||||
|
assert self._zip is not None
|
||||||
|
dir_arc = ""
|
||||||
|
for part in PurePosixPath(arcname).parts[:-1]:
|
||||||
|
dir_arc += f"{part}/"
|
||||||
|
if dir_arc not in self._dirs:
|
||||||
|
self._dirs.add(dir_arc)
|
||||||
|
self._zip.mkdir(dir_arc)
|
||||||
|
|
||||||
|
def add_file(
|
||||||
|
self,
|
||||||
|
source: Path,
|
||||||
|
arcname: str,
|
||||||
|
*,
|
||||||
|
checksum: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
assert self._zip is not None
|
||||||
|
self._ensure_dirs(arcname)
|
||||||
|
self._zip.write(source, arcname=arcname)
|
||||||
|
|
||||||
|
def add_json(self, content: list | dict, arcname: str) -> None:
|
||||||
|
assert self._zip is not None
|
||||||
|
self._ensure_dirs(arcname)
|
||||||
|
self._zip.writestr(arcname, _dumps(content))
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def stream(self, arcname: str) -> Iterator[TextIO]:
|
||||||
|
if self._stream_open:
|
||||||
|
raise RuntimeError("A stream is already open on this sink")
|
||||||
|
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp_name = tempfile.mkstemp(
|
||||||
|
dir=settings.SCRATCH_DIR,
|
||||||
|
prefix="export-manifest-",
|
||||||
|
suffix=".json",
|
||||||
|
)
|
||||||
|
tmp = Path(tmp_name)
|
||||||
|
handle = os.fdopen(fd, "w", encoding="utf-8")
|
||||||
|
self._stream_open = True
|
||||||
|
try:
|
||||||
|
yield handle
|
||||||
|
except BaseException:
|
||||||
|
handle.close()
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
handle.close()
|
||||||
|
self._pending_manifest = (tmp, arcname)
|
||||||
|
finally:
|
||||||
|
self._stream_open = False
|
||||||
|
|
||||||
|
def _finalize(self) -> None:
|
||||||
|
assert self._zip is not None
|
||||||
|
if self._pending_manifest is not None:
|
||||||
|
tmp, arcname = self._pending_manifest
|
||||||
|
self._ensure_dirs(arcname)
|
||||||
|
self._zip.write(tmp, arcname=arcname)
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
self._pending_manifest = None
|
||||||
|
self._zip.close()
|
||||||
|
self._zip = None
|
||||||
|
if self._delete:
|
||||||
|
self._wipe_destination()
|
||||||
|
self._tmp_path.replace(self._zip_path)
|
||||||
|
|
||||||
|
def _wipe_destination(self) -> None:
|
||||||
|
skip = {self._zip_path.resolve(), self._tmp_path.resolve()}
|
||||||
|
for item in self._target.glob("*"):
|
||||||
|
if item.resolve() in skip:
|
||||||
|
continue
|
||||||
|
if item.is_dir():
|
||||||
|
shutil.rmtree(item)
|
||||||
|
else:
|
||||||
|
item.unlink()
|
||||||
|
|
||||||
|
def _abort(self) -> None:
|
||||||
|
if self._zip is not None:
|
||||||
|
self._zip.close()
|
||||||
|
self._zip = None
|
||||||
|
self._tmp_path.unlink(missing_ok=True)
|
||||||
|
if self._pending_manifest is not None:
|
||||||
|
self._pending_manifest[0].unlink(missing_ok=True)
|
||||||
|
self._pending_manifest = None
|
||||||
@@ -1,8 +1,4 @@
|
|||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
|
||||||
import tempfile
|
|
||||||
from itertools import islice
|
from itertools import islice
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
@@ -19,7 +15,6 @@ from django.contrib.auth.models import User
|
|||||||
from django.contrib.contenttypes.models import ContentType
|
from django.contrib.contenttypes.models import ContentType
|
||||||
from django.core import serializers
|
from django.core import serializers
|
||||||
from django.core.management.base import CommandError
|
from django.core.management.base import CommandError
|
||||||
from django.core.serializers.json import DjangoJSONEncoder
|
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from filelock import FileLock
|
from filelock import FileLock
|
||||||
@@ -34,7 +29,15 @@ if TYPE_CHECKING:
|
|||||||
if settings.AUDIT_LOG_ENABLED:
|
if settings.AUDIT_LOG_ENABLED:
|
||||||
from auditlog.models import LogEntry
|
from auditlog.models import LogEntry
|
||||||
|
|
||||||
from documents.file_handling import delete_empty_directories
|
from documents.export.compression import COMPRESSION_CHOICES
|
||||||
|
from documents.export.compression import COMPRESSION_METHODS
|
||||||
|
from documents.export.compression import ZSTD
|
||||||
|
from documents.export.compression import compression_available
|
||||||
|
from documents.export.compression import level_error
|
||||||
|
from documents.export.sinks import DirectoryExportSink
|
||||||
|
from documents.export.sinks import ExportSink
|
||||||
|
from documents.export.sinks import StreamingManifestWriter
|
||||||
|
from documents.export.sinks import ZipExportSink
|
||||||
from documents.file_handling import generate_filename
|
from documents.file_handling import generate_filename
|
||||||
from documents.management.commands.base import PaperlessCommand
|
from documents.management.commands.base import PaperlessCommand
|
||||||
from documents.management.commands.mixins import CryptMixin
|
from documents.management.commands.mixins import CryptMixin
|
||||||
@@ -60,8 +63,6 @@ from documents.settings import EXPORTER_ARCHIVE_NAME
|
|||||||
from documents.settings import EXPORTER_FILE_NAME
|
from documents.settings import EXPORTER_FILE_NAME
|
||||||
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
|
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
|
||||||
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
||||||
from documents.utils import compute_checksum
|
|
||||||
from documents.utils import copy_file_with_basic_stats
|
|
||||||
from paperless import version
|
from paperless import version
|
||||||
from paperless.models import ApplicationConfiguration
|
from paperless.models import ApplicationConfiguration
|
||||||
from paperless_mail.models import MailAccount
|
from paperless_mail.models import MailAccount
|
||||||
@@ -84,87 +85,6 @@ def serialize_queryset_batched(
|
|||||||
yield serializers.serialize("python", chunk)
|
yield serializers.serialize("python", chunk)
|
||||||
|
|
||||||
|
|
||||||
class StreamingManifestWriter:
|
|
||||||
"""Incrementally writes a JSON array to a file, one record at a time.
|
|
||||||
|
|
||||||
Writes to <target>.tmp first; on close(), optionally BLAKE2b-compares
|
|
||||||
with the existing file (--compare-json) and renames or discards accordingly.
|
|
||||||
On exception, discard() deletes the tmp file and leaves the original intact.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
path: Path,
|
|
||||||
*,
|
|
||||||
compare_json: bool = False,
|
|
||||||
files_in_export_dir: "set[Path] | None" = None,
|
|
||||||
) -> None:
|
|
||||||
self._path = path.resolve()
|
|
||||||
self._tmp_path = self._path.with_suffix(self._path.suffix + ".tmp")
|
|
||||||
self._compare_json = compare_json
|
|
||||||
self._files_in_export_dir: set[Path] = (
|
|
||||||
files_in_export_dir if files_in_export_dir is not None else set()
|
|
||||||
)
|
|
||||||
self._file = None
|
|
||||||
self._first = True
|
|
||||||
|
|
||||||
def open(self) -> None:
|
|
||||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
self._file = self._tmp_path.open("w", encoding="utf-8")
|
|
||||||
self._file.write("[")
|
|
||||||
self._first = True
|
|
||||||
|
|
||||||
def write_record(self, record: dict) -> None:
|
|
||||||
if not self._first:
|
|
||||||
self._file.write(",\n")
|
|
||||||
else:
|
|
||||||
self._first = False
|
|
||||||
self._file.write(
|
|
||||||
json.dumps(record, cls=DjangoJSONEncoder, indent=2, ensure_ascii=False),
|
|
||||||
)
|
|
||||||
|
|
||||||
def write_batch(self, records: list[dict]) -> None:
|
|
||||||
for record in records:
|
|
||||||
self.write_record(record)
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
if self._file is None:
|
|
||||||
return
|
|
||||||
self._file.write("\n]")
|
|
||||||
self._file.close()
|
|
||||||
self._file = None
|
|
||||||
self._finalize()
|
|
||||||
|
|
||||||
def discard(self) -> None:
|
|
||||||
if self._file is not None:
|
|
||||||
self._file.close()
|
|
||||||
self._file = None
|
|
||||||
if self._tmp_path.exists():
|
|
||||||
self._tmp_path.unlink()
|
|
||||||
|
|
||||||
def _finalize(self) -> None:
|
|
||||||
"""Compare with existing file (if --compare-json) then rename or discard tmp."""
|
|
||||||
if self._path in self._files_in_export_dir:
|
|
||||||
self._files_in_export_dir.remove(self._path)
|
|
||||||
if self._compare_json:
|
|
||||||
existing_hash = hashlib.blake2b(self._path.read_bytes()).hexdigest()
|
|
||||||
new_hash = hashlib.blake2b(self._tmp_path.read_bytes()).hexdigest()
|
|
||||||
if existing_hash == new_hash:
|
|
||||||
self._tmp_path.unlink()
|
|
||||||
return
|
|
||||||
self._tmp_path.rename(self._path)
|
|
||||||
|
|
||||||
def __enter__(self) -> "StreamingManifestWriter":
|
|
||||||
self.open()
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
|
||||||
if exc_type is not None:
|
|
||||||
self.discard()
|
|
||||||
else:
|
|
||||||
self.close()
|
|
||||||
|
|
||||||
|
|
||||||
class Command(CryptMixin, PaperlessCommand):
|
class Command(CryptMixin, PaperlessCommand):
|
||||||
help = (
|
help = (
|
||||||
"Decrypt and rename all files in our collection into a given target "
|
"Decrypt and rename all files in our collection into a given target "
|
||||||
@@ -276,6 +196,28 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
help="Sets the export zip file name",
|
help="Sets the export zip file name",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--zip-compression",
|
||||||
|
choices=COMPRESSION_CHOICES,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Compression method for the export zip (requires --zip). "
|
||||||
|
"Default: deflated. 'zstd' requires Python 3.14+ on both the "
|
||||||
|
"exporting and importing machine."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--zip-compression-level",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Compression level for the export zip (requires --zip). "
|
||||||
|
"deflated: 0-9, bzip2: 1-9, zstd: -22..22; ignored for "
|
||||||
|
"stored/lzma."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--data-only",
|
"--data-only",
|
||||||
default=False,
|
default=False,
|
||||||
@@ -314,20 +256,13 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
self.passphrase: str | None = options.get("passphrase")
|
self.passphrase: str | None = options.get("passphrase")
|
||||||
self.batch_size: int = options["batch_size"]
|
self.batch_size: int = options["batch_size"]
|
||||||
|
|
||||||
self.files_in_export_dir: set[Path] = set()
|
|
||||||
self.exported_files: set[str] = set()
|
self.exported_files: set[str] = set()
|
||||||
|
|
||||||
# If zipping, save the original target for later and
|
if self.zip_export and (self.compare_checksums or self.compare_json):
|
||||||
# get a temporary directory for the target instead
|
raise CommandError(
|
||||||
temp_dir = None
|
"--compare-checksums and --compare-json have no effect when "
|
||||||
self.original_target = self.target
|
"used with --zip",
|
||||||
if self.zip_export:
|
|
||||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
|
||||||
temp_dir = tempfile.TemporaryDirectory(
|
|
||||||
dir=settings.SCRATCH_DIR,
|
|
||||||
prefix="paperless-export",
|
|
||||||
)
|
)
|
||||||
self.target = Path(temp_dir.name).resolve()
|
|
||||||
|
|
||||||
if not self.target.exists():
|
if not self.target.exists():
|
||||||
raise CommandError("That path doesn't exist")
|
raise CommandError("That path doesn't exist")
|
||||||
@@ -338,33 +273,55 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
if not os.access(self.target, os.W_OK):
|
if not os.access(self.target, os.W_OK):
|
||||||
raise CommandError("That path doesn't appear to be writable")
|
raise CommandError("That path doesn't appear to be writable")
|
||||||
|
|
||||||
try:
|
zip_compression: str | None = options["zip_compression"]
|
||||||
# Prevent any ongoing changes in the documents
|
zip_compression_level: int | None = options["zip_compression_level"]
|
||||||
with FileLock(settings.MEDIA_LOCK):
|
|
||||||
self.dump()
|
|
||||||
|
|
||||||
# We've written everything to the temporary directory in this case,
|
if not self.zip_export and (
|
||||||
# now make an archive in the original target, with all files stored
|
zip_compression is not None or zip_compression_level is not None
|
||||||
if self.zip_export and temp_dir is not None:
|
):
|
||||||
shutil.make_archive(
|
raise CommandError(
|
||||||
self.original_target / options["zip_name"],
|
"--zip-compression and --zip-compression-level require --zip",
|
||||||
format="zip",
|
)
|
||||||
root_dir=temp_dir.name,
|
|
||||||
|
compression_method = zip_compression or "deflated"
|
||||||
|
if self.zip_export:
|
||||||
|
if not compression_available(compression_method):
|
||||||
|
if compression_method == "zstd" and ZSTD is None:
|
||||||
|
raise CommandError(
|
||||||
|
"zstd compression requires Python 3.14 or newer",
|
||||||
)
|
)
|
||||||
|
raise CommandError(
|
||||||
|
f"Compression method '{compression_method}' is not "
|
||||||
|
f"available on this Python runtime",
|
||||||
|
)
|
||||||
|
level_msg = level_error(compression_method, zip_compression_level)
|
||||||
|
if level_msg is not None:
|
||||||
|
raise CommandError(level_msg)
|
||||||
|
|
||||||
finally:
|
sink: ExportSink
|
||||||
# Always cleanup the temporary directory, if one was created
|
if self.zip_export:
|
||||||
if self.zip_export and temp_dir is not None:
|
sink = ZipExportSink(
|
||||||
temp_dir.cleanup()
|
self.target,
|
||||||
|
options["zip_name"],
|
||||||
|
delete=self.delete,
|
||||||
|
compression=COMPRESSION_METHODS[compression_method],
|
||||||
|
compresslevel=zip_compression_level,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sink = DirectoryExportSink(
|
||||||
|
self.target,
|
||||||
|
compare_checksums=self.compare_checksums,
|
||||||
|
compare_json=self.compare_json,
|
||||||
|
delete=self.delete,
|
||||||
|
)
|
||||||
|
|
||||||
def dump(self) -> None:
|
# Prevent any ongoing changes in the documents while exporting
|
||||||
# 1. Take a snapshot of what files exist in the current export folder
|
with FileLock(settings.MEDIA_LOCK), sink:
|
||||||
for x in self.target.glob("**/*"):
|
self.dump(sink)
|
||||||
if x.is_file():
|
|
||||||
self.files_in_export_dir.add(x.resolve())
|
|
||||||
|
|
||||||
# 2. Create manifest, containing all correspondents, types, tags, storage paths
|
def dump(self, sink: ExportSink) -> None:
|
||||||
# note, documents and ui_settings
|
# 1. Create manifest, containing all correspondents, types, tags, storage
|
||||||
|
# paths, note, documents and ui_settings
|
||||||
_excluded_usernames = ["consumer", "AnonymousUser"]
|
_excluded_usernames = ["consumer", "AnonymousUser"]
|
||||||
manifest_key_to_object_query: dict[str, QuerySet[Any]] = {
|
manifest_key_to_object_query: dict[str, QuerySet[Any]] = {
|
||||||
"correspondents": Correspondent.objects.all(),
|
"correspondents": Correspondent.objects.all(),
|
||||||
@@ -427,13 +384,9 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
|
|
||||||
document_manifest: list[dict] = []
|
document_manifest: list[dict] = []
|
||||||
share_link_bundle_manifest: list[dict] = []
|
share_link_bundle_manifest: list[dict] = []
|
||||||
manifest_path = (self.target / "manifest.json").resolve()
|
|
||||||
|
|
||||||
with StreamingManifestWriter(
|
with sink.stream("manifest.json") as handle:
|
||||||
manifest_path,
|
writer = StreamingManifestWriter(handle)
|
||||||
compare_json=self.compare_json,
|
|
||||||
files_in_export_dir=self.files_in_export_dir,
|
|
||||||
) as writer:
|
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
for key, qs in manifest_key_to_object_query.items():
|
for key, qs in manifest_key_to_object_query.items():
|
||||||
if key == "documents":
|
if key == "documents":
|
||||||
@@ -479,84 +432,55 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
# 3. Export files from each document
|
# 2. Export files from each document
|
||||||
for index, document_dict in enumerate(
|
for document_dict in self.track(
|
||||||
self.track(
|
document_manifest,
|
||||||
document_manifest,
|
description="Exporting documents...",
|
||||||
description="Exporting documents...",
|
total=len(document_manifest),
|
||||||
total=len(document_manifest),
|
|
||||||
),
|
|
||||||
):
|
):
|
||||||
document = document_map[document_dict["pk"]]
|
document = document_map[document_dict["pk"]]
|
||||||
|
|
||||||
# 3.1. generate a unique filename
|
# generate a unique filename, then the arcnames for its files
|
||||||
base_name = self.generate_base_name(document)
|
base_name = self.generate_base_name(document)
|
||||||
|
original_arc, thumbnail_arc, archive_arc = (
|
||||||
# 3.2. write filenames into manifest
|
|
||||||
original_target, thumbnail_target, archive_target = (
|
|
||||||
self.generate_document_targets(document, base_name, document_dict)
|
self.generate_document_targets(document, base_name, document_dict)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 3.3. write files to target folder
|
|
||||||
if not self.data_only:
|
if not self.data_only:
|
||||||
self.copy_document_files(
|
self.copy_document_files(
|
||||||
document,
|
document,
|
||||||
original_target,
|
sink,
|
||||||
thumbnail_target,
|
original_arc,
|
||||||
archive_target,
|
thumbnail_arc,
|
||||||
|
archive_arc,
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.split_manifest:
|
if self.split_manifest:
|
||||||
self._write_split_manifest(document_dict, document, base_name)
|
self._write_split_manifest(sink, document_dict, document, base_name)
|
||||||
else:
|
else:
|
||||||
writer.write_record(document_dict)
|
writer.write_record(document_dict)
|
||||||
|
|
||||||
for bundle_dict in share_link_bundle_manifest:
|
for bundle_dict in share_link_bundle_manifest:
|
||||||
bundle = share_link_bundle_map[bundle_dict["pk"]]
|
bundle = share_link_bundle_map[bundle_dict["pk"]]
|
||||||
|
bundle_arc = self.generate_share_link_bundle_target(
|
||||||
bundle_target = self.generate_share_link_bundle_target(
|
|
||||||
bundle,
|
bundle,
|
||||||
bundle_dict,
|
bundle_dict,
|
||||||
)
|
)
|
||||||
|
if not self.data_only and bundle_arc is not None:
|
||||||
if not self.data_only and bundle_target is not None:
|
self.copy_share_link_bundle_file(bundle, sink, bundle_arc)
|
||||||
self.copy_share_link_bundle_file(bundle, bundle_target)
|
|
||||||
|
|
||||||
writer.write_record(bundle_dict)
|
writer.write_record(bundle_dict)
|
||||||
|
|
||||||
# 4.2 write version information to target folder
|
writer.close()
|
||||||
extra_metadata_path = (self.target / "metadata.json").resolve()
|
|
||||||
|
# 3. Write version (and crypto params) to metadata.json
|
||||||
|
# Django stores most crypto values in the field itself; we store
|
||||||
|
# them once here for the whole export
|
||||||
metadata: dict[str, str | int | dict[str, str | int]] = {
|
metadata: dict[str, str | int | dict[str, str | int]] = {
|
||||||
"version": version.__full_version_str__,
|
"version": version.__full_version_str__,
|
||||||
}
|
}
|
||||||
|
|
||||||
# 4.2.1 If needed, write the crypto values into the metadata
|
|
||||||
# Django stores most of these in the field itself, we store them once here
|
|
||||||
if self.passphrase:
|
if self.passphrase:
|
||||||
metadata.update(self.get_crypt_params())
|
metadata.update(self.get_crypt_params())
|
||||||
|
sink.add_json(metadata, "metadata.json")
|
||||||
self.check_and_write_json(
|
|
||||||
metadata,
|
|
||||||
extra_metadata_path,
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.delete:
|
|
||||||
# 5. Remove files which we did not explicitly export in this run
|
|
||||||
if not self.zip_export:
|
|
||||||
for f in self.files_in_export_dir:
|
|
||||||
f.unlink()
|
|
||||||
|
|
||||||
delete_empty_directories(
|
|
||||||
f.parent,
|
|
||||||
self.target,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# 5. Remove anything in the original location (before moving the zip)
|
|
||||||
for item in self.original_target.glob("*"):
|
|
||||||
if item.is_dir():
|
|
||||||
shutil.rmtree(item)
|
|
||||||
else:
|
|
||||||
item.unlink()
|
|
||||||
|
|
||||||
def generate_base_name(self, document: Document) -> Path:
|
def generate_base_name(self, document: Document) -> Path:
|
||||||
"""
|
"""
|
||||||
@@ -584,73 +508,69 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
document: Document,
|
document: Document,
|
||||||
base_name: Path,
|
base_name: Path,
|
||||||
document_dict: dict,
|
document_dict: dict,
|
||||||
) -> tuple[Path, Path | None, Path | None]:
|
) -> tuple[str, str | None, str | None]:
|
||||||
"""
|
"""
|
||||||
Generates the targets for a given document, including the original file, archive file and thumbnail (depending on settings).
|
Generates the relative POSIX arcnames for a document's original, thumbnail
|
||||||
|
and archive files (depending on settings), and records them in the manifest.
|
||||||
"""
|
"""
|
||||||
original_name = base_name
|
original_name = base_name
|
||||||
if self.use_folder_prefix:
|
if self.use_folder_prefix:
|
||||||
original_name = Path("originals") / original_name
|
original_name = Path("originals") / original_name
|
||||||
original_target = (self.target / original_name).resolve()
|
original_arc = original_name.as_posix()
|
||||||
document_dict[EXPORTER_FILE_NAME] = str(original_name)
|
document_dict[EXPORTER_FILE_NAME] = original_arc
|
||||||
|
|
||||||
if not self.no_thumbnail:
|
if not self.no_thumbnail:
|
||||||
thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp")
|
thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp")
|
||||||
if self.use_folder_prefix:
|
if self.use_folder_prefix:
|
||||||
thumbnail_name = Path("thumbnails") / thumbnail_name
|
thumbnail_name = Path("thumbnails") / thumbnail_name
|
||||||
thumbnail_target = (self.target / thumbnail_name).resolve()
|
thumbnail_arc = thumbnail_name.as_posix()
|
||||||
document_dict[EXPORTER_THUMBNAIL_NAME] = str(thumbnail_name)
|
document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_arc
|
||||||
else:
|
else:
|
||||||
thumbnail_target = None
|
thumbnail_arc = None
|
||||||
|
|
||||||
if not self.no_archive and document.has_archive_version:
|
if not self.no_archive and document.has_archive_version:
|
||||||
archive_name = base_name.parent / (base_name.stem + "-archive.pdf")
|
archive_name = base_name.parent / (base_name.stem + "-archive.pdf")
|
||||||
if self.use_folder_prefix:
|
if self.use_folder_prefix:
|
||||||
archive_name = Path("archive") / archive_name
|
archive_name = Path("archive") / archive_name
|
||||||
archive_target = (self.target / archive_name).resolve()
|
archive_arc = archive_name.as_posix()
|
||||||
document_dict[EXPORTER_ARCHIVE_NAME] = str(archive_name)
|
document_dict[EXPORTER_ARCHIVE_NAME] = archive_arc
|
||||||
else:
|
else:
|
||||||
archive_target = None
|
archive_arc = None
|
||||||
|
|
||||||
return original_target, thumbnail_target, archive_target
|
return original_arc, thumbnail_arc, archive_arc
|
||||||
|
|
||||||
def copy_document_files(
|
def copy_document_files(
|
||||||
self,
|
self,
|
||||||
document: Document,
|
document: Document,
|
||||||
original_target: Path,
|
sink: ExportSink,
|
||||||
thumbnail_target: Path | None,
|
original_arc: str,
|
||||||
archive_target: Path | None,
|
thumbnail_arc: str | None,
|
||||||
|
archive_arc: str | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Copies files from the document storage location to the specified target location.
|
Hands the document's files to the sink (original, thumbnail, archive).
|
||||||
|
|
||||||
If the document is encrypted, the files are decrypted before copying them to the target location.
|
|
||||||
"""
|
"""
|
||||||
self.check_and_copy(
|
sink.add_file(document.source_path, original_arc, checksum=document.checksum)
|
||||||
document.source_path,
|
|
||||||
document.checksum,
|
|
||||||
original_target,
|
|
||||||
)
|
|
||||||
|
|
||||||
if thumbnail_target:
|
if thumbnail_arc:
|
||||||
self.check_and_copy(document.thumbnail_path, None, thumbnail_target)
|
sink.add_file(document.thumbnail_path, thumbnail_arc)
|
||||||
|
|
||||||
if archive_target:
|
if archive_arc:
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
assert isinstance(document.archive_path, Path)
|
assert isinstance(document.archive_path, Path)
|
||||||
self.check_and_copy(
|
sink.add_file(
|
||||||
document.archive_path,
|
document.archive_path,
|
||||||
document.archive_checksum,
|
archive_arc,
|
||||||
archive_target,
|
checksum=document.archive_checksum,
|
||||||
)
|
)
|
||||||
|
|
||||||
def generate_share_link_bundle_target(
|
def generate_share_link_bundle_target(
|
||||||
self,
|
self,
|
||||||
bundle: ShareLinkBundle,
|
bundle: ShareLinkBundle,
|
||||||
bundle_dict: dict,
|
bundle_dict: dict,
|
||||||
) -> Path | None:
|
) -> str | None:
|
||||||
"""
|
"""
|
||||||
Generates the export target for a share link bundle file, when present.
|
Generates the relative POSIX arcname for a share link bundle file, if any.
|
||||||
"""
|
"""
|
||||||
if not bundle.file_path:
|
if not bundle.file_path:
|
||||||
return None
|
return None
|
||||||
@@ -666,25 +586,22 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
bundle_dict["fields"]["file_path"] = portable_bundle_path.as_posix()
|
bundle_dict["fields"]["file_path"] = portable_bundle_path.as_posix()
|
||||||
bundle_dict[EXPORTER_SHARE_LINK_BUNDLE_NAME] = export_bundle_path.as_posix()
|
bundle_dict[EXPORTER_SHARE_LINK_BUNDLE_NAME] = export_bundle_path.as_posix()
|
||||||
|
|
||||||
return (self.target / export_bundle_path).resolve()
|
return export_bundle_path.as_posix()
|
||||||
|
|
||||||
def copy_share_link_bundle_file(
|
def copy_share_link_bundle_file(
|
||||||
self,
|
self,
|
||||||
bundle: ShareLinkBundle,
|
bundle: ShareLinkBundle,
|
||||||
bundle_target: Path,
|
sink: ExportSink,
|
||||||
|
bundle_arc: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Copies a share link bundle ZIP into the export directory.
|
Hands a share link bundle ZIP to the sink.
|
||||||
"""
|
"""
|
||||||
bundle_source_path = bundle.absolute_file_path
|
bundle_source_path = bundle.absolute_file_path
|
||||||
if bundle_source_path is None:
|
if bundle_source_path is None:
|
||||||
raise FileNotFoundError(f"Share link bundle {bundle.pk} has no file path")
|
raise FileNotFoundError(f"Share link bundle {bundle.pk} has no file path")
|
||||||
|
|
||||||
self.check_and_copy(
|
sink.add_file(bundle_source_path, bundle_arc)
|
||||||
bundle_source_path,
|
|
||||||
None,
|
|
||||||
bundle_target,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _encrypt_record_inline(self, record: dict) -> None:
|
def _encrypt_record_inline(self, record: dict) -> None:
|
||||||
"""Encrypt sensitive fields in a single record, if passphrase is set."""
|
"""Encrypt sensitive fields in a single record, if passphrase is set."""
|
||||||
@@ -700,6 +617,7 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
|
|
||||||
def _write_split_manifest(
|
def _write_split_manifest(
|
||||||
self,
|
self,
|
||||||
|
sink: ExportSink,
|
||||||
document_dict: dict,
|
document_dict: dict,
|
||||||
document: Document,
|
document: Document,
|
||||||
base_name: Path,
|
base_name: Path,
|
||||||
@@ -721,81 +639,4 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
manifest_name = base_name.with_name(f"{base_name.stem}-manifest.json")
|
manifest_name = base_name.with_name(f"{base_name.stem}-manifest.json")
|
||||||
if self.use_folder_prefix:
|
if self.use_folder_prefix:
|
||||||
manifest_name = Path("json") / manifest_name
|
manifest_name = Path("json") / manifest_name
|
||||||
manifest_name = (self.target / manifest_name).resolve()
|
sink.add_json(content, manifest_name.as_posix())
|
||||||
manifest_name.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.check_and_write_json(content, manifest_name)
|
|
||||||
|
|
||||||
def check_and_write_json(
|
|
||||||
self,
|
|
||||||
content: list[dict] | dict,
|
|
||||||
target: Path,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Writes the source content to the target json file.
|
|
||||||
If --compare-json arg was used, don't write to target file if
|
|
||||||
the file exists and checksum is identical to content checksum.
|
|
||||||
This preserves the file timestamps when no changes are made.
|
|
||||||
"""
|
|
||||||
|
|
||||||
target = target.resolve()
|
|
||||||
perform_write = True
|
|
||||||
if target in self.files_in_export_dir:
|
|
||||||
self.files_in_export_dir.remove(target)
|
|
||||||
if self.compare_json:
|
|
||||||
target_checksum = hashlib.blake2b(target.read_bytes()).hexdigest()
|
|
||||||
src_str = json.dumps(
|
|
||||||
content,
|
|
||||||
cls=DjangoJSONEncoder,
|
|
||||||
indent=2,
|
|
||||||
ensure_ascii=False,
|
|
||||||
)
|
|
||||||
src_checksum = hashlib.blake2b(src_str.encode("utf-8")).hexdigest()
|
|
||||||
if src_checksum == target_checksum:
|
|
||||||
perform_write = False
|
|
||||||
|
|
||||||
if perform_write:
|
|
||||||
target.write_text(
|
|
||||||
json.dumps(
|
|
||||||
content,
|
|
||||||
cls=DjangoJSONEncoder,
|
|
||||||
indent=2,
|
|
||||||
ensure_ascii=False,
|
|
||||||
),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
def check_and_copy(
|
|
||||||
self,
|
|
||||||
source: Path,
|
|
||||||
source_checksum: str | None,
|
|
||||||
target: Path,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Copies the source to the target, if target doesn't exist or the target doesn't seem to match
|
|
||||||
the source attributes
|
|
||||||
"""
|
|
||||||
|
|
||||||
target = target.resolve()
|
|
||||||
if target in self.files_in_export_dir:
|
|
||||||
self.files_in_export_dir.remove(target)
|
|
||||||
|
|
||||||
perform_copy = False
|
|
||||||
|
|
||||||
if target.exists():
|
|
||||||
source_stat = source.stat()
|
|
||||||
target_stat = target.stat()
|
|
||||||
if self.compare_checksums and source_checksum:
|
|
||||||
target_checksum = compute_checksum(target)
|
|
||||||
perform_copy = target_checksum != source_checksum
|
|
||||||
elif (
|
|
||||||
source_stat.st_mtime != target_stat.st_mtime
|
|
||||||
or source_stat.st_size != target_stat.st_size
|
|
||||||
):
|
|
||||||
perform_copy = True
|
|
||||||
else:
|
|
||||||
# Copy if it does not exist
|
|
||||||
perform_copy = True
|
|
||||||
|
|
||||||
if perform_copy:
|
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
copy_file_with_basic_stats(source, target)
|
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ from django.db.models.signals import post_save
|
|||||||
from filelock import FileLock
|
from filelock import FileLock
|
||||||
from guardian.shortcuts import clear_ct_cache
|
from guardian.shortcuts import clear_ct_cache
|
||||||
|
|
||||||
|
from documents.export.compression import compress_type_readable
|
||||||
|
from documents.export.compression import unreadable_method_names
|
||||||
from documents.file_handling import create_source_path_directory
|
from documents.file_handling import create_source_path_directory
|
||||||
from documents.management.commands.base import PaperlessCommand
|
from documents.management.commands.base import PaperlessCommand
|
||||||
from documents.management.commands.mixins import CryptMixin
|
from documents.management.commands.mixins import CryptMixin
|
||||||
@@ -451,6 +453,20 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
if is_zipfile(self.source):
|
if is_zipfile(self.source):
|
||||||
with ZipFile(self.source) as zf:
|
with ZipFile(self.source) as zf:
|
||||||
|
unsupported = {
|
||||||
|
info.compress_type
|
||||||
|
for info in zf.infolist()
|
||||||
|
if not compress_type_readable(info.compress_type)
|
||||||
|
}
|
||||||
|
if unsupported:
|
||||||
|
names = sorted(unreadable_method_names(unsupported))
|
||||||
|
message = (
|
||||||
|
f"This archive uses compression this Python cannot "
|
||||||
|
f"read ({', '.join(names)})."
|
||||||
|
)
|
||||||
|
if "zstd" in names:
|
||||||
|
message += " zstd archives require Python 3.14+."
|
||||||
|
raise CommandError(message)
|
||||||
zf.extractall(tmp_dir)
|
zf.extractall(tmp_dir)
|
||||||
self.source = Path(tmp_dir)
|
self.source = Path(tmp_dir)
|
||||||
self._run_import()
|
self._run_import()
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from documents.export import compression
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompressionMethods:
|
||||||
|
def test_choices_always_include_zstd(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- The compression policy module's CLI choices list
|
||||||
|
WHEN:
|
||||||
|
- Read on any runtime
|
||||||
|
THEN:
|
||||||
|
- zstd is always present; availability is checked separately so
|
||||||
|
argparse never hides it based on the current Python version
|
||||||
|
"""
|
||||||
|
assert compression.COMPRESSION_CHOICES == (
|
||||||
|
"stored",
|
||||||
|
"deflated",
|
||||||
|
"bzip2",
|
||||||
|
"lzma",
|
||||||
|
"zstd",
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("name", "constant"),
|
||||||
|
[
|
||||||
|
("stored", zipfile.ZIP_STORED),
|
||||||
|
("deflated", zipfile.ZIP_DEFLATED),
|
||||||
|
("bzip2", zipfile.ZIP_BZIP2),
|
||||||
|
("lzma", zipfile.ZIP_LZMA),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_method_maps_to_zipfile_constant(self, name: str, constant: int) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A compression method name
|
||||||
|
WHEN:
|
||||||
|
- Looked up in COMPRESSION_METHODS
|
||||||
|
THEN:
|
||||||
|
- It maps to the matching zipfile compression constant
|
||||||
|
"""
|
||||||
|
assert compression.COMPRESSION_METHODS[name] == constant
|
||||||
|
|
||||||
|
def test_stored_and_deflated_always_available(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- The stored and deflated compression methods
|
||||||
|
WHEN:
|
||||||
|
- Checked with compression_available()
|
||||||
|
THEN:
|
||||||
|
- Both are always available (zlib is a hard CPython dependency)
|
||||||
|
"""
|
||||||
|
assert compression.compression_available("stored")
|
||||||
|
assert compression.compression_available("deflated")
|
||||||
|
|
||||||
|
def test_zstd_availability_tracks_runtime(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- The zstd compression method
|
||||||
|
WHEN:
|
||||||
|
- Checked with compression_available() on this runtime
|
||||||
|
THEN:
|
||||||
|
- Availability matches whether Python is 3.14+
|
||||||
|
"""
|
||||||
|
expected: bool = sys.version_info >= (3, 14)
|
||||||
|
assert compression.compression_available("zstd") == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestLevelError:
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "level"),
|
||||||
|
[
|
||||||
|
("deflated", 0),
|
||||||
|
("deflated", 9),
|
||||||
|
("bzip2", 1),
|
||||||
|
("bzip2", 9),
|
||||||
|
("zstd", -22),
|
||||||
|
("zstd", 22),
|
||||||
|
("deflated", None),
|
||||||
|
("stored", None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_valid_levels_return_none(self, method: str, level: int | None) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A method and a level within its valid bounds (or no level)
|
||||||
|
WHEN:
|
||||||
|
- Checked with level_error()
|
||||||
|
THEN:
|
||||||
|
- No error message is returned
|
||||||
|
"""
|
||||||
|
assert compression.level_error(method, level) is None
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "level"),
|
||||||
|
[
|
||||||
|
("deflated", 10),
|
||||||
|
("deflated", -1),
|
||||||
|
("bzip2", 0),
|
||||||
|
("bzip2", 10),
|
||||||
|
("zstd", -23),
|
||||||
|
("zstd", 23),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_out_of_range_levels_return_message(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
level: int,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A method and a level outside its valid bounds
|
||||||
|
WHEN:
|
||||||
|
- Checked with level_error()
|
||||||
|
THEN:
|
||||||
|
- An error message naming the valid range is returned
|
||||||
|
"""
|
||||||
|
msg: str | None = compression.level_error(method, level)
|
||||||
|
assert msg is not None
|
||||||
|
assert "between" in msg
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("method", ["stored", "lzma"])
|
||||||
|
def test_level_on_levelless_method_is_rejected(self, method: str) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A method that ignores compression level (stored, lzma)
|
||||||
|
WHEN:
|
||||||
|
- A level is passed to level_error() anyway
|
||||||
|
THEN:
|
||||||
|
- An error message noting the level has no effect is returned
|
||||||
|
"""
|
||||||
|
msg: str | None = compression.level_error(method, 5)
|
||||||
|
assert msg is not None
|
||||||
|
assert "no effect" in msg
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompressTypeReadable:
|
||||||
|
@pytest.mark.parametrize("ct", [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED])
|
||||||
|
def test_stored_and_deflated_always_readable(self, ct: int) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A stored or deflated compress_type id
|
||||||
|
WHEN:
|
||||||
|
- Checked with compress_type_readable()
|
||||||
|
THEN:
|
||||||
|
- It is always readable
|
||||||
|
"""
|
||||||
|
assert compression.compress_type_readable(ct)
|
||||||
|
|
||||||
|
def test_zstd_compress_type_readability_tracks_runtime(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- The current (93) and legacy (20) zstd compress_type ids
|
||||||
|
WHEN:
|
||||||
|
- Checked with compress_type_readable() on this runtime
|
||||||
|
THEN:
|
||||||
|
- Readability matches whether Python is 3.14+
|
||||||
|
"""
|
||||||
|
# 93 = ZIP_ZSTANDARD; 20 = legacy zstd method id (read-only)
|
||||||
|
expected: bool = sys.version_info >= (3, 14)
|
||||||
|
assert compression.compress_type_readable(93) == expected
|
||||||
|
assert compression.compress_type_readable(20) == expected
|
||||||
|
|
||||||
|
def test_unknown_compress_type_is_unreadable(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- An unrecognized compress_type id
|
||||||
|
WHEN:
|
||||||
|
- Checked with compress_type_readable()
|
||||||
|
THEN:
|
||||||
|
- It is reported as unreadable
|
||||||
|
"""
|
||||||
|
assert not compression.compress_type_readable(9999)
|
||||||
|
|
||||||
|
def test_unreadable_method_names_lists_methods(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A set containing an unknown compress_type id
|
||||||
|
WHEN:
|
||||||
|
- Passed to unreadable_method_names()
|
||||||
|
THEN:
|
||||||
|
- It is reported generically as "method <id>"
|
||||||
|
"""
|
||||||
|
# An unknown method id maps to no name and is reported generically.
|
||||||
|
names: set[str] = compression.unreadable_method_names({9999})
|
||||||
|
assert names == {"method 9999"}
|
||||||
@@ -0,0 +1,332 @@
|
|||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_mock
|
||||||
|
|
||||||
|
from documents.export.sinks import DirectoryExportSink
|
||||||
|
from documents.export.sinks import ExportSink
|
||||||
|
from documents.export.sinks import StreamingManifestWriter
|
||||||
|
from documents.export.sinks import ZipExportSink
|
||||||
|
from documents.export.sinks import _dumps
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def source_file(tmp_path: Path) -> Path:
|
||||||
|
src: Path = tmp_path / "src" / "doc.pdf"
|
||||||
|
src.parent.mkdir(parents=True)
|
||||||
|
src.write_bytes(b"PDF-CONTENT")
|
||||||
|
return src
|
||||||
|
|
||||||
|
|
||||||
|
class TestDumps:
|
||||||
|
def test_dumps_is_indented_unicode_json(self) -> None:
|
||||||
|
result: str = _dumps({"a": "é", "b": 1})
|
||||||
|
assert '"é"' in result # ensure_ascii=False keeps unicode literal
|
||||||
|
assert "\n" in result # indent=2 produces newlines
|
||||||
|
assert json.loads(result) == {"a": "é", "b": 1}
|
||||||
|
|
||||||
|
|
||||||
|
class TestStreamingManifestWriter:
|
||||||
|
def test_writes_json_array_of_records(self) -> None:
|
||||||
|
handle: io.StringIO = io.StringIO()
|
||||||
|
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||||
|
writer.write_batch([{"pk": 1}, {"pk": 2}])
|
||||||
|
writer.write_record({"pk": 3})
|
||||||
|
writer.close()
|
||||||
|
assert json.loads(handle.getvalue()) == [{"pk": 1}, {"pk": 2}, {"pk": 3}]
|
||||||
|
|
||||||
|
def test_empty_manifest_is_valid_empty_array(self) -> None:
|
||||||
|
handle: io.StringIO = io.StringIO()
|
||||||
|
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||||
|
writer.close()
|
||||||
|
assert json.loads(handle.getvalue()) == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestDirectoryExportSink:
|
||||||
|
def test_add_file_copies_to_relative_arcname(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
) as sink:
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf")
|
||||||
|
assert (target / "originals" / "doc.pdf").read_bytes() == b"PDF-CONTENT"
|
||||||
|
|
||||||
|
def test_add_json_writes_file(self, tmp_path: Path) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
) as sink:
|
||||||
|
sink.add_json({"version": "x"}, "metadata.json")
|
||||||
|
assert json.loads((target / "metadata.json").read_text()) == {"version": "x"}
|
||||||
|
|
||||||
|
def test_stream_writes_manifest(self, tmp_path: Path) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
) as sink:
|
||||||
|
with sink.stream("manifest.json") as handle:
|
||||||
|
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||||
|
writer.write_record({"pk": 1})
|
||||||
|
writer.close()
|
||||||
|
assert json.loads((target / "manifest.json").read_text()) == [{"pk": 1}]
|
||||||
|
|
||||||
|
def test_add_file_skips_when_size_and_mtime_match(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
# Pre-existing target with identical size+mtime but DIFFERENT content:
|
||||||
|
# if add_file skips (no compare-checksums), the old content survives.
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
existing: Path = target / "originals" / "doc.pdf"
|
||||||
|
existing.parent.mkdir(parents=True)
|
||||||
|
# Same byte length as the source but different content + matching mtime,
|
||||||
|
# so a size/mtime comparison treats it as unchanged and skips the copy.
|
||||||
|
existing.write_bytes(b"X" * len(b"PDF-CONTENT"))
|
||||||
|
stat = source_file.stat()
|
||||||
|
os.utime(existing, (stat.st_atime, stat.st_mtime))
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
) as sink:
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf", checksum="abc")
|
||||||
|
assert existing.read_bytes() == b"X" * len(b"PDF-CONTENT") # skipped
|
||||||
|
|
||||||
|
def test_add_file_recopies_when_compare_checksums_differ(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
existing: Path = target / "originals" / "doc.pdf"
|
||||||
|
existing.parent.mkdir(parents=True)
|
||||||
|
existing.write_bytes(b"X" * len(b"PDF-CONTENT"))
|
||||||
|
stat = source_file.stat()
|
||||||
|
os.utime(existing, (stat.st_atime, stat.st_mtime))
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=True,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
) as sink:
|
||||||
|
# wrong checksum forces recopy despite matching size/mtime
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf", checksum="not-the-real-sum")
|
||||||
|
assert existing.read_bytes() == b"PDF-CONTENT" # recopied
|
||||||
|
|
||||||
|
def test_delete_prunes_unwritten_snapshot_files(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
stale: Path = target / "stale.pdf"
|
||||||
|
stale.write_bytes(b"STALE")
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=True,
|
||||||
|
) as sink:
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf")
|
||||||
|
assert not stale.exists()
|
||||||
|
assert (target / "originals" / "doc.pdf").exists()
|
||||||
|
|
||||||
|
def test_no_delete_keeps_unwritten_files(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
stale: Path = target / "stale.pdf"
|
||||||
|
stale.write_bytes(b"STALE")
|
||||||
|
with DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
) as sink:
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf")
|
||||||
|
assert stale.exists()
|
||||||
|
|
||||||
|
|
||||||
|
class TestZipExportSink:
|
||||||
|
def test_round_trip_files_json_and_stream(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with ZipExportSink(target, "export", delete=False) as sink:
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf")
|
||||||
|
sink.add_json({"version": "x"}, "metadata.json")
|
||||||
|
with sink.stream("manifest.json") as handle:
|
||||||
|
writer = StreamingManifestWriter(handle)
|
||||||
|
writer.write_record({"pk": 1})
|
||||||
|
writer.close()
|
||||||
|
zip_path: Path = target / "export.zip"
|
||||||
|
assert zip_path.exists()
|
||||||
|
assert not (target / "export.zip.tmp").exists()
|
||||||
|
with zipfile.ZipFile(zip_path) as zf:
|
||||||
|
names = set(zf.namelist())
|
||||||
|
assert {"originals/doc.pdf", "metadata.json", "manifest.json"} <= names
|
||||||
|
assert zf.read("originals/doc.pdf") == b"PDF-CONTENT"
|
||||||
|
assert json.loads(zf.read("manifest.json")) == [{"pk": 1}]
|
||||||
|
|
||||||
|
def test_nested_arcname_emits_directory_marker(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with ZipExportSink(target, "export", delete=False) as sink:
|
||||||
|
sink.add_file(source_file, "originals/doc.pdf")
|
||||||
|
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||||
|
assert "originals/" in zf.namelist()
|
||||||
|
|
||||||
|
def test_flat_arcname_has_no_directory_markers(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with ZipExportSink(target, "export", delete=False) as sink:
|
||||||
|
sink.add_file(source_file, "doc.pdf")
|
||||||
|
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||||
|
assert all(not n.endswith("/") for n in zf.namelist())
|
||||||
|
|
||||||
|
def test_exception_leaves_no_zip_and_no_tmp(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
with ZipExportSink(target, "export", delete=False) as sink:
|
||||||
|
sink.add_file(source_file, "doc.pdf")
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
assert not (target / "export.zip").exists()
|
||||||
|
assert not (target / "export.zip.tmp").exists()
|
||||||
|
|
||||||
|
def test_delete_wipes_destination_on_success(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
(target / "preexisting.txt").write_text("old")
|
||||||
|
(target / "olddir").mkdir()
|
||||||
|
with ZipExportSink(target, "export", delete=True) as sink:
|
||||||
|
sink.add_file(source_file, "doc.pdf")
|
||||||
|
assert (target / "export.zip").exists()
|
||||||
|
assert not (target / "preexisting.txt").exists()
|
||||||
|
assert not (target / "olddir").exists()
|
||||||
|
|
||||||
|
def test_abort_with_delete_does_not_wipe_destination(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
source_file: Path,
|
||||||
|
) -> None:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
(target / "preexisting.txt").write_text("old")
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
with ZipExportSink(target, "export", delete=True) as sink:
|
||||||
|
sink.add_file(source_file, "doc.pdf")
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
assert (target / "preexisting.txt").exists()
|
||||||
|
assert not (target / "export.zip").exists()
|
||||||
|
|
||||||
|
|
||||||
|
class TestZipExportSinkCompression:
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "constant"),
|
||||||
|
[
|
||||||
|
("stored", zipfile.ZIP_STORED),
|
||||||
|
("deflated", zipfile.ZIP_DEFLATED),
|
||||||
|
("bzip2", zipfile.ZIP_BZIP2),
|
||||||
|
("lzma", zipfile.ZIP_LZMA),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_compression_and_level_forwarded_to_zipfile(
|
||||||
|
self,
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
method: str,
|
||||||
|
constant: int,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A ZipExportSink constructed with a compression method and level
|
||||||
|
WHEN:
|
||||||
|
- The sink is opened
|
||||||
|
THEN:
|
||||||
|
- zipfile.ZipFile is constructed with those values forwarded
|
||||||
|
unchanged (whether ZipFile actually compresses is Python's own
|
||||||
|
contract, not ours, so this checks the call args, not a real
|
||||||
|
archive)
|
||||||
|
"""
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
zip_cls = mocker.patch("documents.export.sinks.zipfile.ZipFile")
|
||||||
|
sink = ZipExportSink(target, "export", compression=constant, compresslevel=5)
|
||||||
|
sink._open()
|
||||||
|
zip_cls.assert_called_once_with(
|
||||||
|
mocker.ANY,
|
||||||
|
"w",
|
||||||
|
compression=constant,
|
||||||
|
compresslevel=5,
|
||||||
|
allowZip64=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestStreamContract:
|
||||||
|
@pytest.fixture(params=["dir", "zip"])
|
||||||
|
def sink(self, request: pytest.FixtureRequest, tmp_path: Path) -> ExportSink:
|
||||||
|
target: Path = tmp_path / "out"
|
||||||
|
target.mkdir()
|
||||||
|
if request.param == "dir":
|
||||||
|
return DirectoryExportSink(
|
||||||
|
target,
|
||||||
|
compare_checksums=False,
|
||||||
|
compare_json=False,
|
||||||
|
delete=False,
|
||||||
|
)
|
||||||
|
return ZipExportSink(target, "export", delete=False)
|
||||||
|
|
||||||
|
def test_second_concurrent_stream_is_rejected(self, sink: ExportSink) -> None:
|
||||||
|
with sink:
|
||||||
|
with sink.stream("manifest.json"):
|
||||||
|
with pytest.raises(RuntimeError, match="already open"):
|
||||||
|
with sink.stream("other.json"):
|
||||||
|
pass
|
||||||
@@ -6,6 +6,8 @@ from datetime import timedelta
|
|||||||
from io import StringIO
|
from io import StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
from zipfile import ZIP_DEFLATED
|
||||||
|
from zipfile import ZIP_LZMA
|
||||||
from zipfile import ZipFile
|
from zipfile import ZipFile
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -426,7 +428,7 @@ class TestExportImport(
|
|||||||
st_mtime_1 = (self.target / "manifest.json").stat().st_mtime
|
st_mtime_1 = (self.target / "manifest.json").stat().st_mtime
|
||||||
|
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
"documents.export.sinks.copy_file_with_basic_stats",
|
||||||
) as m:
|
) as m:
|
||||||
self._do_export()
|
self._do_export()
|
||||||
m.assert_not_called()
|
m.assert_not_called()
|
||||||
@@ -437,7 +439,7 @@ class TestExportImport(
|
|||||||
Path(self.d1.source_path).touch()
|
Path(self.d1.source_path).touch()
|
||||||
|
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
"documents.export.sinks.copy_file_with_basic_stats",
|
||||||
) as m:
|
) as m:
|
||||||
self._do_export()
|
self._do_export()
|
||||||
self.assertEqual(m.call_count, 1)
|
self.assertEqual(m.call_count, 1)
|
||||||
@@ -464,7 +466,7 @@ class TestExportImport(
|
|||||||
self.assertIsFile(self.target / "manifest.json")
|
self.assertIsFile(self.target / "manifest.json")
|
||||||
|
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
"documents.export.sinks.copy_file_with_basic_stats",
|
||||||
) as m:
|
) as m:
|
||||||
self._do_export()
|
self._do_export()
|
||||||
m.assert_not_called()
|
m.assert_not_called()
|
||||||
@@ -475,7 +477,7 @@ class TestExportImport(
|
|||||||
self.d2.save()
|
self.d2.save()
|
||||||
|
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
"documents.export.sinks.copy_file_with_basic_stats",
|
||||||
) as m:
|
) as m:
|
||||||
self._do_export(compare_checksums=True)
|
self._do_export(compare_checksums=True)
|
||||||
self.assertEqual(m.call_count, 1)
|
self.assertEqual(m.call_count, 1)
|
||||||
@@ -1058,6 +1060,190 @@ class TestExportImport(
|
|||||||
|
|
||||||
self.assertEqual(Document.objects.all().count(), 4)
|
self.assertEqual(Document.objects.all().count(), 4)
|
||||||
|
|
||||||
|
def test_zip_with_compare_flags_raises(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export to a zip file
|
||||||
|
WHEN:
|
||||||
|
- --compare-checksums or --compare-json is also passed
|
||||||
|
THEN:
|
||||||
|
- A CommandError is raised (the flags are no-ops in zip mode)
|
||||||
|
"""
|
||||||
|
for flag in ("--compare-checksums", "--compare-json"):
|
||||||
|
with self.subTest(flag=flag):
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
flag,
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_compression_flags_require_zip(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export without --zip
|
||||||
|
WHEN:
|
||||||
|
- --zip-compression or --zip-compression-level is passed anyway
|
||||||
|
THEN:
|
||||||
|
- A CommandError is raised (the flags are meaningless without --zip)
|
||||||
|
"""
|
||||||
|
for args in (
|
||||||
|
["--zip-compression", "lzma"],
|
||||||
|
["--zip-compression-level", "5"],
|
||||||
|
):
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
*args,
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_compression_level_out_of_range_raises(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export to a zip file
|
||||||
|
WHEN:
|
||||||
|
- --zip-compression-level is outside the chosen method's valid range
|
||||||
|
THEN:
|
||||||
|
- A CommandError is raised
|
||||||
|
"""
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"deflated",
|
||||||
|
"--zip-compression-level",
|
||||||
|
"99",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_compression_level_rejected_for_stored(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export to a zip file with --zip-compression stored
|
||||||
|
WHEN:
|
||||||
|
- --zip-compression-level is also passed
|
||||||
|
THEN:
|
||||||
|
- A CommandError is raised (stored ignores level entirely)
|
||||||
|
"""
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"stored",
|
||||||
|
"--zip-compression-level",
|
||||||
|
"5",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zip_compression_level_rejected_for_lzma(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export to a zip file with --zip-compression lzma
|
||||||
|
WHEN:
|
||||||
|
- --zip-compression-level is also passed
|
||||||
|
THEN:
|
||||||
|
- A CommandError is raised (lzma ignores level entirely)
|
||||||
|
"""
|
||||||
|
with self.assertRaises(CommandError):
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"lzma",
|
||||||
|
"--zip-compression-level",
|
||||||
|
"5",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_zstd_unavailable_raises_friendly_error(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A Python runtime without zstd support (< 3.14)
|
||||||
|
WHEN:
|
||||||
|
- --zip-compression zstd is requested
|
||||||
|
THEN:
|
||||||
|
- A CommandError naming the Python version requirement is raised
|
||||||
|
"""
|
||||||
|
with self.assertRaises(CommandError) as e:
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"zstd",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
self.assertIn("3.14", str(e.exception))
|
||||||
|
|
||||||
|
def test_zip_compression_flag_resolves_to_sink_constant(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export to a zip file with --zip-compression lzma
|
||||||
|
WHEN:
|
||||||
|
- The export runs
|
||||||
|
THEN:
|
||||||
|
- ZipExportSink is constructed with the resolved ZIP_LZMA constant
|
||||||
|
(whether zipfile actually compresses with the chosen method is
|
||||||
|
Python's own contract, and ZipExportSink's own tests already
|
||||||
|
cover the forwarding; what this command owns is resolving the
|
||||||
|
CLI string to the right constant, so assert that resolution
|
||||||
|
directly)
|
||||||
|
"""
|
||||||
|
with mock.patch(
|
||||||
|
"documents.management.commands.document_exporter.ZipExportSink",
|
||||||
|
) as sink_cls:
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
"--zip-compression",
|
||||||
|
"lzma",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
sink_cls.assert_called_once_with(
|
||||||
|
mock.ANY,
|
||||||
|
mock.ANY,
|
||||||
|
delete=False,
|
||||||
|
compression=ZIP_LZMA,
|
||||||
|
compresslevel=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_default_zip_compression_resolves_to_deflate(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A request to export to a zip file with no --zip-compression flag
|
||||||
|
WHEN:
|
||||||
|
- The export runs
|
||||||
|
THEN:
|
||||||
|
- ZipExportSink is constructed with the default ZIP_DEFLATED
|
||||||
|
constant and compresslevel=None, matching pre-existing behavior
|
||||||
|
"""
|
||||||
|
with mock.patch(
|
||||||
|
"documents.management.commands.document_exporter.ZipExportSink",
|
||||||
|
) as sink_cls:
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
self.target,
|
||||||
|
"--zip",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
sink_cls.assert_called_once_with(
|
||||||
|
mock.ANY,
|
||||||
|
mock.ANY,
|
||||||
|
delete=False,
|
||||||
|
compression=ZIP_DEFLATED,
|
||||||
|
compresslevel=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.management
|
@pytest.mark.management
|
||||||
class TestCryptExportImport(
|
class TestCryptExportImport(
|
||||||
|
|||||||
@@ -525,6 +525,35 @@ class TestCommandImport(
|
|||||||
self.assertEqual(doc.tags.count(), 1)
|
self.assertEqual(doc.tags.count(), 1)
|
||||||
self.assertEqual(doc.tags.first().name, "batch-flush-tag")
|
self.assertEqual(doc.tags.first().name, "batch-flush-tag")
|
||||||
|
|
||||||
|
def test_import_rejects_unreadable_compression(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A zip archive with an entry whose compression this Python can't read
|
||||||
|
WHEN:
|
||||||
|
- Import is attempted
|
||||||
|
THEN:
|
||||||
|
- A CommandError naming the issue is raised, before extraction
|
||||||
|
"""
|
||||||
|
import zipfile
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
archive = Path(self.dirs.scratch_dir) / "export.zip"
|
||||||
|
with zipfile.ZipFile(archive, "w") as zf:
|
||||||
|
zf.writestr("manifest.json", "[]")
|
||||||
|
|
||||||
|
with mock.patch(
|
||||||
|
"documents.management.commands.document_importer.compress_type_readable",
|
||||||
|
return_value=False,
|
||||||
|
):
|
||||||
|
with self.assertRaises(CommandError) as e:
|
||||||
|
call_command(
|
||||||
|
"document_importer",
|
||||||
|
str(archive),
|
||||||
|
"--no-progress-bar",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
self.assertIn("compression", str(e.exception))
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.management
|
@pytest.mark.management
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
|||||||
Reference in New Issue
Block a user