mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-30 06:27:14 +00:00
Performance: Stream JSON during import for memory improvements (#12276)
* Perf: stream manifest parsing with ijson in document_importer Replace bulk json.load of the full manifest (which materializes the entire JSON array into memory) with incremental ijson streaming. Eliminates self.manifest entirely — records are never all in memory at once. - Add ijson>=3.2 dependency - New module-level iter_manifest_records() generator - load_manifest_files() collects paths only; no parsing at load time - check_manifest_validity() streams without accumulating records - decrypt_secret_fields() streams each manifest to a .decrypted.json temp file record-by-record; temp files cleaned up after file copy - _import_files_from_manifest() collects only document records (small fraction of manifest) for the tqdm progress bar Measured on 200 docs + 200 CustomFieldInstances: - Streaming validation: peak memory 3081 KiB -> 333 KiB (89% reduction) - Stream-decrypt to file: peak memory 3081 KiB -> 549 KiB (82% reduction) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Perf: slim dict in _import_files_from_manifest, discard fields When collecting document records for the file-copy step, extract only the 4 keys the loop actually uses (pk + 3 exported filename keys) and discard the full fields dict (content, checksum, tags, etc.). Peak memory for the document-record list: 939 KiB -> 375 KiB (60% reduction). Wall time unchanged.
This commit is contained in:
@@ -119,15 +119,22 @@ class TestCommandImport(
|
||||
# No read permissions
|
||||
original_path.chmod(0o222)
|
||||
|
||||
manifest_path = Path(temp_dir) / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"model": "documents.document",
|
||||
EXPORTER_FILE_NAME: "original.pdf",
|
||||
EXPORTER_ARCHIVE_NAME: "archive.pdf",
|
||||
},
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
cmd = Command()
|
||||
cmd.source = Path(temp_dir)
|
||||
cmd.manifest = [
|
||||
{
|
||||
"model": "documents.document",
|
||||
EXPORTER_FILE_NAME: "original.pdf",
|
||||
EXPORTER_ARCHIVE_NAME: "archive.pdf",
|
||||
},
|
||||
]
|
||||
cmd.manifest_paths = [manifest_path]
|
||||
cmd.data_only = False
|
||||
with self.assertRaises(CommandError) as cm:
|
||||
cmd.check_manifest_validity()
|
||||
@@ -296,7 +303,7 @@ class TestCommandImport(
|
||||
(self.dirs.scratch_dir / "manifest.json").touch()
|
||||
|
||||
# We're not building a manifest, so it fails, but this test doesn't care
|
||||
with self.assertRaises(json.decoder.JSONDecodeError):
|
||||
with self.assertRaises(CommandError):
|
||||
call_command(
|
||||
"document_importer",
|
||||
"--no-progress-bar",
|
||||
@@ -325,7 +332,7 @@ class TestCommandImport(
|
||||
)
|
||||
|
||||
# We're not building a manifest, so it fails, but this test doesn't care
|
||||
with self.assertRaises(json.decoder.JSONDecodeError):
|
||||
with self.assertRaises(CommandError):
|
||||
call_command(
|
||||
"document_importer",
|
||||
"--no-progress-bar",
|
||||
|
||||
Reference in New Issue
Block a user