amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-31 23:17:19 +00:00
Code Issues Projects Releases Wiki Activity

Perf: slim dict in _import_files_from_manifest, discard fields

Browse Source 
When collecting document records for the file-copy step, extract only
the 4 keys the loop actually uses (pk + 3 exported filename keys) and
discard the full fields dict (content, checksum, tags, etc.).

Peak memory for the document-record list: 939 KiB -> 375 KiB (60% reduction).
Wall time unchanged.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Trenton H
2026-03-07 13:20:29 -08:00
co-authored by Claude Sonnet 4.6
parent dfd370700a
commit db41c45128
2 changed files with 35 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/management/commands/document_importer.py
+8 -3
View File
@@ -370,7 +370,12 @@ class Command(CryptMixin, BaseCommand):
self.stdout.write("Copy files into paperless...")
document_records = [
record
{
"pk": record["pk"],
EXPORTER_FILE_NAME: record[EXPORTER_FILE_NAME],
EXPORTER_THUMBNAIL_NAME: record.get(EXPORTER_THUMBNAIL_NAME),
EXPORTER_ARCHIVE_NAME: record.get(EXPORTER_ARCHIVE_NAME),
}
for manifest_path in self.manifest_paths
for record in iter_manifest_records(manifest_path)
if record["model"] == "documents.document"
@@ -382,13 +387,13 @@ class Command(CryptMixin, BaseCommand):
doc_file = record[EXPORTER_FILE_NAME]
document_path = self.source / doc_file
if EXPORTER_THUMBNAIL_NAME in record:
if record[EXPORTER_THUMBNAIL_NAME]:
thumb_file = record[EXPORTER_THUMBNAIL_NAME]
thumbnail_path = (self.source / thumb_file).resolve()
else:
thumbnail_path = None
if EXPORTER_ARCHIVE_NAME in record:
if record[EXPORTER_ARCHIVE_NAME]:
archive_file = record[EXPORTER_ARCHIVE_NAME]
archive_path = self.source / archive_file
else:
src/documents/tests/test_importer_profile_phase4.py
+27
View File
@@ -99,3 +99,30 @@ class TestImporterProfilePhase4(DirectoriesMixin, SampleDirMixin, TestCase):
first = False
out.write("\n]\n")
tmp_path.unlink(missing_ok=True)
# Baseline: full record list (old _import_files_from_manifest approach)
with profile_block("baseline: full record list (doc records only)"):
_ = [
record
for path in manifest_paths
for record in iter_manifest_records(path)
if record["model"] == "documents.document"
]
# New: slim dict list (current _import_files_from_manifest approach)
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_THUMBNAIL_NAME
with profile_block("new: slim dict list (4 keys only)"):
_ = [
{
"pk": record["pk"],
EXPORTER_FILE_NAME: record[EXPORTER_FILE_NAME],
EXPORTER_THUMBNAIL_NAME: record.get(EXPORTER_THUMBNAIL_NAME),
EXPORTER_ARCHIVE_NAME: record.get(EXPORTER_ARCHIVE_NAME),
}
for path in manifest_paths
for record in iter_manifest_records(path)
if record["model"] == "documents.document"
]