amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-02 09:02:18 +00:00
Code Issues Projects Releases Wiki Activity

Feature: Further reduce document importer memory usage (#12707)

Browse Source 
* Replaces loaddata with streaming bulk_create

Replaces call_command('loaddata') with a streaming implementation that
reads manifest records one at a time via ijson, accumulates per-model
batches up to --batch-size, and flushes via bulk_create.  This reduces
peak memory and no longer scales directly with the size of the import.

* fix(importer): avoid guardian lru_cache poisoning; include M2M through tables in check_constraints

clear_cache() inside the import transaction emptied Django's ContentType
manager cache while fixture PKs were live, causing downstream ContentType
lookups to repopulate guardian's separate @lru_cache(None) with
fixture-PK objects. After the TestCase transaction rolled back to
original PKs, guardian's lru_cache held stale fixture ContentType
objects, causing MixedContentTypeError in unrelated subsequent tests.

Remove clear_cache() since it was defending against a theoretical
stale-cache scenario that doesn't occur in a proper same-install restore.

Fix check_constraints() to explicitly include auto-created M2M through
tables (populated by .set() after bulk_create) alongside the model tables,
addressing the gap where join-table FK violations would have gone
undetected.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* Excludes the consumer and AnonymousUser from any models which might have a FK relation to it.  This prevents orphan things like UI setting, which have a relation to no existing user

* Splits into more sub functions for Sonar

* Improvements to the typing of the new functions

* Coverage for some error cases, and removes handling for pk only models.  No need to support these

* Final coverage gaps

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Trenton H
2026-05-04 16:36:05 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent 209c5d2deb
commit e822e72964
4 changed files with 558 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/tests/test_management_importer.py
+305
View File
@@ -11,6 +11,7 @@ from django.core.management.base import CommandError
from django.test import TestCase
from documents.management.commands.document_importer import Command
from documents.management.commands.document_importer import _deserialize_record
from documents.models import Document
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
@@ -397,3 +398,307 @@ class TestCommandImport(
# There should be no error or warnings. Therefore the output should be empty.
self.assertEqual(stdout_str, "")
def test_batch_size_argument_accepted(self) -> None:
"""
GIVEN:
- A valid source directory with an empty manifest
WHEN:
- Import is called with --batch-size 100
THEN:
- No argument parsing error is raised
"""
manifest_file = self.dirs.scratch_dir / "manifest.json"
manifest_file.write_text("[]")
try:
call_command(
"document_importer",
"--no-progress-bar",
"--batch-size",
"100",
str(self.dirs.scratch_dir),
skip_checks=True,
)
except CommandError:
pass # Expected: empty manifest or missing files, not an argument error
except SystemExit as e:
self.fail(f"--batch-size raised SystemExit (unrecognized argument?): {e}")
def test_m2m_relations_restored_after_data_only_import(self) -> None:
"""
GIVEN:
- A manifest with a Tag (pk=100) and a Document (pk=100) with
tags: [100] in the fields
WHEN:
- Data-only import is performed
THEN:
- Document.objects.get(pk=100).tags.count() == 1
- The tag's name is preserved correctly
"""
tag_record = {
"model": "documents.tag",
"pk": 100,
"fields": {"name": "imported-tag"},
}
doc_record = {
"model": "documents.document",
"pk": 100,
"fields": {
"title": "Tagged Doc",
"content": "test content",
"checksum": "1093cf6e32adbd16b06969df09215d42c4a3a8938cc18b39455953f08d1ff2ab",
"filename": "0001000.pdf",
"mime_type": "application/pdf",
"modified": "2024-01-01T00:00:00Z",
"added": "2024-01-01T00:00:00Z",
"tags": [100],
"correspondent": None,
"document_type": None,
"storage_path": None,
},
}
manifest_file = self.dirs.scratch_dir / "manifest.json"
manifest_file.write_text(json.dumps([tag_record, doc_record]))
call_command(
"document_importer",
"--no-progress-bar",
"--data-only",
str(self.dirs.scratch_dir),
skip_checks=True,
)
doc = Document.objects.get(pk=100)
self.assertEqual(doc.tags.count(), 1)
self.assertEqual(doc.tags.first().name, "imported-tag")
def test_mid_batch_flush_triggered_by_small_batch_size(self) -> None:
"""
GIVEN:
- A manifest with two records (Tag + Document)
- --batch-size 1 so each record fills a batch immediately
WHEN:
- Import is performed
THEN:
- flush_model() fires mid-loop (before flush_all) and the import
completes correctly with the M2M relation intact
"""
tag_record = {
"model": "documents.tag",
"pk": 200,
"fields": {"name": "batch-flush-tag"},
}
doc_record = {
"model": "documents.document",
"pk": 200,
"fields": {
"title": "Batch Flush Doc",
"content": "test",
"checksum": "2093cf6e32adbd16b06969df09215d42c4a3a8938cc18b39455953f08d1ff2ab",
"filename": "0002000.pdf",
"mime_type": "application/pdf",
"modified": "2024-01-01T00:00:00Z",
"added": "2024-01-01T00:00:00Z",
"tags": [200],
"correspondent": None,
"document_type": None,
"storage_path": None,
},
}
manifest_file = self.dirs.scratch_dir / "manifest.json"
manifest_file.write_text(json.dumps([tag_record, doc_record]))
call_command(
"document_importer",
"--no-progress-bar",
"--data-only",
"--batch-size",
"1",
str(self.dirs.scratch_dir),
skip_checks=True,
)
doc = Document.objects.get(pk=200)
self.assertEqual(doc.tags.count(), 1)
self.assertEqual(doc.tags.first().name, "batch-flush-tag")
@pytest.mark.management
@pytest.mark.django_db
class TestDeserializeRecord:
def test_simple_model_no_relations(self) -> None:
"""
GIVEN:
- A manifest record for a Correspondent (no M2M fields)
WHEN:
- _deserialize_record is called
THEN:
- Returns the correct model class, a Correspondent instance with
correct field values, and an empty m2m_data dict
"""
record = {
"model": "documents.correspondent",
"pk": 42,
"fields": {
"name": "ACME Corp",
"match": "",
"matching_algorithm": 1,
"is_insensitive": False,
"owner": None,
},
}
model, instance, m2m_data = _deserialize_record(record)
assert model.__name__ == "Correspondent"
assert instance.pk == 42
assert instance.name == "ACME Corp"
assert m2m_data == {}
def test_fk_field_stored_on_attname(self) -> None:
"""
GIVEN:
- A manifest record for a Document with a FK to a Correspondent
WHEN:
- _deserialize_record is called
THEN:
- The FK integer is stored on field.attname (correspondent_id),
not the descriptor attribute (correspondent)
"""
record = {
"model": "documents.document",
"pk": 1,
"fields": {
"title": "Test Doc",
"correspondent": 42,
"content": "",
"checksum": "abc123abc123abc123abc123abc123ab",
"filename": "0000001.pdf",
"mime_type": "application/pdf",
},
}
_, instance, _ = _deserialize_record(record)
assert instance.correspondent_id == 42
def test_m2m_field_collected_in_m2m_data(self) -> None:
"""
GIVEN:
- A manifest record for a Document with a tags M2M list
WHEN:
- _deserialize_record is called
THEN:
- M2M PKs are returned in m2m_data under the field name
"""
record = {
"model": "documents.document",
"pk": 1,
"fields": {
"title": "Test",
"tags": [1, 3, 7],
"content": "",
"checksum": "abc123abc123abc123abc123abc123ab",
"filename": "0000001.pdf",
"mime_type": "application/pdf",
},
}
_, _, m2m_data = _deserialize_record(record)
assert m2m_data["tags"] == [1, 3, 7]
def test_null_fk_stored_as_none(self) -> None:
"""
GIVEN:
- A manifest record with a nullable FK set to null
WHEN:
- _deserialize_record is called
THEN:
- The FK attname is None, not 0 or a string
"""
record = {
"model": "documents.document",
"pk": 2,
"fields": {
"title": "Test",
"correspondent": None,
"content": "",
"checksum": "def456def456def456def456def456de",
"filename": "0000002.pdf",
"mime_type": "application/pdf",
},
}
_, instance, _ = _deserialize_record(record)
assert instance.correspondent_id is None
def test_unknown_model_raises_deserialization_error(self) -> None:
"""
GIVEN:
- A manifest record with a model label that does not exist
WHEN:
- _deserialize_record is called
THEN:
- DeserializationError is raised
"""
from django.core.serializers.base import DeserializationError
record = {"model": "documents.doesnotexist", "pk": 1, "fields": {}}
with pytest.raises(DeserializationError):
_deserialize_record(record)
def test_invalid_pk_raises_deserialization_error(self) -> None:
"""
GIVEN:
- A manifest record whose pk value cannot be coerced to the field type
WHEN:
- _deserialize_record is called
THEN:
- DeserializationError is raised mentioning the bad pk value
"""
from django.core.serializers.base import DeserializationError
record = {"model": "documents.correspondent", "pk": "not-an-int", "fields": {}}
with pytest.raises(
DeserializationError,
match="Could not coerce pk=not-an-int",
):
_deserialize_record(record)
def test_invalid_scalar_field_value_raises_deserialization_error(self) -> None:
"""
GIVEN:
- A manifest record with a scalar field whose value cannot be coerced
WHEN:
- _deserialize_record is called
THEN:
- DeserializationError is raised mentioning the field and bad value
"""
from django.core.serializers.base import DeserializationError
record = {
"model": "documents.correspondent",
"pk": 1,
"fields": {"matching_algorithm": "not-an-int"},
}
with pytest.raises(
DeserializationError,
match="Could not coerce matching_algorithm=",
):
_deserialize_record(record)
def test_unknown_field_name_raises_field_does_not_exist(self) -> None:
"""
GIVEN:
- A manifest record with a field name that does not exist on the model
WHEN:
- _deserialize_record is called
THEN:
- FieldDoesNotExist is raised
"""
from django.core.exceptions import FieldDoesNotExist
record = {
"model": "documents.correspondent",
"pk": 1,
"fields": {"no_such_field_on_correspondent": "value"},
}
with pytest.raises(FieldDoesNotExist):
_deserialize_record(record)