mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-06 08:10:35 +00:00
jurassicparkicecreamandshamoon
62cc31fbdb
* Feature: store barcode contents, list and search them New setting PAPERLESS_CONSUMER_STORE_BARCODE_VALUES (off by default) stores all barcodes found during consumption with the document: page, type and content. They are listed on the metadata tab with a copy button, returned by the documents API and searchable with barcodes: in the advanced search. Versions keep their own barcodes, reprocessing reads them again. Refs #9898 * Tests: cover the remaining barcode branches Covers unchanged and failing barcode reads on reprocessing, unsupported files and DocumentBarcode.__str__, and uses toHaveLength in the barcode list spec as suggested by SonarCloud. * Address review: keep barcodes when they can't be read, format choices - Reprocessing and new versions keep the stored barcodes when the file can't be scanned or the scan fails, and replace them atomically. - Format is a TextChoices of the zxing-cpp formats, with a test. - Shared scan code, latest_version helper, TypedDict, serializer reuse. - Barcodes in the split manifest, export/import tests, pytest-style tests. * Barcode tests: TIFF reprocess, format check both ways, fixtures - Reprocessing a TIFF with TIFF support off keeps the stored barcodes. - The format test also fails when zxing-cpp drops a format. - Fixtures in place of the sample dir mixin, plugin-level disabled test. - Format labels aren't translated, OpenAPI enum named BarcodeFormatEnum. * Review: module-level zxing reader, shorter barcode docs - read_barcodes_zxing is a module-level function used by scan_pdf. - Drop the trivial __str__ test, mark it no cover. - Shorten the barcode docs and remove the duplicate in configuration.md. * Fix header * Use utility class * Return barcodes from the metadata endpoint only Drop the barcodes field and its prefetches from the document serializer, as agreed in the review. Also remove the now empty component stylesheet. --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
96 lines
3.6 KiB
Python
96 lines
3.6 KiB
Python
"""Every declared JSON subpath must actually be written to the index.
|
|
|
|
PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` ->
|
|
{"user", "note"}), but nothing coupled that declaration to what
|
|
``_backend.py``'s document builder actually writes into the JSON blob at
|
|
index time. A subpath declared but never written would be
|
|
queryable-but-always-empty -- syntactically valid, silently matching
|
|
nothing -- with no test failure anywhere.
|
|
|
|
This indexes one real document carrying values for every JSON field
|
|
(a Note, a CustomFieldInstance, a DocumentBarcode) and inspects the document's own stored
|
|
JSON payload, rather than running field-specific queries: that way a
|
|
future JSON field's subpaths are covered automatically, without a new
|
|
per-subpath query having to be added by hand each time.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
import tantivy
|
|
from whoosh_compat import FieldKind
|
|
|
|
from documents.models import CustomField
|
|
from documents.models import CustomFieldInstance
|
|
from documents.models import Document
|
|
from documents.models import Note
|
|
from documents.search._fields import PUBLIC_FIELDS
|
|
from paperless_testing.factories import DocumentBarcodeFactory
|
|
from paperless_testing.factories import UserFactory
|
|
|
|
if TYPE_CHECKING:
|
|
from documents.search._backend import TantivyBackend
|
|
|
|
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
|
|
|
|
|
class TestJsonSubpathsAreWrittenAtIndexTime:
|
|
def test_every_declared_json_subpath_appears_in_the_stored_document(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document with a Note, a CustomFieldInstance and a
|
|
DocumentBarcode attached
|
|
WHEN:
|
|
- The document is indexed via TantivyBackend.add_or_update
|
|
THEN:
|
|
- Every subpath PUBLIC_FIELDS declares for notes/custom_fields/barcodes
|
|
is present as a key in the document's stored JSON payload
|
|
"""
|
|
user = UserFactory(username="completeness-user")
|
|
field = CustomField.objects.create(
|
|
name="Completeness Field",
|
|
data_type=CustomField.FieldDataType.STRING,
|
|
)
|
|
doc = Document.objects.create(
|
|
title="Completeness doc",
|
|
content="x",
|
|
checksum="json-subpath-completeness",
|
|
)
|
|
Note.objects.create(document=doc, user=user, note="a note")
|
|
CustomFieldInstance.objects.create(
|
|
document=doc,
|
|
field=field,
|
|
value_text="a value",
|
|
)
|
|
DocumentBarcodeFactory(document=doc, value="a barcode")
|
|
backend.add_or_update(doc)
|
|
|
|
index = backend._index
|
|
searcher = index.searcher()
|
|
hits = searcher.search(
|
|
tantivy.Query.term_query(index.schema, "id", doc.pk),
|
|
limit=1,
|
|
).hits
|
|
assert hits, "the document was not indexed"
|
|
stored = searcher.doc(hits[0][1]).to_dict()
|
|
|
|
json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON]
|
|
assert json_fields, "no JSON fields declared - fixture is stale"
|
|
for field_spec in json_fields:
|
|
stored_values = stored.get(field_spec.name)
|
|
assert stored_values, (
|
|
f"{field_spec.name} was not written to the index at all"
|
|
)
|
|
written_keys = stored_values[0].keys()
|
|
for subpath in field_spec.subpaths:
|
|
assert subpath in written_keys, (
|
|
f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS "
|
|
"but _backend.py's document builder never writes it - it "
|
|
"would be queryable but always empty"
|
|
)
|