Feature: store barcode contents, list and search them (#14276)

* Feature: store barcode contents, list and search them

New setting PAPERLESS_CONSUMER_STORE_BARCODE_VALUES (off by default)
stores all barcodes found during consumption with the document: page,
type and content. They are listed on the metadata tab with a copy
button, returned by the documents API and searchable with barcodes:
in the advanced search. Versions keep their own barcodes, reprocessing
reads them again. Refs #9898

* Tests: cover the remaining barcode branches

Covers unchanged and failing barcode reads on reprocessing, unsupported files and DocumentBarcode.__str__, and uses toHaveLength in the barcode list spec as suggested by SonarCloud.

* Address review: keep barcodes when they can't be read, format choices

- Reprocessing and new versions keep the stored barcodes when the file
  can't be scanned or the scan fails, and replace them atomically.
- Format is a TextChoices of the zxing-cpp formats, with a test.
- Shared scan code, latest_version helper, TypedDict, serializer reuse.
- Barcodes in the split manifest, export/import tests, pytest-style tests.

* Barcode tests: TIFF reprocess, format check both ways, fixtures

- Reprocessing a TIFF with TIFF support off keeps the stored barcodes.
- The format test also fails when zxing-cpp drops a format.
- Fixtures in place of the sample dir mixin, plugin-level disabled test.
- Format labels aren't translated, OpenAPI enum named BarcodeFormatEnum.

* Review: module-level zxing reader, shorter barcode docs

- read_barcodes_zxing is a module-level function used by scan_pdf.
- Drop the trivial __str__ test, mark it no cover.
- Shorten the barcode docs and remove the duplicate in configuration.md.

* Fix header

* Use utility class

* Return barcodes from the metadata endpoint only

Drop the barcodes field and its prefetches from the document serializer,
as agreed in the review. Also remove the now empty component stylesheet.

---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src/documents/barcodes.py

@@ -18,6 +18,7 @@ from documents.converters import convert_from_tiff_to_pdf
 from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentMetadataOverrides
 from documents.data_models import DocumentSource
+from documents.data_models import StoredBarcode
 from documents.models import Document
 from documents.models import PaperlessTask
 from documents.models import Tag
@@ -47,6 +48,7 @@ class Barcode:
     page: int
     value: str
     settings: BarcodeConfig
+    format: str = ""
 
     @property
     def is_separator(self) -> bool:
@@ -78,6 +80,12 @@ class Barcode:
                 return True
         return False
 
+    def stored(self) -> StoredBarcode:
+        """
+        The barcode as it is stored with a document, page 1-indexed
+        """
+        return {"page": self.page + 1, "value": self.value, "format": self.format}
+
 
 class BarcodePlugin(ConsumeTaskPlugin):
     NAME: str = "BarcodePlugin"
@@ -89,16 +97,12 @@ class BarcodePlugin(ConsumeTaskPlugin):
           - ASN from barcode detection is enabled or
           - Barcode support is enabled and the mime type is supported
         """
-        if self.settings.barcode_enable_tiff_support:
-            supported_mimes: set[str] = {"application/pdf", "image/tiff"}
-        else:
-            supported_mimes = {"application/pdf"}
-
         return (
             self.settings.barcode_enable_asn
             or self.settings.barcodes_enabled
             or self.settings.barcode_enable_tag
-        ) and self.input_doc.mime_type in supported_mimes
+            or self.settings.barcode_store_values
+        ) and self.input_doc.mime_type in scannable_mime_types(self.settings)
 
     def get_settings(self) -> BarcodeConfig:
         """
@@ -244,6 +248,10 @@ class BarcodePlugin(ConsumeTaskPlugin):
         if self.settings.barcode_enable_asn and (located_asn := self.asn) is not None:
             self._apply_detected_asn(located_asn)
 
+        # After splitting too, so each split document keeps its own barcodes
+        if self.settings.barcode_store_values:
+            self.metadata.barcodes = [x.stored() for x in self.barcodes] or None
+
     def cleanup(self) -> None:
         self.temp_dir.cleanup()
 
@@ -262,22 +270,6 @@ class BarcodePlugin(ConsumeTaskPlugin):
         )
         self._tiff_conversion_done = True
 
-    @staticmethod
-    def read_barcodes_zxing(image: Image.Image) -> list[str]:
-        barcodes = []
-
-        import zxingcpp
-
-        detected_barcodes = zxingcpp.read_barcodes(image)
-        for barcode in detected_barcodes:
-            if barcode.text:
-                barcodes.append(barcode.text)
-                logger.debug(
-                    f"Barcode of type {barcode.format} found: {barcode.text}",
-                )
-
-        return barcodes
-
     def detect(self) -> None:
         """
         Scan all pages of the PDF as images, updating barcodes and the pages
@@ -291,60 +283,12 @@ class BarcodePlugin(ConsumeTaskPlugin):
         self.convert_from_tiff_to_pdf()
 
         try:
-            # Read number of pages from pdf
-            with Pdf.open(self.pdf_file) as pdf:
-                num_of_pages = len(pdf.pages)
-            logger.debug(f"PDF has {num_of_pages} pages")
-
-            # Get limit from configuration
-            barcode_max_pages: int = (
-                num_of_pages
-                if self.settings.barcode_max_pages == 0
-                else self.settings.barcode_max_pages
+            self.barcodes = scan_pdf(
+                self.pdf_file,
+                self.settings,
+                Path(self.temp_dir.name),
             )
 
-            if barcode_max_pages < num_of_pages:  # pragma: no cover
-                logger.debug(
-                    f"Barcodes detection will be limited to the first {barcode_max_pages} pages",
-                )
-
-            # Loop al page
-            for current_page_number in range(min(num_of_pages, barcode_max_pages)):
-                logger.debug(f"Processing page {current_page_number}")
-
-                # Convert page to image
-                page = convert_from_path(
-                    self.pdf_file,
-                    dpi=self.settings.barcode_dpi,
-                    output_folder=self.temp_dir.name,
-                    first_page=current_page_number + 1,
-                    last_page=current_page_number + 1,
-                )[0]
-
-                # Remember filename, since it is lost by upscaling
-                page_filepath = Path(page.filename)
-                logger.debug(f"Image is at {page_filepath}")
-
-                # Upscale image if configured
-                factor = self.settings.barcode_upscale
-                if factor > 1.0:
-                    logger.debug(
-                        f"Upscaling image by {factor} for better barcode detection",
-                    )
-                    x, y = page.size
-                    page = page.resize(
-                        (round(x * factor), (round(y * factor))),
-                    )
-
-                # Detect barcodes
-                for barcode_value in self.read_barcodes_zxing(page):
-                    self.barcodes.append(
-                        Barcode(current_page_number, barcode_value, self.settings),
-                    )
-
-                # Delete temporary image file
-                page_filepath.unlink()
-
         # Password protected files can't be checked
         # This is the exception raised for those
         except PasswordError as e:
@@ -534,3 +478,111 @@ class BarcodePlugin(ConsumeTaskPlugin):
                 document_paths.append(savepath)
 
             return document_paths
+
+
+def scannable_mime_types(settings: BarcodeConfig) -> set[str]:
+    """
+    The file types the barcode scan supports with the current settings
+    """
+    if settings.barcode_enable_tiff_support:
+        return {"application/pdf", "image/tiff"}
+    return {"application/pdf"}
+
+
+def read_barcodes_zxing(image: Image.Image) -> list[tuple[str, str]]:
+    """
+    Returns the text and format (zxing enum name) of each barcode found in
+    the image
+    """
+    barcodes = []
+
+    import zxingcpp
+
+    detected_barcodes = zxingcpp.read_barcodes(image)
+    for barcode in detected_barcodes:
+        if barcode.text:
+            barcodes.append((barcode.text, barcode.format.name))
+            logger.debug(
+                f"Barcode of type {barcode.format} found: {barcode.text}",
+            )
+
+    return barcodes
+
+
+def scan_pdf(pdf_path: Path, settings: BarcodeConfig, work_dir: Path) -> list[Barcode]:
+    """
+    Scans the pages of a PDF as images for barcodes. Errors are not caught,
+    so callers can tell a failed scan from one that found nothing.
+    """
+    barcodes: list[Barcode] = []
+
+    with Pdf.open(pdf_path) as pdf:
+        num_of_pages = len(pdf.pages)
+    logger.debug(f"PDF has {num_of_pages} pages")
+
+    # Get limit from configuration
+    barcode_max_pages: int = (
+        num_of_pages if settings.barcode_max_pages == 0 else settings.barcode_max_pages
+    )
+
+    if barcode_max_pages < num_of_pages:  # pragma: no cover
+        logger.debug(
+            f"Barcodes detection will be limited to the first {barcode_max_pages} pages",
+        )
+
+    for current_page_number in range(min(num_of_pages, barcode_max_pages)):
+        logger.debug(f"Processing page {current_page_number}")
+
+        # Convert page to image
+        page = convert_from_path(
+            pdf_path,
+            dpi=settings.barcode_dpi,
+            output_folder=work_dir,
+            first_page=current_page_number + 1,
+            last_page=current_page_number + 1,
+        )[0]
+
+        # Remember filename, since it is lost by upscaling
+        page_filepath = Path(page.filename)
+        logger.debug(f"Image is at {page_filepath}")
+
+        # Upscale image if configured
+        factor = settings.barcode_upscale
+        if factor > 1.0:
+            logger.debug(
+                f"Upscaling image by {factor} for better barcode detection",
+            )
+            x, y = page.size
+            page = page.resize(
+                (round(x * factor), (round(y * factor))),
+            )
+
+        for barcode_value, barcode_format in read_barcodes_zxing(page):
+            barcodes.append(
+                Barcode(current_page_number, barcode_value, settings, barcode_format),
+            )
+
+        # Delete temporary image file
+        page_filepath.unlink()
+
+    return barcodes
+
+
+def read_barcode_values(
+    path: Path,
+    mime_type: str,
+    settings: BarcodeConfig,
+    work_dir: Path,
+) -> list[StoredBarcode] | None:
+    """
+    Reads the barcodes of a file outside of the consumption plugins: for new
+    versions, which skip the barcode plugin, and when reprocessing.
+
+    Returns None if the file can't be scanned with the current settings.
+    Errors while scanning are raised.
+    """
+    if mime_type not in scannable_mime_types(settings):
+        return None
+    if mime_type == "image/tiff":
+        path = convert_from_tiff_to_pdf(path, work_dir)
+    return [x.stored() for x in scan_pdf(path, settings, work_dir)]