mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-30 07:44:54 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6704ac39e3 | ||
|
|
f05b4b1ef0 | ||
|
|
350684cd6b | ||
|
|
668fa77428 | ||
|
|
5bd72014a6 | ||
|
|
bbb9c86ba4 |
@@ -948,11 +948,10 @@ for display in the web interface.
|
||||
|
||||
!!! note
|
||||
|
||||
The **remote OCR parser** (Azure AI) also honors this setting: when
|
||||
no archive is requested (`never`, or `auto` with a born-digital PDF),
|
||||
the remote engine is skipped entirely and locally-extracted text is
|
||||
used instead, avoiding an unnecessary API call and a duplicate text
|
||||
layer.
|
||||
The **remote OCR parser** (Azure AI) always produces a searchable
|
||||
PDF and stores it as the archive copy, regardless of this setting.
|
||||
`ARCHIVE_FILE_GENERATION=never` has no effect when the remote
|
||||
parser handles a document.
|
||||
|
||||
#### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN}
|
||||
|
||||
|
||||
@@ -187,11 +187,10 @@ PAPERLESS_ARCHIVE_FILE_GENERATION=auto
|
||||
|
||||
### Remote OCR parser
|
||||
|
||||
If you use the **remote OCR parser** (Azure AI), `ARCHIVE_FILE_GENERATION` is
|
||||
honored the same way as for the local engine: when no archive is requested
|
||||
(`never`, or `auto` with a born-digital PDF), the remote engine is skipped
|
||||
entirely and locally-extracted text is used instead, avoiding an unnecessary
|
||||
API call and a duplicate text layer.
|
||||
If you use the **remote OCR parser** (Azure AI), note that it always produces a
|
||||
searchable PDF and stores it as the archive copy. `ARCHIVE_FILE_GENERATION=never`
|
||||
has no effect for documents handled by the remote parser - the archive is produced
|
||||
unconditionally by the remote engine.
|
||||
|
||||
## Search Index (Whoosh -> Tantivy)
|
||||
|
||||
|
||||
+5
@@ -68,6 +68,11 @@
|
||||
></ng-select>
|
||||
} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.DocumentLink) {
|
||||
<pngx-input-document-link [(ngModel)]="atom.value" class="w-25 form-select doc-link-select p-0" placeholder="Search docs..." i18n-placeholder [minimal]="true"></pngx-input-document-link>
|
||||
} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Monetary) {
|
||||
<input class="w-25 form-control rounded-end" type="text" inputmode="decimal"
|
||||
[ngModel]="atom.value"
|
||||
(ngModelChange)="setMonetaryValue(atom, $event)"
|
||||
[disabled]="disabled">
|
||||
} @else {
|
||||
<input class="w-25 form-control rounded-end" type="text" [(ngModel)]="atom.value" [disabled]="disabled">
|
||||
}
|
||||
|
||||
+24
@@ -1,5 +1,6 @@
|
||||
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'
|
||||
import { provideHttpClientTesting } from '@angular/common/http/testing'
|
||||
import { LOCALE_ID } from '@angular/core'
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
||||
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
|
||||
import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
|
||||
@@ -41,6 +42,12 @@ const customFields = [
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
name: 'Test Monetary Field',
|
||||
data_type: CustomFieldDataType.Monetary,
|
||||
extra_data: { default_currency: 'EUR' },
|
||||
},
|
||||
]
|
||||
|
||||
describe('CustomFieldsQueryDropdownComponent', () => {
|
||||
@@ -61,6 +68,7 @@ describe('CustomFieldsQueryDropdownComponent', () => {
|
||||
providers: [
|
||||
provideHttpClient(withInterceptorsFromDi()),
|
||||
provideHttpClientTesting(),
|
||||
{ provide: LOCALE_ID, useValue: 'de' },
|
||||
],
|
||||
}).compileComponents()
|
||||
|
||||
@@ -150,6 +158,22 @@ describe('CustomFieldsQueryDropdownComponent', () => {
|
||||
expect(options2).toEqual([])
|
||||
})
|
||||
|
||||
it('should normalize localized monetary comparison values', () => {
|
||||
const atom = new CustomFieldQueryAtom([3, 'exact', null])
|
||||
|
||||
component.setMonetaryValue(atom, '1.234,56')
|
||||
|
||||
expect(atom.value).toEqual('1234.56')
|
||||
})
|
||||
|
||||
it('should preserve API-formatted monetary comparison values', () => {
|
||||
const atom = new CustomFieldQueryAtom([3, 'exact', null])
|
||||
|
||||
component.setMonetaryValue(atom, '1234.56')
|
||||
|
||||
expect(atom.value).toEqual('1234.56')
|
||||
})
|
||||
|
||||
it('should remove an element from the selection model', () => {
|
||||
const expression = new CustomFieldQueryExpression()
|
||||
const atom = new CustomFieldQueryAtom()
|
||||
|
||||
+21
-1
@@ -1,9 +1,14 @@
|
||||
import { NgTemplateOutlet } from '@angular/common'
|
||||
import {
|
||||
getLocaleNumberSymbol,
|
||||
NgTemplateOutlet,
|
||||
NumberSymbol,
|
||||
} from '@angular/common'
|
||||
import {
|
||||
Component,
|
||||
EventEmitter,
|
||||
inject,
|
||||
Input,
|
||||
LOCALE_ID,
|
||||
Output,
|
||||
QueryList,
|
||||
signal,
|
||||
@@ -212,6 +217,7 @@ export class CustomFieldQueriesModel {
|
||||
})
|
||||
export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPermissions {
|
||||
protected customFieldsService = inject(CustomFieldsService)
|
||||
private locale = inject(LOCALE_ID)
|
||||
|
||||
public CustomFieldQueryComponentType = CustomFieldQueryElementType
|
||||
public CustomFieldQueryOperator = CustomFieldQueryOperator
|
||||
@@ -376,4 +382,18 @@ export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPerm
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
setMonetaryValue(atom: CustomFieldQueryAtom, value: string) {
|
||||
// Normalize the decimal symbol e.g. . vs , by locale
|
||||
const decimalSymbol = getLocaleNumberSymbol(
|
||||
this.locale,
|
||||
NumberSymbol.Decimal
|
||||
)
|
||||
if (decimalSymbol !== '.' && value.includes(decimalSymbol)) {
|
||||
const groupSymbol = getLocaleNumberSymbol(this.locale, NumberSymbol.Group)
|
||||
value = value.split(groupSymbol).join('').split(decimalSymbol).join('.')
|
||||
}
|
||||
|
||||
atom.value = value
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,13 +129,25 @@ describe('PngxPdfViewerComponent', () => {
|
||||
;(component as any).applyScale()
|
||||
expect(viewer.currentScaleValue).toBe(PdfZoomScale.PageFit)
|
||||
expect(viewer.currentScale).toBe(2)
|
||||
})
|
||||
|
||||
it('does not reapply scale for page-only changes', async () => {
|
||||
await initComponent()
|
||||
|
||||
const pdf = (component as any).pdf as { numPages: number }
|
||||
pdf.numPages = 3
|
||||
const viewer = (component as any).pdfViewer as PDFViewer
|
||||
viewer.setDocument(pdf)
|
||||
const applyScaleSpy = jest.spyOn(component as any, 'applyScale')
|
||||
component.page = 2
|
||||
;(component as any).lastViewerPage = 2
|
||||
;(component as any).applyViewerState()
|
||||
|
||||
component.ngOnChanges({
|
||||
page: new SimpleChange(1, 2, false),
|
||||
})
|
||||
|
||||
expect(viewer.currentPageNumber).toBe(2)
|
||||
expect((component as any).lastViewerPage).toBeUndefined()
|
||||
expect(applyScaleSpy).toHaveBeenCalled()
|
||||
expect(applyScaleSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not reset the viewer when it is already on the requested page', async () => {
|
||||
|
||||
@@ -116,7 +116,10 @@ export class PngxPdfViewerComponent
|
||||
changes['zoomScale'] ||
|
||||
changes['rotation']
|
||||
) {
|
||||
this.applyViewerState()
|
||||
// Prevent loop with page / scale application see https://github.com/paperless-ngx/paperless-ngx/issues/13404
|
||||
this.applyViewerState(
|
||||
!!(changes['zoom'] || changes['zoomScale'] || changes['rotation'])
|
||||
)
|
||||
}
|
||||
|
||||
if (changes['searchQuery']) {
|
||||
@@ -240,7 +243,7 @@ export class PngxPdfViewerComponent
|
||||
}
|
||||
}
|
||||
|
||||
private applyViewerState(): void {
|
||||
private applyViewerState(applyScale = true): void {
|
||||
if (!this.pdfViewer) {
|
||||
return
|
||||
}
|
||||
@@ -264,7 +267,7 @@ export class PngxPdfViewerComponent
|
||||
if (this.page === this.lastViewerPage) {
|
||||
this.lastViewerPage = undefined
|
||||
}
|
||||
if (hasPages) {
|
||||
if (hasPages && applyScale) {
|
||||
this.applyScale()
|
||||
}
|
||||
this.dispatchFindIfReady()
|
||||
|
||||
+15
-25
@@ -57,9 +57,7 @@ from paperless.models import ArchiveFileGenerationChoices
|
||||
from paperless.parsers import ParserContext
|
||||
from paperless.parsers import ParserProtocol
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH
|
||||
from paperless.parsers.utils import extract_pdf_text
|
||||
from paperless.parsers.utils import is_tagged_pdf
|
||||
from paperless.parsers.utils import pdf_born_digital_text
|
||||
|
||||
LOGGING_NAME: Final[str] = "paperless.consumer"
|
||||
|
||||
@@ -138,53 +136,45 @@ def should_produce_archive(
|
||||
|
||||
# Must produce a PDF so the frontend can display the original format at all.
|
||||
if parser.requires_pdf_rendition:
|
||||
_log.debug("Archive: yes — parser requires PDF rendition for frontend display")
|
||||
_log.debug("Archive: yes - parser requires PDF rendition for frontend display")
|
||||
return True
|
||||
|
||||
# Parser cannot produce an archive (e.g. TextDocumentParser).
|
||||
if not parser.can_produce_archive:
|
||||
_log.debug("Archive: no — parser cannot produce archives")
|
||||
_log.debug("Archive: no - parser cannot produce archives")
|
||||
return False
|
||||
|
||||
generation = OcrConfig().archive_file_generation
|
||||
|
||||
if generation == ArchiveFileGenerationChoices.ALWAYS:
|
||||
_log.debug("Archive: yes — ARCHIVE_FILE_GENERATION=always")
|
||||
_log.debug("Archive: yes - ARCHIVE_FILE_GENERATION=always")
|
||||
return True
|
||||
if generation == ArchiveFileGenerationChoices.NEVER:
|
||||
_log.debug("Archive: no — ARCHIVE_FILE_GENERATION=never")
|
||||
_log.debug("Archive: no - ARCHIVE_FILE_GENERATION=never")
|
||||
return False
|
||||
|
||||
# auto: produce archives for scanned/image documents; skip for born-digital PDFs.
|
||||
if mime_type.startswith("image/"):
|
||||
_log.debug("Archive: yes — image document, ARCHIVE_FILE_GENERATION=auto")
|
||||
_log.debug("Archive: yes - image document, ARCHIVE_FILE_GENERATION=auto")
|
||||
return True
|
||||
if mime_type == "application/pdf":
|
||||
text = extract_pdf_text(document_path)
|
||||
has_text = text is not None and len(text) > 0
|
||||
if has_text and is_tagged_pdf(document_path):
|
||||
text, born_digital = pdf_born_digital_text(document_path, log=_log)
|
||||
text_length = len(text) if text else 0
|
||||
if born_digital:
|
||||
_log.debug(
|
||||
"Archive: no — born-digital PDF (structure tags detected),"
|
||||
"Archive: no - born-digital PDF (text_length=%d),"
|
||||
" ARCHIVE_FILE_GENERATION=auto",
|
||||
text_length,
|
||||
)
|
||||
return False
|
||||
if text is None or len(text) <= PDF_TEXT_MIN_LENGTH:
|
||||
_log.debug(
|
||||
"Archive: yes — scanned PDF (text_length=%d ≤ %d),"
|
||||
" ARCHIVE_FILE_GENERATION=auto",
|
||||
len(text) if text else 0,
|
||||
PDF_TEXT_MIN_LENGTH,
|
||||
)
|
||||
return True
|
||||
_log.debug(
|
||||
"Archive: no — born-digital PDF (text_length=%d > %d),"
|
||||
"Archive: yes - scanned/textless PDF (text_length=%d),"
|
||||
" ARCHIVE_FILE_GENERATION=auto",
|
||||
len(text),
|
||||
PDF_TEXT_MIN_LENGTH,
|
||||
text_length,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
_log.debug(
|
||||
"Archive: no — MIME type %r not eligible for auto archive generation",
|
||||
"Archive: no - MIME type %r not eligible for auto archive generation",
|
||||
mime_type,
|
||||
)
|
||||
return False
|
||||
|
||||
@@ -36,6 +36,9 @@ def send_email(
|
||||
|
||||
TODO: re-evaluate this pending https://code.djangoproject.com/ticket/35581 / https://github.com/django/django/pull/18966
|
||||
"""
|
||||
if "\r" in subject or "\n" in subject:
|
||||
subject = " ".join(line.strip(" \t") for line in subject.splitlines())
|
||||
|
||||
email = EmailMessage(
|
||||
subject=subject,
|
||||
body=body,
|
||||
|
||||
@@ -386,10 +386,19 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
raise DeserializationError(
|
||||
f"{model.__name__} has no updatable fields; PK-only models are not supported by the importer",
|
||||
)
|
||||
# MySQL/MariaDB support upserts via ON DUPLICATE KEY UPDATE but,
|
||||
# unlike PostgreSQL/SQLite, cannot target a specific unique field
|
||||
# for the conflict -- passing unique_fields there raises
|
||||
# NotSupportedError.
|
||||
unique_fields = (
|
||||
[model._meta.pk.attname]
|
||||
if connection.features.supports_update_conflicts_with_target
|
||||
else None
|
||||
)
|
||||
model.objects.bulk_create( # type: ignore[attr-defined]
|
||||
instances,
|
||||
update_conflicts=True,
|
||||
unique_fields=[model._meta.pk.attname],
|
||||
unique_fields=unique_fields,
|
||||
update_fields=update_fields,
|
||||
)
|
||||
loaded_models.add(model)
|
||||
|
||||
@@ -75,7 +75,7 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
|
||||
{
|
||||
"documents": [self.doc1.pk, self.doc2.pk],
|
||||
"addresses": "hello@paperless-ngx.com,test@example.com",
|
||||
"subject": "Bulk email test",
|
||||
"subject": "Bulk email\n test",
|
||||
"message": "Here are your documents",
|
||||
},
|
||||
),
|
||||
|
||||
@@ -1329,7 +1329,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
||||
with self.get_consumer(self.test_file) as c:
|
||||
c.run()
|
||||
# Verify no pre-consume script subprocess was invoked
|
||||
# (run_subprocess may still be called by _extract_text_for_archive_check)
|
||||
# (run_subprocess may still be called by pdf_born_digital_text via pdftotext)
|
||||
script_calls = [
|
||||
call
|
||||
for call in m.call_args_list
|
||||
@@ -1354,7 +1354,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
||||
self.assertTrue(m.called)
|
||||
|
||||
# Find the call that invoked the pre-consume script
|
||||
# (run_subprocess may also be called by _extract_text_for_archive_check)
|
||||
# (run_subprocess may also be called by pdf_born_digital_text via pdftotext)
|
||||
script_call = next(
|
||||
call
|
||||
for call in m.call_args_list
|
||||
|
||||
@@ -134,60 +134,32 @@ class TestShouldProduceArchive:
|
||||
assert should_produce_archive(parser, mime, Path("/tmp/doc")) is expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extracted_text", "expected"),
|
||||
("born_digital", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"This is a born-digital PDF with lots of text content. " * 10,
|
||||
False,
|
||||
id="born-digital-long-text-skips-archive",
|
||||
),
|
||||
pytest.param(None, True, id="no-text-scanned-produces-archive"),
|
||||
pytest.param("tiny", True, id="short-text-treated-as-scanned"),
|
||||
pytest.param(True, False, id="born-digital-skips-archive"),
|
||||
pytest.param(False, True, id="not-born-digital-produces-archive"),
|
||||
],
|
||||
)
|
||||
def test_auto_pdf_archive_decision(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
settings,
|
||||
extracted_text: str | None,
|
||||
born_digital: bool, # noqa: FBT001
|
||||
expected: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""Archive decision tracks pdf_born_digital_text()'s verdict exactly.
|
||||
|
||||
should_produce_archive() defers entirely to pdf_born_digital_text()
|
||||
for the has-real-text decision, so both callers of that predicate
|
||||
(this function and RasterisedDocumentParser.parse()) always agree.
|
||||
"""
|
||||
settings.ARCHIVE_FILE_GENERATION = "auto"
|
||||
mocker.patch("documents.consumer.is_tagged_pdf", return_value=False)
|
||||
mocker.patch("documents.consumer.extract_pdf_text", return_value=extracted_text)
|
||||
mocker.patch(
|
||||
"documents.consumer.pdf_born_digital_text",
|
||||
return_value=("some text", born_digital),
|
||||
)
|
||||
parser = _parser_instance(can_produce=True, requires_rendition=False)
|
||||
assert (
|
||||
should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
|
||||
is expected
|
||||
)
|
||||
|
||||
def test_tagged_pdf_skips_archive_in_auto_mode(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
settings,
|
||||
) -> None:
|
||||
"""Tagged PDFs (e.g. Word exports) with real text are treated as born-digital, even below PDF_TEXT_MIN_LENGTH."""
|
||||
settings.ARCHIVE_FILE_GENERATION = "auto"
|
||||
mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
|
||||
mocker.patch("documents.consumer.extract_pdf_text", return_value="tiny")
|
||||
parser = _parser_instance(can_produce=True, requires_rendition=False)
|
||||
assert (
|
||||
should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
|
||||
is False
|
||||
)
|
||||
|
||||
def test_tagged_pdf_without_text_produces_archive(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
settings,
|
||||
) -> None:
|
||||
"""A tagged PDF with no actual extractable text (e.g. some scanner firmware) is not
|
||||
trusted as born-digital — the tag alone must not bypass OCR."""
|
||||
settings.ARCHIVE_FILE_GENERATION = "auto"
|
||||
mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
|
||||
mocker.patch("documents.consumer.extract_pdf_text", return_value=None)
|
||||
parser = _parser_instance(can_produce=True, requires_rendition=False)
|
||||
assert (
|
||||
should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
|
||||
is True
|
||||
)
|
||||
|
||||
@@ -3,9 +3,7 @@ Built-in remote-OCR document parser.
|
||||
|
||||
Handles documents by sending them to a configured remote OCR engine
|
||||
(currently Azure AI Vision / Document Intelligence) and retrieving both
|
||||
the extracted text and a searchable PDF with an embedded text layer. For
|
||||
born-digital PDFs that need no archive copy, the remote call is skipped
|
||||
entirely in favor of locally-extracted text (see ``RemoteDocumentParser.parse``).
|
||||
the extracted text and a searchable PDF with an embedded text layer.
|
||||
|
||||
When no engine is configured, ``score()`` returns ``None`` so the parser
|
||||
is effectively invisible to the registry — the tesseract parser handles
|
||||
@@ -23,8 +21,6 @@ from typing import Self
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from paperless.parsers.utils import extract_pdf_text
|
||||
from paperless.parsers.utils import post_process_text
|
||||
from paperless.version import __full_version_str__
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -73,11 +69,8 @@ class RemoteDocumentParser:
|
||||
"""Parse documents via a remote OCR API (currently Azure AI Vision).
|
||||
|
||||
This parser sends documents to a remote engine that returns both
|
||||
extracted text and a searchable PDF with an embedded text layer,
|
||||
except when ``parse()`` is called with ``produce_archive=False`` for
|
||||
a PDF, in which case the remote call is skipped and only locally
|
||||
extracted text is returned (no archive). It does not depend on
|
||||
Tesseract or ocrmypdf.
|
||||
extracted text and a searchable PDF with an embedded text layer.
|
||||
It does not depend on Tesseract or ocrmypdf.
|
||||
|
||||
Class attributes
|
||||
----------------
|
||||
@@ -166,11 +159,8 @@ class RemoteDocumentParser:
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
Always True — the remote engine is capable of returning a PDF
|
||||
with an embedded text layer to serve as the archive copy.
|
||||
Whether it actually does so for a given document depends on
|
||||
``produce_archive`` passed to :meth:`parse` (see there for when
|
||||
the remote engine call, and thus archive generation, is skipped).
|
||||
Always True — the remote engine always returns a PDF with an
|
||||
embedded text layer that serves as the archive copy.
|
||||
"""
|
||||
return True
|
||||
|
||||
@@ -227,12 +217,6 @@ class RemoteDocumentParser:
|
||||
) -> None:
|
||||
"""Send the document to the remote engine and store results.
|
||||
|
||||
When *produce_archive* is False for a PDF, the caller (via
|
||||
``documents.consumer.should_produce_archive``) has already determined
|
||||
that the document is born-digital and needs no archive — skip the
|
||||
remote engine entirely rather than re-OCRing it and creating a
|
||||
duplicate text layer.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
document_path:
|
||||
@@ -240,8 +224,8 @@ class RemoteDocumentParser:
|
||||
mime_type:
|
||||
Detected MIME type of the document.
|
||||
produce_archive:
|
||||
Whether an archive copy is wanted. For PDFs, False skips the
|
||||
remote engine and uses locally-extracted text instead.
|
||||
Ignored — the remote engine always returns a searchable PDF,
|
||||
which is stored as the archive copy regardless of this flag.
|
||||
"""
|
||||
config = RemoteEngineConfig(
|
||||
engine=settings.REMOTE_OCR_ENGINE,
|
||||
@@ -256,16 +240,6 @@ class RemoteDocumentParser:
|
||||
self._text = ""
|
||||
return
|
||||
|
||||
if not produce_archive and mime_type == "application/pdf":
|
||||
logger.debug(
|
||||
"Remote OCR: skipped — no archive requested, "
|
||||
"using locally-extracted text",
|
||||
)
|
||||
self._text = (
|
||||
post_process_text(extract_pdf_text(document_path, log=logger)) or ""
|
||||
)
|
||||
return
|
||||
|
||||
if config.engine == "azureai":
|
||||
self._text = self._azure_ai_vision_parse(document_path, config)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ from paperless.models import CleanChoices
|
||||
from paperless.models import ModeChoices
|
||||
from paperless.models import OutputTypeChoices
|
||||
from paperless.parsers.utils import extract_pdf_text
|
||||
from paperless.parsers.utils import pdf_has_digital_text
|
||||
from paperless.parsers.utils import is_born_digital_text
|
||||
from paperless.parsers.utils import post_process_text
|
||||
from paperless.parsers.utils import read_file_handle_unicode_errors
|
||||
from paperless.version import __full_version_str__
|
||||
@@ -509,9 +509,9 @@ class RasterisedDocumentParser:
|
||||
|
||||
if mime_type == "application/pdf":
|
||||
text_original = self.extract_text(None, document_path)
|
||||
original_has_text = pdf_has_digital_text(
|
||||
document_path,
|
||||
original_has_text = is_born_digital_text(
|
||||
text_original,
|
||||
document_path,
|
||||
log=self.log,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -65,63 +65,6 @@ def is_tagged_pdf(
|
||||
return False
|
||||
|
||||
|
||||
def pdf_has_digital_text(
|
||||
path: Path,
|
||||
text: str | None,
|
||||
log: logging.Logger | None = None,
|
||||
) -> bool:
|
||||
"""Return True if a PDF already has a usable, born-digital text layer.
|
||||
|
||||
Combines the tagged-PDF check with an extracted-text length check.
|
||||
Shared by the tesseract and remote OCR parsers to decide whether
|
||||
OCR_MODE=auto/off should skip (re-)OCRing a document.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Absolute path to the PDF file.
|
||||
text:
|
||||
Text already extracted from the PDF (e.g. via ``extract_pdf_text``),
|
||||
or ``None``.
|
||||
log:
|
||||
Logger for warnings. Falls back to the module-level logger when omitted.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
``True`` when the document already contains a text layer.
|
||||
"""
|
||||
return is_tagged_pdf(path, log=log) or (
|
||||
text is not None and len(text) > PDF_TEXT_MIN_LENGTH
|
||||
)
|
||||
|
||||
|
||||
def post_process_text(text: str | None) -> str | None:
|
||||
"""Normalise whitespace in extracted OCR/PDF text and strip NUL bytes.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text:
|
||||
Raw extracted text, or ``None``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
Cleaned text, or ``None`` when *text* is falsy.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
|
||||
no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
|
||||
no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
|
||||
|
||||
# TODO: this needs a rework
|
||||
# replace \0 prevents issues with saving to postgres.
|
||||
# text may contain \0 when this character is present in PDF files.
|
||||
return no_trailing_whitespace.strip().replace("\0", " ")
|
||||
|
||||
|
||||
def extract_pdf_text(
|
||||
path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
@@ -168,6 +111,88 @@ def extract_pdf_text(
|
||||
return None
|
||||
|
||||
|
||||
def post_process_text(text: str | None) -> str | None:
|
||||
"""Normalize extracted PDF/OCR text: collapse whitespace, strip padding.
|
||||
|
||||
Returns ``None`` for ``None`` or whitespace-only input, so callers can
|
||||
treat "no text" and "only layout padding" the same way.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
|
||||
no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
|
||||
no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
|
||||
|
||||
# replace \0 prevents issues with saving to postgres.
|
||||
# text may contain \0 when this character is present in PDF files.
|
||||
result = no_trailing_whitespace.strip().replace("\0", " ")
|
||||
return result or None
|
||||
|
||||
|
||||
def is_born_digital_text(
|
||||
text: str | None,
|
||||
path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
) -> bool:
|
||||
"""Decide whether already-extracted, normalized PDF text counts as born-digital.
|
||||
|
||||
This is the single source of truth for "does this PDF already have real
|
||||
text", used both to decide whether to produce an archive file and to
|
||||
decide whether OCR can be skipped. Both decisions must agree, or a
|
||||
tagged-but-textless PDF can end up with no archive AND a forced OCR pass
|
||||
(see GH #13387): raw ``pdftotext -layout`` output can be non-empty
|
||||
(whitespace/form-feed padding) even when there is no real content, so
|
||||
*text* must already be normalized via :func:`post_process_text`, not the
|
||||
raw extraction.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text:
|
||||
The normalized extracted text (or ``None``) to evaluate.
|
||||
path:
|
||||
Absolute path to the PDF file, used for the tagged-PDF check.
|
||||
log:
|
||||
Logger for warnings. Falls back to the module-level logger when omitted.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
Whether the PDF counts as born-digital (has real text, and is either
|
||||
tagged or exceeds ``PDF_TEXT_MIN_LENGTH``).
|
||||
"""
|
||||
if not text:
|
||||
return False
|
||||
return is_tagged_pdf(path, log=log) or len(text) > PDF_TEXT_MIN_LENGTH
|
||||
|
||||
|
||||
def pdf_born_digital_text(
|
||||
path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
) -> tuple[str | None, bool]:
|
||||
"""Extract a PDF's text and decide whether it should be treated as born-digital.
|
||||
|
||||
Convenience wrapper around :func:`is_born_digital_text` for callers that
|
||||
don't already have the PDF's text extracted (e.g. the archive-generation
|
||||
decision, which runs before any parser has touched the file).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Absolute path to the PDF file.
|
||||
log:
|
||||
Logger for warnings. Falls back to the module-level logger when omitted.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[str | None, bool]
|
||||
The normalized extracted text (or ``None``), and whether the PDF
|
||||
counts as born-digital.
|
||||
"""
|
||||
text = post_process_text(extract_pdf_text(path, log=log))
|
||||
return text, is_born_digital_text(text, path, log=log)
|
||||
|
||||
|
||||
def read_file_handle_unicode_errors(
|
||||
filepath: Path,
|
||||
log: logging.Logger | None = None,
|
||||
|
||||
@@ -36,6 +36,23 @@ def samples_dir() -> Path:
|
||||
return (Path(__file__).parent / "samples").resolve()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def tagged_no_text_pdf_file(samples_dir: Path) -> Path:
|
||||
"""Path to a tagged PDF whose only "text" is pdftotext layout padding.
|
||||
|
||||
Reproduces GH #13387: ``/MarkInfo /Marked true`` is set, but the only
|
||||
extractable content is a form-feed byte, not real text. Lives here
|
||||
rather than in parsers/conftest.py so both parser tests and
|
||||
paperless/tests/test_parser_utils.py can use it.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
Absolute path to ``tesseract/tagged-but-no-text.pdf``.
|
||||
"""
|
||||
return samples_dir / "tesseract" / "tagged-but-no-text.pdf"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_registry() -> Generator[None, None, None]:
|
||||
"""Reset the parser registry before and after every test.
|
||||
|
||||
@@ -336,117 +336,6 @@ class TestRemoteParserParse:
|
||||
assert remote_parser.get_date() is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse() — produce_archive=False skips the remote engine (PDFs only)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRemoteParserSkipsWhenNoArchiveWanted:
|
||||
"""When the caller has already decided no archive is needed for a PDF
|
||||
(documents.consumer.should_produce_archive), the remote engine call is
|
||||
skipped entirely in favor of locally-extracted text.
|
||||
"""
|
||||
|
||||
def test_pdf_skips_azure_when_no_archive_requested(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN: produce_archive=False for a PDF
|
||||
WHEN: parse() is called
|
||||
THEN: Azure is never invoked, no archive is produced, and text
|
||||
comes from local pdftotext extraction
|
||||
"""
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
azure_client.begin_analyze_document.assert_not_called()
|
||||
assert remote_parser.get_archive_path() is None
|
||||
assert remote_parser.get_text() != ""
|
||||
|
||||
def test_pdf_no_archive_requested_text_matches_local_extraction(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN: produce_archive=False for a PDF
|
||||
WHEN: parse() is called
|
||||
THEN: the returned text is exactly the locally-extracted text,
|
||||
not anything from the (unused) Azure mock
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.remote.extract_pdf_text",
|
||||
return_value="Local digital text.",
|
||||
)
|
||||
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
assert remote_parser.get_text() == "Local digital text."
|
||||
|
||||
def test_pdf_no_archive_requested_closes_no_client(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
) -> None:
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
azure_client.close.assert_not_called()
|
||||
|
||||
def test_non_pdf_still_calls_azure_when_no_archive_requested(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
) -> None:
|
||||
"""
|
||||
Images have no local-text fallback, so produce_archive=False does
|
||||
not skip the remote engine for non-PDF MIME types.
|
||||
"""
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"image/png",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
azure_client.begin_analyze_document.assert_called_once()
|
||||
assert remote_parser.get_text() == _DEFAULT_TEXT
|
||||
|
||||
@pytest.mark.usefixtures("no_engine_settings")
|
||||
def test_unconfigured_engine_takes_precedence_over_skip(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
) -> None:
|
||||
"""An unconfigured engine still short-circuits before the
|
||||
produce_archive check, returning empty text as before.
|
||||
"""
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
assert remote_parser.get_text() == ""
|
||||
assert remote_parser.get_archive_path() is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse() — Azure failure path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -21,7 +21,7 @@ from documents.parsers import run_convert
|
||||
from paperless.models import ModeChoices
|
||||
from paperless.parsers import ParserProtocol
|
||||
from paperless.parsers.tesseract import RasterisedDocumentParser
|
||||
from paperless.parsers.utils import post_process_text
|
||||
from paperless.parsers.utils import is_tagged_pdf
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@@ -151,36 +151,6 @@ class TestRasterisedDocumentParserLifecycle:
|
||||
assert tempdir is not None and not tempdir.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# post_process_text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPostProcessText:
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"simple string",
|
||||
"simple string",
|
||||
id="collapse-spaces",
|
||||
),
|
||||
pytest.param(
|
||||
"simple newline\n testing string",
|
||||
"simple newline\ntesting string",
|
||||
id="preserve-newline",
|
||||
),
|
||||
pytest.param(
|
||||
"utf-8 строка с пробелами в конце ", # noqa: RUF001
|
||||
"utf-8 строка с пробелами в конце", # noqa: RUF001
|
||||
id="utf8-trailing-spaces",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_post_process_text(self, source: str, expected: str) -> None:
|
||||
assert post_process_text(source) == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page count
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -910,25 +880,25 @@ class TestSkipArchive:
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tesseract_parser: RasterisedDocumentParser,
|
||||
tesseract_samples_dir: Path,
|
||||
tagged_no_text_pdf_file: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF that reports itself as tagged (/MarkInfo /Marked true) but
|
||||
has no actual extractable text (some scanner firmware produces
|
||||
this — see GitHub issue #13349)
|
||||
- A real PDF that reports itself as tagged (/MarkInfo /Marked
|
||||
true) but whose only pdftotext output is layout padding (a
|
||||
lone form-feed byte), not real text (see GitHub issue #13387,
|
||||
originally reported against #13349's tagged-PDF handling)
|
||||
- Mode: auto, produce_archive=False
|
||||
WHEN:
|
||||
- Document is parsed
|
||||
THEN:
|
||||
- The tag alone is not trusted as "has text"; OCRmyPDF still runs
|
||||
"""
|
||||
assert is_tagged_pdf(tagged_no_text_pdf_file) is True
|
||||
tesseract_parser.settings.mode = ModeChoices.AUTO
|
||||
mocker.patch("paperless.parsers.tesseract.is_tagged_pdf", return_value=True)
|
||||
mocker.patch.object(tesseract_parser, "extract_text", return_value=None)
|
||||
mock_ocr = mocker.patch("ocrmypdf.ocr")
|
||||
tesseract_parser.parse(
|
||||
tesseract_samples_dir / "multi-page-images.pdf",
|
||||
tagged_no_text_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
Binary file not shown.
@@ -4,10 +4,18 @@ from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from paperless.parsers.utils import is_tagged_pdf
|
||||
from paperless.parsers.utils import pdf_born_digital_text
|
||||
from paperless.parsers.utils import post_process_text
|
||||
from paperless.parsers.utils import read_file_handle_unicode_errors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
SAMPLES = Path(__file__).parent / "samples" / "tesseract"
|
||||
|
||||
|
||||
@@ -60,3 +68,105 @@ class TestIsTaggedPdf:
|
||||
bad = tmp_path / "bad.pdf"
|
||||
bad.write_bytes(b"not a pdf")
|
||||
assert is_tagged_pdf(bad) is False
|
||||
|
||||
|
||||
class TestPostProcessText:
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"simple string",
|
||||
"simple string",
|
||||
id="collapse-spaces",
|
||||
),
|
||||
pytest.param(
|
||||
"simple newline\n testing string",
|
||||
"simple newline\ntesting string",
|
||||
id="preserve-newline",
|
||||
),
|
||||
pytest.param(
|
||||
"utf-8 строка с пробелами в конце ", # noqa: RUF001
|
||||
"utf-8 строка с пробелами в конце", # noqa: RUF001
|
||||
id="utf8-trailing-spaces",
|
||||
),
|
||||
pytest.param(None, None, id="none-input"),
|
||||
pytest.param("", None, id="empty-string"),
|
||||
pytest.param(" \n\x0c \n ", None, id="whitespace-and-formfeed-only"),
|
||||
],
|
||||
)
|
||||
def test_post_process_text(
|
||||
self,
|
||||
source: str | None,
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
assert post_process_text(source) == expected
|
||||
|
||||
|
||||
class TestPdfBornDigitalText:
|
||||
"""Regression coverage for GH #13387.
|
||||
|
||||
should_produce_archive() and RasterisedDocumentParser.parse() must agree
|
||||
on whether a PDF has real text, so both go through this one function.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extracted", "tagged", "expected_text", "expected_born_digital"),
|
||||
[
|
||||
pytest.param("tiny", True, "tiny", True, id="tagged-with-real-text"),
|
||||
pytest.param("tiny", False, "tiny", False, id="untagged-below-min-length"),
|
||||
pytest.param(
|
||||
"x" * 51,
|
||||
False,
|
||||
"x" * 51,
|
||||
True,
|
||||
id="untagged-above-min-length",
|
||||
),
|
||||
pytest.param(None, True, None, False, id="tagged-but-no-text"),
|
||||
],
|
||||
)
|
||||
def test_born_digital_decision(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
extracted: str | None,
|
||||
tagged: bool, # noqa: FBT001
|
||||
expected_text: str | None,
|
||||
expected_born_digital: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF whose pdftotext output and /MarkInfo tag status vary
|
||||
WHEN:
|
||||
- pdf_born_digital_text() is called
|
||||
THEN:
|
||||
- The normalized text and born-digital verdict match; the tag
|
||||
alone never counts as "has text"
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.utils.extract_pdf_text",
|
||||
return_value=extracted,
|
||||
)
|
||||
mocker.patch("paperless.parsers.utils.is_tagged_pdf", return_value=tagged)
|
||||
text, born_digital = pdf_born_digital_text(tmp_path / "doc.pdf")
|
||||
assert text == expected_text
|
||||
assert born_digital is expected_born_digital
|
||||
|
||||
def test_tagged_but_textless_pdf_is_not_born_digital(
|
||||
self,
|
||||
tagged_no_text_pdf_file: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A real PDF that is tagged (/MarkInfo /Marked true) but whose
|
||||
only "text" is layout padding (a stray form-feed byte)
|
||||
WHEN:
|
||||
- pdf_born_digital_text() is called with no mocking
|
||||
THEN:
|
||||
- The normalized text is None and the PDF is not treated as
|
||||
born-digital. The raw, unnormalized pdftotext output is
|
||||
non-empty for this file, which is exactly what caused the
|
||||
archive decision to disagree with the OCR decision in #13387.
|
||||
"""
|
||||
text, born_digital = pdf_born_digital_text(tagged_no_text_pdf_file)
|
||||
assert text is None
|
||||
assert born_digital is False
|
||||
|
||||
Reference in New Issue
Block a user