Merge branch 'dev' into feature/ocr-templates

This commit is contained in:
shamoon committed 2026-10-19 21:37:23 -07:00
commit cf6d16edcd
516 files changed
+137596 -100899

No files matched your search

+24 -137
View File
@@ -1,88 +1,41 @@
import shutil
import zoneinfo
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
import filelock
import pytest
from django.contrib.auth import get_user_model
from django.contrib.contenttypes.models import ContentType
from guardian.shortcuts import clear_ct_cache
from pytest_django.fixtures import Settings
from rest_framework.test import APIClient
from documents.tests.factories import DocumentFactory
UserModelT = get_user_model()
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from documents.models import Document
@dataclass(frozen=True, slots=True)
class PaperlessDirs:
"""Standard Paperless-ngx directory layout for tests."""
media: Path
originals: Path
archive: Path
thumbnails: Path
from paperless_testing.dirs import PaperlessDirs
@pytest.fixture(scope="session")
def samples_dir() -> Path:
def document_samples_dir() -> Path:
"""Path to the shared test sample documents."""
return Path(__file__).parent / "samples" / "documents"
@pytest.fixture()
def paperless_dirs(tmp_path: Path) -> PaperlessDirs:
"""Create and return the directory structure for testing."""
media = tmp_path / "media"
dirs = PaperlessDirs(
media=media,
originals=media / "documents" / "originals",
archive=media / "documents" / "archive",
thumbnails=media / "documents" / "thumbnails",
)
for d in (dirs.originals, dirs.archive, dirs.thumbnails):
d.mkdir(parents=True)
return dirs
@pytest.fixture()
def _media_settings(paperless_dirs: PaperlessDirs, settings) -> None:
"""Configure Django settings to point at temp directories."""
settings.MEDIA_ROOT = paperless_dirs.media
settings.ORIGINALS_DIR = paperless_dirs.originals
settings.ARCHIVE_DIR = paperless_dirs.archive
settings.THUMBNAIL_DIR = paperless_dirs.thumbnails
settings.MEDIA_LOCK = paperless_dirs.media / "media.lock"
settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"}
settings.APP_LOGO = ""
@pytest.fixture()
def sample_doc(
paperless_dirs: PaperlessDirs,
_media_settings: None,
samples_dir: Path,
paperless_dirs: "PaperlessDirs",
document_samples_dir: Path,
) -> "Document":
"""Create a document with valid files and matching checksums."""
with filelock.FileLock(paperless_dirs.media / "media.lock"):
with filelock.FileLock(paperless_dirs.media_lock):
shutil.copy(
samples_dir / "originals" / "0000001.pdf",
paperless_dirs.originals / "0000001.pdf",
document_samples_dir / "originals" / "0000001.pdf",
paperless_dirs.originals_dir / "0000001.pdf",
)
shutil.copy(
samples_dir / "archive" / "0000001.pdf",
paperless_dirs.archive / "0000001.pdf",
document_samples_dir / "archive" / "0000001.pdf",
paperless_dirs.archive_dir / "0000001.pdf",
)
shutil.copy(
samples_dir / "thumbnails" / "0000001.webp",
paperless_dirs.thumbnails / "0000001.webp",
document_samples_dir / "thumbnails" / "0000001.webp",
paperless_dirs.thumbnail_dir / "0000001.webp",
)
return DocumentFactory(
@@ -97,88 +50,22 @@ def sample_doc(
)
@pytest.fixture()
def _search_index(
tmp_path: Path,
settings: Settings,
) -> Generator[None, None, None]:
"""Create a temp index directory and point INDEX_DIR at it.
@pytest.fixture
def _search_index(paperless_dirs: "PaperlessDirs") -> None:
"""Point the search backend at a fresh, empty index directory.
Resets the backend singleton before and after so each test gets a clean
index rather than reusing a stale singleton from another test.
paperless_dirs owns INDEX_DIR and resets the backend singleton on both
sides of the test, so requesting it is all that is needed.
"""
from documents.search import reset_backend
index_dir = tmp_path / "index"
index_dir.mkdir()
settings.INDEX_DIR = index_dir
reset_backend()
yield
reset_backend()
@pytest.fixture()
def settings_timezone(settings: Settings) -> zoneinfo.ZoneInfo:
return zoneinfo.ZoneInfo(settings.TIME_ZONE)
@pytest.fixture
def rest_api_client():
def searchable_document(_search_index: None) -> "Document":
"""One searchable document, for tests about what the search endpoint
returns rather than about what it finds.
"""
The basic DRF ApiClient
"""
yield APIClient()
from documents.search import get_backend
@pytest.fixture()
def regular_user(django_user_model: type[UserModelT]) -> UserModelT:
"""Unprivileged authenticated user for permission boundary tests."""
return django_user_model.objects.create_user(username="regular", password="regular")
@pytest.fixture()
def admin_client(rest_api_client: APIClient, admin_user: UserModelT) -> APIClient:
"""Admin client pre-authenticated and sending the v10 Accept header."""
rest_api_client.force_authenticate(user=admin_user)
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
return rest_api_client
@pytest.fixture()
def v9_client(rest_api_client: APIClient, admin_user: UserModelT) -> APIClient:
"""Admin client pre-authenticated and sending the v9 Accept header."""
rest_api_client.force_authenticate(user=admin_user)
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=9")
return rest_api_client
@pytest.fixture()
def user_client(rest_api_client: APIClient, regular_user: UserModelT) -> APIClient:
"""Regular-user client pre-authenticated and sending the v10 Accept header."""
rest_api_client.force_authenticate(user=regular_user)
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
return rest_api_client
@pytest.fixture(autouse=True)
def _clear_content_type_caches() -> None:
"""Clear Django's ContentType cache and guardian's lru_cache before each test.
Tests that delete and reinsert ContentType/Permission rows (e.g. the
importer) corrupt both caches. Without this fixture a subsequent test on
the same xdist worker sees stale ContentType objects and guardian raises
MixedContentTypeError.
"""
ContentType.objects.clear_cache()
clear_ct_cache()
@pytest.fixture(scope="session", autouse=True)
def faker_session_locale():
"""Set Faker locale for reproducibility."""
return "en_US"
@pytest.fixture(scope="session", autouse=True)
def faker_seed():
return 12345
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
-98
View File
@@ -1,98 +0,0 @@
"""
Factory-boy factories for documents app models.
"""
from __future__ import annotations
import factory
from django.contrib.auth import get_user_model
from factory.django import DjangoModelFactory
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import PaperlessTask
from documents.models import StoragePath
from documents.models import Tag
UserModelT = get_user_model()
class CorrespondentFactory(DjangoModelFactory[Correspondent]):
class Meta:
model = Correspondent
name = factory.Sequence(lambda n: f"{factory.Faker('company')} {n}")
match = ""
matching_algorithm = MatchingModel.MATCH_NONE
class DocumentTypeFactory(DjangoModelFactory[DocumentType]):
class Meta:
model = DocumentType
name = factory.Sequence(lambda n: f"{factory.Faker('bs')} {n}")
match = ""
matching_algorithm = MatchingModel.MATCH_NONE
class TagFactory(DjangoModelFactory[Tag]):
class Meta:
model = Tag
name = factory.Sequence(lambda n: f"{factory.Faker('word')} {n}")
match = ""
matching_algorithm = MatchingModel.MATCH_NONE
is_inbox_tag = False
class StoragePathFactory(DjangoModelFactory[StoragePath]):
class Meta:
model = StoragePath
name = factory.Sequence(
lambda n: f"{factory.Faker('file_path', depth=2, extension='')} {n}",
)
path = factory.LazyAttribute(lambda o: f"{o.name}/{{title}}")
match = ""
matching_algorithm = MatchingModel.MATCH_NONE
class DocumentFactory(DjangoModelFactory[Document]):
class Meta:
model = Document
title = factory.Faker("sentence", nb_words=4)
checksum = factory.Faker("sha256")
content = factory.Faker("paragraph")
correspondent = None
document_type = None
storage_path = None
class UserFactory(DjangoModelFactory[UserModelT]):
class Meta:
model = UserModelT
username = factory.Sequence(lambda n: f"user{n}")
is_staff = False
is_superuser = False
password = factory.django.Password("test")
class Params:
superuser = factory.Trait(is_staff=True, is_superuser=True)
staff = factory.Trait(is_staff=True)
class PaperlessTaskFactory(DjangoModelFactory[PaperlessTask]):
class Meta:
model = PaperlessTask
task_id = factory.Faker("uuid4")
task_type = PaperlessTask.TaskType.CONSUME_FILE
trigger_source = PaperlessTask.TriggerSource.WEB_UI
status = PaperlessTask.Status.PENDING
input_data = factory.LazyFunction(dict)
result_data = None
acknowledged = False
+10
View File
@@ -0,0 +1,10 @@
import re
def dummy_preprocess(content: str) -> str:
"""
Simpler, faster pre-processing for testing purposes
"""
content = content.lower().strip()
content = re.sub(r"\s+", " ", content)
return content
@@ -15,11 +15,11 @@ from rich.console import Console
from documents.management.commands.document_sanity_checker import Command
from documents.sanity_checker import SanityCheckMessages
from documents.tests.factories import DocumentFactory
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from documents.models import Document
from documents.tests.conftest import PaperlessDirs
from paperless_testing.dirs import PaperlessDirs
def _render_to_string(messages: SanityCheckMessages) -> str:
@@ -71,7 +71,7 @@ class TestRenderResultsWithIssues:
assert "INFO" in output
assert "No OCR data" in output
@pytest.mark.usefixtures("_media_settings")
@pytest.mark.usefixtures("paperless_dirs")
def test_global_message(self) -> None:
msgs = SanityCheckMessages()
msgs.warning(None, "Orphaned file: /tmp/stray.pdf")
@@ -87,7 +87,7 @@ class TestRenderResultsWithIssues:
assert "Thumbnail missing" in output
assert "Checksum mismatch" in output
@pytest.mark.usefixtures("_media_settings")
@pytest.mark.usefixtures("paperless_dirs")
def test_unknown_doc_pk(self) -> None:
msgs = SanityCheckMessages()
msgs.error(99999, "Ghost document")
@@ -184,7 +184,6 @@ class TestDocumentSanityCheckerCommand:
assert "ERROR" in output
assert "Original of document does not exist" in output
@pytest.mark.usefixtures("_media_settings")
def test_checksum_mismatch(self, paperless_dirs: PaperlessDirs) -> None:
"""Lightweight document with zero-byte files triggers checksum mismatch."""
doc = DocumentFactory(
@@ -1 +1 @@
sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحب ا كيف حالك hebrew שלום מה שלומך emoji symbol µ math ₀ x² dx π 3 14159 e ρ ε₀ currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحبًا كيف حالك hebrew שלום מה שלומך emoji ️ symbol µ math x dx π 3 14159 e ρ ε currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
@@ -0,0 +1,30 @@
"""Field references and leaf constructors shared by the unit-level tests.
These are pure whoosh-compat AST values with no state and no database
behind them, so they stay plain module-level functions rather than
fixtures. ``NOTES`` is the reason this module exists at all: notes is a
JSON field, so addressing its text means naming a subpath, and spelling
``wc.FieldRef("notes", "note")`` out once per test file invites two of
them to disagree.
"""
from __future__ import annotations
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
CONTENT = wc.FieldRef("content")
TITLE = wc.FieldRef("title")
NOTES = wc.FieldRef("notes", "note")
BIGRAM_CONTENT = wc.FieldRef("bigram_content")
BIGRAM_TITLE = wc.FieldRef("bigram_title")
def content(text: str) -> wc_ast.Term:
"""A Term on the content field, the default leaf these tests widen."""
return wc_ast.Term(field=CONTENT, text=text)
def bigram(text: str) -> wc_ast.Term:
"""A Term on the bigram side of content."""
return wc_ast.Term(field=BIGRAM_CONTENT, text=text)
+59 -15
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import tempfile
from typing import TYPE_CHECKING
import pytest
@@ -10,25 +9,21 @@ from documents.search._backend import TantivyBackend
from documents.search._backend import reset_backend
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Generator
from pathlib import Path
from pytest_django.fixtures import Settings
@pytest.fixture
def index_dir(tmp_path: Path, settings: Settings) -> Path:
path = tmp_path / "index"
path.mkdir()
settings.INDEX_DIR = path
return path
from documents.models import Document
from paperless_testing.dirs import PaperlessDirs
@pytest.fixture
def backend() -> Generator[TantivyBackend, None, None]:
b = TantivyBackend() # path=None → in-memory index
def backend(paperless_dirs: PaperlessDirs) -> Generator[TantivyBackend, None, None]:
b = TantivyBackend(path=paperless_dirs.index_dir)
b.open()
try:
yield b
@@ -37,9 +32,58 @@ def backend() -> Generator[TantivyBackend, None, None]:
reset_backend()
@pytest.fixture
def long_cjk_run() -> str:
"""A CJK run too long for the content analyzer to keep.
48 characters, 144 UTF-8 bytes: one token longer than remove_long's
129-byte limit, so the content side analyzes to nothing and only the
bigram field can match it.
"""
return "東京都の公共文書について" * 4
@pytest.fixture
def index_document(backend: TantivyBackend) -> Callable[..., Document]:
"""Build a Document with DocumentFactory and add it to the index.
The factory supplies a unique checksum, so a test only passes the
fields its assertion actually depends on.
"""
def _index(**kwargs: object) -> Document:
doc = DocumentFactory(**kwargs)
backend.add_or_update(doc)
return doc
return _index
@pytest.fixture
def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]:
"""Run a query as no particular user and return the matching pks."""
def _matched_ids(query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
return _matched_ids
@pytest.fixture(scope="module")
def index() -> tantivy.Index:
"""A real Tantivy index for parse-acceptance tests (module scope for speed)."""
idx = tantivy.Index(build_schema(), path=tempfile.mkdtemp())
register_tokenizers(idx, "english")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index for the parse-only tests.
These never index a document, so one index per module is shared
read-only across that module's tests.
"""
idx = tantivy.Index(build_schema(), path=None)
register_tokenizers(idx, "")
return idx
@pytest.fixture
def fuzzy_enabled(settings: Settings) -> None:
"""Enable the fuzzy blend clause. The threshold doubles as a minimum
score filter, so it is set to 0.0: every hit passes and the test sees
the clause's matching behaviour, not the filter's."""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
@@ -0,0 +1,508 @@
"""Result-level acceptance corpus: real documents indexed via build_schema(),
real queries run through parse_user_query(), matched-document-ID sets
asserted, not intermediate ASTs or query strings. This is paperless-ngx's
analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
Supersedes test_query.py's TestParseUserQuery result-level cases.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import DocumentType
from documents.models import Note
from documents.models import StoragePath
from documents.search._query import parse_user_query
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
@pytest.fixture
def indexed_documents(index_document: Callable[..., Document]) -> dict[str, int]:
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
docs = {
"invoice_2020": index_document(
title="Invoice 2020",
content="invoice total due",
archive_serial_number=100,
),
"invoice_2021": index_document(
title="Invoice 2021",
content="invoice total due",
archive_serial_number=101,
),
"invoice_2023": index_document(
title="Invoice 2023",
content="invoice total due",
archive_serial_number=102,
),
"receipt_2022": index_document(
title="Receipt 2022",
content="receipt total due",
archive_serial_number=103,
),
}
return {label: doc.pk for label, doc in docs.items()}
class TestIssue13568BracketWildcard:
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
not fold to a prefix query that silently drops it."""
def test_bracket_class_wildcard_matches_only_in_range_years(
self,
matched_ids: Callable[[str], set[int]],
indexed_documents: dict[str, int],
) -> None:
"""
GIVEN:
- Four indexed documents titled Invoice 2020/2021/2023 and
Receipt 2022
WHEN:
- "title:202[0-1]*" is searched ([0-1], not [0-3], is
deliberate: the fixture's trailing digits are 0/1/2/3, so a
[0-3] class would match all four and pass even if the
character class were silently dropped and folded to an
unconstrained "202*" prefix; [0-1] partitions the fixture
into a genuine in-range/out-of-range split)
THEN:
- Only the 2020 and 2021 documents match, proving the bracket
character class survived (issue #13568's original bug)
"""
matched = matched_ids("title:202[0-1]*")
expected = {
indexed_documents["invoice_2020"],
indexed_documents["invoice_2021"],
}
assert matched == expected, (
"title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
"- if this matches everything, the wildcard's character class was "
"silently dropped (issue #13568's original bug)"
)
class TestFieldBoosts:
def test_title_boost_ranks_title_match_above_content_only_match(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
) -> None:
"""
GIVEN:
- One document whose title contains the query word and another
whose content (not title) contains it
WHEN:
- The query word is searched unfielded
THEN:
- The title match ranks first, proving our title field boost
actually affects ranking
"""
title_match = index_document(
title="urgent",
content="nothing else relevant",
)
index_document(
title="nothing",
content="urgent matter here",
)
query = parse_user_query(backend._index, "urgent", UTC)
searcher = backend._index.searcher()
results = searcher.search(query, limit=10)
ranked_ids = [
searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
]
assert ranked_ids[0] == title_match.pk
class TestJsonSubpaths:
def test_notes_user_matches_document_with_that_note_author(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a Note authored by "alice" and a second,
unrelated document with no note
WHEN:
- "notes.user:alice" is searched
THEN:
- Only the document with alice's note matches
"""
alice = UserFactory(username="alice")
doc_with_note = DocumentFactory(
title="Has note",
content="x",
)
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
backend.add_or_update(doc_with_note)
index_document(title="No note", content="x")
matched = matched_ids("notes.user:alice")
assert matched == {doc_with_note.pk}
def test_custom_fields_name_and_value_combine(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a "Contract Number" custom field valued
"policy", and a second document with a differently-named
custom field also valued "policy"
WHEN:
- 'custom_fields.name:"Contract Number" custom_fields.value:policy'
is searched
THEN:
- Only the document whose field name AND value both match is
returned
"""
field = CustomField.objects.create(
name="Contract Number",
data_type=CustomField.FieldDataType.STRING,
)
other_field = CustomField.objects.create(
name="Other Field",
data_type=CustomField.FieldDataType.STRING,
)
matching = DocumentFactory(
title="Matching",
content="x",
)
CustomFieldInstance.objects.create(
document=matching,
field=field,
value_text="policy",
)
backend.add_or_update(matching)
non_matching = DocumentFactory(
title="Non-matching",
content="x",
)
CustomFieldInstance.objects.create(
document=non_matching,
field=other_field,
value_text="policy",
)
backend.add_or_update(non_matching)
matched = matched_ids(
'custom_fields.name:"Contract Number" custom_fields.value:policy',
)
assert matched == {matching.pk}
class TestUnregisteredIdFieldFoldsToLiteralText:
"""tag_id, owner_id, etc. are intentionally excluded from the
FieldRegistry - always internal index columns, never meant to be
query-addressable. Prove an unregistered field folds to a literal
text search that matches nothing, rather than erroring."""
def test_tag_id_query_matches_nothing(
self,
matched_ids: Callable[[str], set[int]],
indexed_documents: dict[str, int],
) -> None:
"""
GIVEN:
- A real indexed corpus and "tag_id", a field intentionally
excluded from the FieldRegistry (an internal index column,
never meant to be query-addressable)
WHEN:
- "tag_id:5" is searched
THEN:
- It folds to a literal text search and matches nothing,
rather than erroring
"""
matched = matched_ids("tag_id:5")
assert matched == set()
class TestFuzzyBlendSurvivesWhooshGrammar:
"""A query mixing whoosh-only grammar (a date keyword) with a typo'd
free-text word must still fuzzy-match the intended document when
ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. Fuzzy widening happens
inside the already-parsed AST (_widen_leaf, via emit()), never by
re-parsing the raw query string, so whoosh grammar that tantivy's own
parser rejects cannot knock the fuzzy side out."""
def test_typo_fuzzy_matches_alongside_date_keyword(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings,
) -> None:
"""
GIVEN:
- ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
indexed with content "receipt total due"
WHEN:
- The query blends whoosh-only grammar tantivy's own parser
rejects ("added:today") with a one-transposition misspelling
of a word in the indexed content
THEN:
- The document still matches, because fuzzy widening happens
inside the already-parsed AST, never by re-parsing the raw
query string, so grammar tantivy's parser cannot handle
cannot knock the fuzzy side out
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
with time_machine.travel(FROZEN_NOW, tick=False):
doc = index_document(
title="Receipt March",
content="receipt total due",
archive_serial_number=900,
)
# Sanity: the exact spelling matches through the exact clause.
assert doc.pk in matched_ids("added:today receipt")
# The regression: the misspelling (one transposition) only
# matches via the fuzzy clause, and "added:today" is
# whoosh-only grammar tantivy's parser rejects, so raw-string
# fuzzy parsing skips the clause entirely and this returns
# nothing. The typo is deliberate; keep codespell away from it.
typo_query = "added:today reciept" # codespell:ignore reciept
assert doc.pk in matched_ids(typo_query)
def test_negated_words_do_not_fuzzy_match(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings,
) -> None:
"""
GIVEN:
- ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
containing the NOT'd word ("receipt") but not the positive
word ("total"), so nothing matches the exact clause -- the
shape a naive fuzzy string built from ALL words (including
the NOT'd one) would make this document the sole hit,
normalize its score to 1.0, and survive any threshold (a
shape with an exact-matching sibling document would NOT
discriminate: normalization would rank the resurfaced
document far below the exact match and the threshold would
cut it even for a naive implementation)
WHEN:
- "added:today total NOT receipt" is searched
THEN:
- The document does not match; a term the user excluded must
not resurface through the fuzzy clause
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
with time_machine.travel(FROZEN_NOW, tick=False):
index_document(
title="Receipt Archive",
content="receipt archived stack",
archive_serial_number=901,
)
assert matched_ids("added:today total NOT receipt") == set()
class TestUnquotedDateKeywordPhrases:
"""The unquoted spelling (added:previous month) is honored natively by
whoosh-compat's own grammar for this closed phrase vocabulary, no
app-level rewrite is involved. Pins that the historically supported
spelling keeps working now that paperless no longer pre-quotes it."""
@pytest.fixture
def period_documents(
self,
index_document: Callable[..., Document],
) -> dict[str, int]:
with time_machine.travel(FROZEN_NOW, tick=False):
in_may = index_document(
title="May Doc",
content="statement",
archive_serial_number=910,
added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
)
in_june = index_document(
title="June Doc",
content="statement",
archive_serial_number=911,
added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
)
return {"in_may": in_may.pk, "in_june": in_june.pk}
@pytest.mark.parametrize(
"query",
[
pytest.param("added:previous month", id="unquoted"),
pytest.param('added:"previous month"', id="quoted"),
pytest.param("added:Previous Month", id="unquoted-mixed-case"),
],
)
def test_unquoted_matches_the_same_documents_as_quoted(
self,
matched_ids: Callable[[str], set[int]],
period_documents: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- Two documents added in different months, time frozen so
only one falls in "previous month"
WHEN:
- The same date-keyword phrase is spelled unquoted, quoted,
and unquoted with mixed case
THEN:
- All three spellings match the same document; paperless no
longer pre-quotes this phrase before parsing, relying on
whoosh-compat's own grammar to accept it unquoted natively
"""
with time_machine.travel(FROZEN_NOW, tick=False):
assert matched_ids(query) == {period_documents["in_may"]}
@pytest.mark.parametrize(
"query",
[
pytest.param("added:this month", id="this-month"),
pytest.param("added:this year", id="this-year"),
pytest.param("added:previous week", id="previous-week"),
pytest.param("added:previous quarter", id="previous-quarter"),
pytest.param("added:previous year", id="previous-year"),
pytest.param("created:previous month", id="created-field"),
pytest.param("modified:previous month", id="modified-field"),
],
)
def test_every_phrase_and_date_field_parses_without_error(
self,
matched_ids: Callable[[str], set[int]],
period_documents: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- Our real schema and every date-keyword phrase in the
vocabulary, against every date field we expose (added,
created, modified)
WHEN:
- Each combination is searched
THEN:
- It parses and searches cleanly against our schema (no
SearchQueryError, so no HTTP 400); exact window semantics
are whoosh-compat's own and are pinned in its own suite
"""
with time_machine.travel(FROZEN_NOW, tick=False):
matched_ids(query)
def test_text_field_keyword_words_are_ordinary_text(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
period_documents: dict[str, int],
) -> None:
"""
GIVEN:
- period_documents (indexed by added-date) and a third
document whose title literally contains the words
"previous month"
WHEN:
- "title:previous month" is searched
THEN:
- Only the document whose title contains those words matches;
"previous month" after a TEXT field (or unfielded) is
ordinary text, not a date phrase, so the date-window
documents do not match
"""
with time_machine.travel(FROZEN_NOW, tick=False):
wordy = index_document(
title="Notes from the previous month",
content="meeting notes",
archive_serial_number=912,
)
assert matched_ids("title:previous month") == {wordy.pk}
class TestFieldAliases:
"""type:/path: are registry aliases for document_type:/storage_path:.
The only other alias coverage is parse-shape; these prove resolution
end-to-end against a real index."""
def test_type_alias_and_canonical_name_match_the_same_document(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with document_type "invoice", and a decoy
document with no type whose content merely mentions
"invoice" (document_type is itself a default search field,
so if alias resolution ever broke and "type:invoice"
demoted to unfielded text, the token would STILL match the
typed document through the field value; the decoy carrying
the query word in content is what makes a demoted search
distinguishable, since it would then match both documents
and fail the exact-set assertion -- the title avoids
stemming to "type": English stems Typed -> type)
WHEN:
- "type:invoice" and "document_type:invoice" are each
searched
THEN:
- Both resolve to the same document, proving the "type" alias
and its canonical field name agree end-to-end against a
real index
"""
invoice_type = DocumentType.objects.create(name="invoice")
typed = index_document(
title="First",
content="quarterly statement",
document_type=invoice_type,
)
index_document(
title="Second",
content="invoice mentioned in body",
)
assert matched_ids("type:invoice") == {typed.pk}
assert matched_ids("document_type:invoice") == {typed.pk}
def test_path_alias_and_canonical_name_match_the_same_document(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document stored under storage_path "archive", and a decoy
document with no storage_path whose content merely mentions
"archive" (storage_path is NOT a default search field
today, so a demoted "path:archive" already matches nothing;
the content decoy keeps this test discriminating even if
storage_path ever joins the defaults)
WHEN:
- "path:archive" and "storage_path:archive" are each searched
THEN:
- Both resolve to the same document, proving the "path" alias
and its canonical field name agree end-to-end against a
real index
"""
archive = StoragePath.objects.create(name="archive", path="archive/{title}")
stored = index_document(
title="Stored",
content="quarterly statement",
storage_path=archive,
)
index_document(
title="Loose",
content="archive mentioned in body",
)
assert matched_ids("path:archive") == {stored.pk}
assert matched_ids("storage_path:archive") == {stored.pk}
+43
View File
@@ -0,0 +1,43 @@
"""_any_of, the clause-list helper.
One test, for the empty-list case its callers never produce but which the
helper still has to answer for.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.search._query import _any_of
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestAnyOf:
def test_an_empty_clause_list_matches_nothing(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
) -> None:
"""
GIVEN:
- An indexed document, and no clauses at all
WHEN:
- _any_of is called with an empty list and the result is run
THEN:
- It matches no documents, rather than raising or matching
everything
"""
index_document(title="x", content="x")
results = backend._index.searcher().search(_any_of([]), limit=10)
assert len(results.hits) == 0
+244 -16
View File
@@ -1,12 +1,12 @@
import json
import logging
from datetime import date
from pathlib import Path
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.db import connection
from django.test.utils import CaptureQueriesContext
from guardian.shortcuts import assign_perm
from pytest_mock import MockerFixture
from documents.models import CustomField
@@ -18,12 +18,16 @@ from documents.search._backend import TantivyBackend
from documents.search._backend import WriteBatch
from documents.search._backend import get_backend
from documents.search._backend import reset_backend
from documents.search._schema import REBUILD_MARKER
from documents.search._schema import needs_rebuild
from documents.signals.handlers import add_to_index
from documents.tests.factories import CorrespondentFactory
from documents.tests.factories import DocumentFactory
from documents.tests.factories import DocumentTypeFactory
from documents.tests.factories import TagFactory
from documents.tests.factories import UserFactory
from paperless_testing.dirs import PaperlessDirs
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import DocumentTypeFactory
from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_object
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@@ -189,7 +193,7 @@ class TestAddOrUpdateIds:
pk=1,
owner=owner,
)
assign_perm("view_document", user, doc)
grant_object(user, doc, "view_document")
with backend.batch_update() as batch:
batch.add_or_update_ids([doc.pk])
@@ -209,7 +213,7 @@ class TestAddOrUpdateIds:
pk=1,
owner=owner,
)
assign_perm("view_document", group, doc)
grant_object(group, doc, "view_document")
with backend.batch_update() as batch:
batch.add_or_update_ids([doc.pk])
@@ -289,6 +293,80 @@ class TestAddOrUpdateIds:
assert backend.search_ids("updated", user=None) == [doc.pk]
class TestVersionsAreIndexedAsTheirRoot:
"""Only root documents are indexed, with their effective content, so
every write path that is handed a version indexes its root instead."""
@staticmethod
def _root_with_version() -> tuple[Document, Document]:
root = DocumentFactory(title="Statement", content="stale text")
version = DocumentFactory(
title="Statement",
content="latest text",
root_document=root,
version_index=1,
)
return root, version
def test_add_or_update_indexes_the_root_of_a_version(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A root document with a version
WHEN:
- The version is passed to add_or_update
THEN:
- The root is indexed with the version's text, and the version is not
"""
root, version = self._root_with_version()
backend.add_or_update(version)
assert backend.search_ids("latest", user=None) == [root.pk]
assert backend.search_ids("stale", user=None) == []
def test_add_or_update_ids_indexes_each_root_once(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A root document with a version
WHEN:
- Both ids are passed to add_or_update_ids
THEN:
- The root is indexed once and the version is not indexed
"""
root, version = self._root_with_version()
with backend.batch_update() as batch:
batch.add_or_update_ids([version.pk, root.pk])
assert backend.search_ids("Statement", user=None) == [root.pk]
assert backend.search_ids("latest", user=None) == [root.pk]
def test_add_or_update_ids_resolves_a_lone_version(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A root document with a version
WHEN:
- Only the version's id is passed to add_or_update_ids
THEN:
- The root is indexed
"""
root, version = self._root_with_version()
with backend.batch_update() as batch:
batch.add_or_update_ids([version.pk])
assert backend.search_ids("latest", user=None) == [root.pk]
class TestSearch:
"""Test search query parsing and matching via search_ids."""
@@ -763,8 +841,8 @@ class TestSearchIds:
def test_respects_permission_filter(self, backend: TantivyBackend) -> None:
"""search_ids must respect user permission filtering."""
owner = User.objects.create_user("ids_owner")
other = User.objects.create_user("ids_other")
owner = UserFactory(username="ids_owner")
other = UserFactory(username="ids_other")
doc = Document.objects.create(
title="private doc",
content="secret keyword",
@@ -824,6 +902,53 @@ class TestRebuild:
backend.rebuild(Document.objects.all(), iter_wrapper=wrapper)
assert 30 in seen
def test_successful_rebuild_leaves_index_up_to_date(
self,
backend: TantivyBackend,
paperless_dirs: PaperlessDirs,
) -> None:
"""
GIVEN:
- A backend and one document
WHEN:
- rebuild() completes
THEN:
- needs_rebuild() is False and no rebuild marker remains
"""
DocumentFactory.create()
backend.rebuild(Document.objects.all())
assert needs_rebuild(paperless_dirs.index_dir) is False
assert not (paperless_dirs.index_dir / REBUILD_MARKER).exists()
def test_interrupted_rebuild_is_retried(
self,
backend: TantivyBackend,
paperless_dirs: PaperlessDirs,
) -> None:
"""
GIVEN:
- A rebuild that dies while indexing documents (e.g. the database
connection is lost)
WHEN:
- needs_rebuild() is checked afterwards
THEN:
- It is True, even though the empty index was already stamped with
current settings, so the next start rebuilds instead of reporting
the index as up to date
"""
DocumentFactory.create()
def die(pairs):
raise RuntimeError("terminating connection due to administrator command")
yield # pragma: no cover
with pytest.raises(RuntimeError):
backend.rebuild(Document.objects.all(), iter_wrapper=die)
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_includes_group_granted_viewers(self, backend: TantivyBackend) -> None:
"""Rebuild must index viewer ids for group-only grants, not just direct ones.
@@ -843,7 +968,7 @@ class TestRebuild:
content="group secret keyword",
owner=owner,
)
assign_perm("view_document", group, doc)
grant_object(group, doc, "view_document")
backend.rebuild(Document.objects.all())
@@ -855,6 +980,79 @@ class TestRebuild:
assert ids == [doc.pk]
class TestCreatedDateOutOfRange:
"""The index stores dates as nanosecond i64 values (1677-09-22 to 2262-04-11).
A document whose created date falls outside that window must not abort
indexing: it is indexed without a created value and a warning names it.
"""
@pytest.mark.parametrize(
("created", "expected_warnings"),
[
pytest.param(date(1677, 9, 22), 0, id="first-representable-day"),
pytest.param(date(2262, 4, 11), 0, id="last-representable-day"),
pytest.param(date(1677, 9, 21), 1, id="day-before-first"),
pytest.param(date(2262, 4, 12), 1, id="day-after-last"),
pytest.param(date(16, 8, 30), 1, id="two-digit-year-read-as-year-16"),
pytest.param(date(9999, 12, 31), 1, id="max-python-date"),
],
)
def test_add_or_update_indexes_document_and_warns_when_out_of_range(
self,
backend: TantivyBackend,
caplog: pytest.LogCaptureFixture,
created: date,
expected_warnings: int,
) -> None:
"""
GIVEN:
- A document with a created date at or beyond the index date limits
WHEN:
- The document is added to the index
THEN:
- The document is indexed and searchable either way
- A warning naming the document is logged only for out-of-range dates
"""
doc = DocumentFactory(created=created, content="boundarycontent")
with caplog.at_level(logging.WARNING, logger="paperless.search"):
backend.add_or_update(doc)
assert backend.search_ids("boundarycontent", user=None) == [doc.pk]
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == expected_warnings
if expected_warnings:
assert f"Document {doc.pk}" in warnings[0].getMessage()
def test_rebuild_continues_past_out_of_range_document(
self,
backend: TantivyBackend,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A document with an unrepresentable created date among valid ones
WHEN:
- The index is rebuilt
THEN:
- Rebuild completes and every document is searchable
- A warning names the offending document
"""
good = DocumentFactory(created=date(2016, 8, 30), content="rebuildcontent")
bad = DocumentFactory(created=date(16, 8, 30), content="rebuildcontent")
with caplog.at_level(logging.WARNING, logger="paperless.search"):
backend.rebuild(Document.objects.all())
assert sorted(backend.search_ids("rebuildcontent", user=None)) == sorted(
[good.pk, bad.pk],
)
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == 1
assert f"Document {bad.pk}" in warnings[0].getMessage()
class TestAutocomplete:
"""Test autocomplete functionality."""
@@ -938,6 +1136,34 @@ class TestMoreLikeThis:
assert 150 not in ids
assert 151 in ids
def test_more_like_this_ids_seeded_by_version_uses_root(
self,
backend: TantivyBackend,
) -> None:
"""A version is not indexed, so it must be looked up as its root."""
root = DocumentFactory.create(
title="Important document",
content="financial information report",
)
version = DocumentFactory.create(
title="Important document",
content="financial information report",
root_document=root,
version_index=1,
)
other = DocumentFactory.create(
title="Another document",
content="financial information report",
)
backend.add_or_update(root)
backend.add_or_update(other)
ids = backend.more_like_this_ids(doc_id=version.pk, user=None)
assert other.pk in ids
assert root.pk not in ids
assert version.pk not in ids
class TestSingleton:
"""Test get_backend() and reset_backend() singleton lifecycle."""
@@ -948,7 +1174,8 @@ class TestSingleton:
yield
reset_backend()
def test_returns_same_instance_on_repeated_calls(self, index_dir) -> None:
@pytest.mark.usefixtures("paperless_dirs")
def test_returns_same_instance_on_repeated_calls(self) -> None:
"""Singleton pattern: repeated calls to get_backend() must return the same instance."""
assert get_backend() is get_backend()
@@ -965,7 +1192,8 @@ class TestSingleton:
assert b1 is not b2
assert b2._path == tmp_path / "b"
def test_reset_forces_new_instance(self, index_dir) -> None:
@pytest.mark.usefixtures("paperless_dirs")
def test_reset_forces_new_instance(self) -> None:
"""reset_backend() must force creation of a new backend instance on next get_backend() call."""
b1 = get_backend()
reset_backend()
@@ -1071,7 +1299,7 @@ class TestFieldHandling:
def test_notes_include_user_information(self, backend: TantivyBackend) -> None:
"""Notes must be indexed with user information when available for structured queries."""
user = User.objects.create_user("notewriter")
user = UserFactory(username="notewriter")
doc = Document.objects.create(
title="Doc with notes",
content="test",
@@ -1173,7 +1401,7 @@ class TestHighlightHits:
notes.note: prefix so the query targets notes content directly, but
the snippet is generated from notes_text which stores the same text.
"""
user = User.objects.create_user("hl_noteuser")
user = UserFactory(username="hl_noteuser")
doc = Document.objects.create(
title="Doc with matching note",
content="unrelated content",
@@ -0,0 +1,95 @@
"""Stored barcode contents in the search index.
Barcodes are a JSON field like notes and custom fields: barcodes: resolves to
barcodes.value:, and a plain query without the prefix does not look at them.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import DocumentBarcode
from paperless_testing.factories import DocumentBarcodeFactory
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestBarcodeSearch:
@pytest.fixture
def with_barcodes(self, backend: TantivyBackend) -> Document:
document = DocumentFactory(title="Letter", content="x")
DocumentBarcodeFactory(
document=document,
value="WIFI:T:WPA;S:Guest-WLAN;P:crocodile123;;",
)
DocumentBarcodeFactory(
document=document,
page=2,
value="DE89370400440532013000",
format=DocumentBarcode.Format.CODE128,
)
backend.add_or_update(document)
return document
def test_bare_barcodes_prefix_searches_values(
self,
with_barcodes: Document,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with stored barcodes, and a decoy document whose
content (not a barcode) contains the same word
WHEN:
- A bare "barcodes:" prefix query is run
THEN:
- Only the document whose barcode matches is returned, also for a
part of a barcode between separators
"""
index_document(title="Decoy", content="crocodile123 in the text")
assert matched_ids("barcodes:crocodile123") == {with_barcodes.pk}
assert matched_ids("barcodes.value:DE89370400440532013000") == {
with_barcodes.pk,
}
def test_barcodes_format_subpath(
self,
with_barcodes: Document,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a QR code and a Code 128 barcode
WHEN:
- The format subpath is queried
THEN:
- The document is found by its barcode formats
"""
assert matched_ids("barcodes.format:qrcode") == {with_barcodes.pk}
assert matched_ids("barcodes.format:aztec") == set()
def test_plain_query_ignores_barcodes(
self,
with_barcodes: Document,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a barcode value not found in its text
WHEN:
- The value is searched without a field prefix
THEN:
- Nothing is found, as with notes and custom fields
"""
assert matched_ids("crocodile123") == set()
@@ -0,0 +1,82 @@
"""``checksum`` wildcard patterns stay literal end to end, once user queries
route through whoosh-compat.
The registry-level fact (the pattern normalizer folds a KEYWORD pattern
rather than stemming it) is pinned on its own in
``test_keyword_pattern_literal.py``. This proves it actually reaches a real
query: ``checksum:ceded*`` must match only the document whose checksum
starts with "ceded", not the one whose checksum stems to the same run.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567"
CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567"
class TestChecksumPrefixQueries:
@pytest.fixture
def indexed(self, backend: TantivyBackend) -> None:
for i, checksum in enumerate((CEDEF00D, CEDEDEAD)):
doc = Document.objects.create(
title=f"Checksum doc {i}",
content="invoices for the quarter",
checksum=checksum,
archive_serial_number=940 + i,
)
backend.add_or_update(doc)
def _ids(self, backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def test_prefix_matches_only_the_document_that_starts_with_it(
self,
backend: TantivyBackend,
indexed: None,
) -> None:
"""
GIVEN:
- Two documents indexed with checksums that share a stem when
run through the English stemmer ("cedef00d..." and
"cededead...") but only one literally starts with "ceded"
WHEN:
- "checksum:ceded*" is searched
THEN:
- Only the document whose checksum literally starts with
"ceded" matches; the pattern normalizer folds a KEYWORD
pattern rather than stemming it, so this reaches a real
query end to end
"""
matched = self._ids(backend, "checksum:ceded*")
expected = Document.objects.get(checksum=CEDEDEAD).pk
assert matched == {expected}
def test_text_prefix_still_reaches_the_stemmed_index(
self,
backend: TantivyBackend,
indexed: None,
) -> None:
"""
GIVEN:
- Two documents indexed with content "invoices for the
quarter"
WHEN:
- "invoice*" is searched against the TEXT content field
THEN:
- Both documents match, confirming the checksum field's
literal-pattern behavior is specific to KEYWORD fields and
does not affect TEXT field wildcard matching against
stemmed terms
"""
assert len(self._ids(backend, "invoice*")) == 2
@@ -0,0 +1,193 @@
"""CJK bigram matching in QUERY-mode searches.
The bigram fields exist so CJK runs are matchable at all (the default
analyzers keep a whitespace-free CJK run as one indivisible token), but
matching them must not widen the query beyond what the user asked for: a
CJK term the query excludes, or restricts to one field, must not come back
through them.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestCjkParseFailureDegradesGracefully:
def test_a_cjk_run_tantivy_cannot_parse_drops_the_clause_only(self) -> None:
"""
GIVEN:
- A CJK run and an index-like object whose parse_query is
forced to raise
WHEN:
- _parse_cjk_text is called
THEN:
- It returns None instead of propagating, so a CJK run tantivy
cannot parse only drops the bigram clause rather than
failing the whole query. Broad on purpose (bare except
Exception), unlike the fuzzy blend's narrower ValueError
guard: a CJK run is not filtered to a guaranteed-safe token
set the way the fuzzy blend's word string is, so the exact
failure mode tantivy could raise here is not pinned down
"""
from documents.search._query import _parse_cjk_text
class _RaisingIndex:
def parse_query(self, *args: object, **kwargs: object) -> object:
raise RuntimeError("synthetic parse failure")
assert _parse_cjk_text(_RaisingIndex(), "東京", ["bigram_content"]) is None
def test_no_cjk_text_at_all_returns_none_without_parsing(self) -> None:
"""
GIVEN:
- A raw query string with no CJK characters at all
WHEN:
- _build_cjk_query (the simple TEXT/TITLE-mode builder) is
called directly
THEN:
- It returns None without ever attempting to parse anything.
The only real caller already guards this with _has_cjk(),
so this is defensive: it keeps the function safe to call on
its own, not a path a real search currently reaches
"""
from documents.search._query import _build_cjk_query
assert _build_cjk_query(None, "invoice total due", ["bigram_content"]) is None
class TestCjkClauseFollowsTheParsedQuery:
def test_negated_cjk_term_is_excluded(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two documents both matching "invoice", one whose content
also contains 漢字
WHEN:
- "invoice NOT 漢字" is searched
THEN:
- Only the document without 漢字 matches; 'invoice NOT 漢字'
must not return the document containing 漢字
"""
with_cjk = index_document(
title="Invoice A",
content="invoice total 漢字",
)
without_cjk = index_document(
title="Invoice B",
content="invoice total only",
)
assert matched_ids("invoice") == {with_cjk.pk, without_cjk.pk}
assert matched_ids("invoice NOT 漢字") == {without_cjk.pk}
@pytest.mark.parametrize(
("threshold", "expected"),
[
pytest.param(None, {"titled"}, id="fuzzy_off"),
pytest.param(0.0, {"titled"}, id="fuzzy_on"),
],
)
def test_fielded_cjk_term_searches_only_that_field(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings: SettingsWrapper,
threshold: float | None,
expected: set[str],
) -> None:
"""
GIVEN:
- One document with 東京 in its title, another with 東京 only
in its content, and ADVANCED_FUZZY_SEARCH_THRESHOLD either
off or on
WHEN:
- "title:東京" is searched
THEN:
- Only the titled document matches, whether fuzzy is on or
off: both the CJK alternative and the fuzzy alternative are
widened in place on the fielded leaf, so 'title:東京' still
must not match a document whose 東京 is only in the content
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
content_only = index_document(
title="Tokyo report",
content="東京都の人口は約1400万人です",
)
titled = index_document(
title="東京都の報告書",
content="an english summary",
)
pks = {"titled": titled.pk, "content_only": content_only.pk}
assert matched_ids("東京") == set(pks.values())
assert matched_ids("title:東京") == {pks[label] for label in expected}
def test_cjk_on_a_non_default_field_is_not_widened(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with 東京 in its content
WHEN:
- "notes:東京" is searched (a field outside the default
search fields)
THEN:
- Nothing matches; a CJK term restricted to a field outside
the default search fields has no bigram companion to widen
to, so it must not fall back to matching 東京 in the
content
"""
index_document(
title="Tokyo report",
content="東京都の人口は約1400万人です",
)
assert matched_ids("notes:東京") == set()
def test_bare_cjk_term_still_matches_every_default_field(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- One document with 重要 in its content, another with 重要 in
its title
WHEN:
- "重要" and "重要 OR report" are each searched unfielded
THEN:
- Both documents match either way; bigram matching exists
so that an unfielded CJK run matches wherever it is
indexed, and does so alongside a latin term
"""
in_content = index_document(
title="report",
content="本文に重要な情報",
)
in_title = index_document(
title="重要な報告書",
content="english only",
)
assert matched_ids("重要") == {in_content.pk, in_title.pk}
assert matched_ids("重要 OR report") == {
in_content.pk,
in_title.pk,
}
@@ -0,0 +1,120 @@
"""The emit-only FieldRegistry a CJK-widened query tree is emitted against.
It is the parse registry plus the internal bigram fields. These tests pin
what the bigram half has to get right on its own: combining a run's bigrams
with AND wherever the leaf sits, producing exactly the terms the index
holds, and naming only fields the index schema actually has.
"""
from __future__ import annotations
import pytest
import tantivy
import whoosh_compat.ast as wc_ast
from documents.search._query import _CJK_BIGRAM_FIELDS
from documents.search._query import _get_emit_field_registry
from documents.search._schema import field_descriptors
from documents.search._tokenizer import register_tokenizers
from documents.tests.search._ast_helpers import BIGRAM_CONTENT
from documents.tests.search._ast_helpers import CONTENT
pytestmark = pytest.mark.search
class TestEmitFieldRegistry:
def test_a_multi_bigram_run_requires_every_bigram_even_under_an_or(
self,
) -> None:
"""
GIVEN:
- A three-character CJK run on bigram_content, sitting inside
an Or (where the CJK widener always places it)
WHEN:
- The tree is analyzed against the emit registry
THEN:
- The run's two bigrams are combined with And, not Or. Left
to Multitoken.DEFAULT they would inherit the enclosing Or,
so "東京都" would match a document containing only 京都
"""
tree = wc_ast.Or(
children=(
wc_ast.Term(field=BIGRAM_CONTENT, text="東京都"),
wc_ast.Term(field=CONTENT, text="report"),
),
)
analyzed = wc_ast.analyze(tree, _get_emit_field_registry(None))
assert analyzed == wc_ast.Or(
children=(
wc_ast.And(
children=(
wc_ast.Term(field=BIGRAM_CONTENT, text="東京"),
wc_ast.Term(field=BIGRAM_CONTENT, text="京都"),
),
),
wc_ast.Term(field=CONTENT, text="report"),
),
)
@pytest.mark.parametrize(
"text",
[
pytest.param("東京都の公共文書", id="japanese"),
pytest.param("北京市人民政府", id="chinese"),
pytest.param("서울특별시", id="korean"),
],
)
def test_every_query_bigram_is_a_term_the_index_holds(self, text: str) -> None:
"""
GIVEN:
- A document whose bigram_content was indexed by the tokenizer
register_tokenizers() installs
WHEN:
- The emit registry's bigram analyzer tokenizes the same text
THEN:
- Every token it produces is found as an indexed term. The
query side and the index side each build their own bigram
analyzer, so this pins the two staying equivalent
"""
sb = tantivy.SchemaBuilder()
sb.add_text_field(
"bigram_content",
stored=False,
tokenizer_name="bigram_analyzer",
)
index = tantivy.Index(sb.build(), path=None)
register_tokenizers(index, None)
writer = index.writer()
doc = tantivy.Document()
doc.add_text("bigram_content", text)
writer.add_document(doc)
writer.commit()
index.reload()
resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)
assert resolved is not None
tokens = resolved.spec.analyzer(text)
assert tokens
searcher = index.searcher()
for token in tokens:
query = tantivy.Query.term_query(index.schema, "bigram_content", token)
assert searcher.search(query, limit=1).count == 1, token
def test_every_bigram_field_is_in_the_index_schema(self) -> None:
"""
GIVEN:
- The bigram fields the emit registry declares, and the index
schema's field descriptors
WHEN:
- Each bigram field name is looked up in the schema
THEN:
- Every one is present. Every index is built from these
descriptors, or rebuilt when its stored fingerprint of them
differs, so this is what guarantees each widened bigram leaf
names a field the index has
"""
schema_names = {descriptor.name for descriptor in field_descriptors()}
assert set(_CJK_BIGRAM_FIELDS.values()) <= schema_names
@@ -0,0 +1,498 @@
"""_widen_cjk_leaf, the rewrite_leaf hook that makes CJK runs matchable in
QUERY mode.
Unit tests against hand-built trees, no index needed: the hook on its own,
then the tree wc.analyze() builds around it. Result-level proof against a
real corpus lives in test_cjk_widening.py.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
from documents.search._query import _DEFAULT_SEARCH_FIELDS
from documents.search._query import _FIELD_BOOSTS
from documents.search._query import _get_emit_field_registry
from documents.search._query import _widen_leaf
from documents.search._registry import get_field_registry
from documents.tests.search._ast_helpers import BIGRAM_CONTENT
from documents.tests.search._ast_helpers import BIGRAM_TITLE
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import NOTES
from documents.tests.search._ast_helpers import TITLE
from documents.tests.search._ast_helpers import bigram
from documents.tests.search._ast_helpers import content
if TYPE_CHECKING:
from collections.abc import Callable
pytestmark = pytest.mark.search
def _widened(original: wc_ast.Node, bigram_side: wc_ast.Node) -> wc_ast.Or:
return wc_ast.Or(children=(original, bigram_side))
def _widen_cjk_leaf(leaf: wc_ast.Term | wc_ast.Phrase) -> wc_ast.Node:
"""The hook as the CJK work used it: no fuzzy side, nothing negated."""
return _widen_leaf(leaf, fuzzy=False, negated=frozenset())
def _analyze(tree: wc_ast.Node) -> wc_ast.Node:
return wc.analyze(
tree,
_get_emit_field_registry(None),
rewrite_leaf=_widen_cjk_leaf,
)
class TestTheHook:
@pytest.mark.parametrize(
"leaf",
[
pytest.param(content("invoice"), id="latin_term"),
pytest.param(
wc_ast.Term(field=NOTES, text="東京"),
id="non_default_field",
),
pytest.param(wc_ast.Term(field=None, text="東京"), id="unfielded"),
],
)
def test_leaf_is_returned_unchanged(self, leaf: wc_ast.Term) -> None:
"""
GIVEN:
- A leaf with no CJK text, a CJK term on a field outside the
default search fields, or an unfielded CJK term
WHEN:
- The hook is called with it
THEN:
- It returns the very same object, which tells analyze() to
keep the leaf's ordinary analysis
"""
assert _widen_cjk_leaf(leaf) is leaf
def test_a_leaf_the_tokenizer_drops_entirely_is_returned_unchanged(self) -> None:
"""
GIVEN:
- A term whose text matches _CJK_RE (U+2E80, a CJK Radicals
Supplement codepoint, so _has_cjk is True) but which the
content analyzer's simple tokenizer yields no token for at
all: a few hundred codepoints across the CJK blocks share
this gap between the regex and the tokenizer
WHEN:
- The hook is called with it
THEN:
- Neither cjk_terms nor latin_terms gains a piece, and the
leaf is returned unchanged rather than becoming an empty Or
"""
leaf = content("⺀")
assert _widen_cjk_leaf(leaf) is leaf
def test_the_original_leaf_object_is_the_first_alternative(self) -> None:
"""
GIVEN:
- A CJK term on content
WHEN:
- The hook widens it
THEN:
- The Or's first child is the leaf object itself, not an equal
copy: analyze() only keeps the leaf's own enclosing-group
analysis for that exact object
"""
leaf = content("東京")
widened = _widen_cjk_leaf(leaf)
assert isinstance(widened, wc_ast.Or)
assert widened.children[0] is leaf
def test_a_cjk_term_gains_a_bigram_alternative_on_its_own_field(self) -> None:
"""
GIVEN:
- A CJK term on title
WHEN:
- The hook widens it
THEN:
- The bigram side targets bigram_title, the leaf's own field's
companion, so a fielded term stays fielded
"""
leaf = wc_ast.Term(field=TITLE, text="東京")
assert _widen_cjk_leaf(leaf) == _widened(
leaf,
wc_ast.Term(field=BIGRAM_TITLE, text="東京"),
)
def test_each_cjk_run_gets_its_own_bigram_leaf(self) -> None:
"""
GIVEN:
- One term whose text holds two CJK runs split by an
interpunct (東京・大阪)
WHEN:
- The hook widens it
THEN:
- The bigram side is Or(bigram 東京, bigram 大阪), one leaf
per run. Joining the runs with a space into one leaf would
produce bigrams spanning the space, which only exist when
the two runs sit side by side in a document. The runs are
alternatives, not requirements: a multi-run term matches on
any one of its runs, which is what the separate bigram
clause did before this change
"""
leaf = content("東京・大阪")
assert _widen_cjk_leaf(leaf) == _widened(
leaf,
wc_ast.Or(children=(bigram("東京"), bigram("大阪"))),
)
def test_each_run_keeps_its_own_bigrams_required(self) -> None:
"""
GIVEN:
- A term whose two runs each produce more than one bigram
(東京都・大阪府)
WHEN:
- The hook widens it and the tree is analyzed
THEN:
- Each run's bigrams stay And-combined inside the cross-run
Or. The Or is between runs only; multitoken=AND on the
bigram FieldSpec is what requires a run's own bigrams, and
it does not inherit from the enclosing group. Without this,
東京都 would match a document holding only 京都
"""
assert _analyze(content("東京都・大阪府")) == wc_ast.Or(
children=(
wc_ast.And(children=(content("東京都"), content("大阪府"))),
wc_ast.And(children=(bigram("東京"), bigram("京都"))),
wc_ast.And(children=(bigram("大阪"), bigram("阪府"))),
),
)
def test_a_phrase_widens_to_a_bigram_term_not_a_bigram_phrase(self) -> None:
"""
GIVEN:
- A quoted CJK phrase on content
WHEN:
- The hook widens it
THEN:
- The bigram side is a plain Term: the bigram analyzer puts
every token at position 0, so a positional Phrase against it
could never match
"""
leaf = wc_ast.Phrase(field=CONTENT, text="東京都")
assert _widen_cjk_leaf(leaf) == _widened(leaf, bigram("東京都"))
def test_a_multi_word_phrase_requires_every_run(self) -> None:
"""
GIVEN:
- A quoted CJK phrase holding two words
WHEN:
- The hook widens it
THEN:
- The runs are And-combined, where a Term's runs are
Or-combined (test_each_cjk_run_gets_its_own_bigram_leaf).
The parser's default group is And, so an Or here would let
the quoted phrase match strictly more than the same two
words unquoted, and quoting must not widen a search
"""
leaf = wc_ast.Phrase(field=CONTENT, text="東京都 大阪府")
assert _widen_cjk_leaf(leaf) == _widened(
leaf,
wc_ast.And(children=(bigram("東京都"), bigram("大阪府"))),
)
@pytest.mark.parametrize(
("leaf", "alternative"),
[
pytest.param(
content("東京-report"),
wc_ast.And(children=(bigram("東京"), content("report"))),
id="separated_term",
),
pytest.param(
wc_ast.Phrase(field=CONTENT, text="東京 report"),
wc_ast.And(children=(bigram("東京"), content("report"))),
id="phrase",
),
pytest.param(content("東京report"), bigram("東京"), id="glued"),
],
)
def test_latin_is_required_only_where_the_analyzer_splits_it_off(
self,
leaf: wc_ast.Term | wc_ast.Phrase,
alternative: wc_ast.Node,
) -> None:
"""
GIVEN:
- A leaf mixing CJK and latin text, either split into separate
tokens by the content analyzer (東京-report, "東京 report")
or glued into one token (東京report)
WHEN:
- The hook widens it
THEN:
- A separate latin token is required on the leaf's own field
beside the CJK run's bigrams. Glued latin is not required:
the index only holds it inside the whole unspaced token, so
requiring it would lose documents the CJK run finds
"""
assert _widen_cjk_leaf(leaf) == _widened(leaf, alternative)
def test_only_latin_pieces_after_a_tokenizer_drop_still_widens(self) -> None:
"""
GIVEN:
- A term combining a tokenizer-dropped CJK codepoint (U+2E80)
with a latin word (⺀report): _has_cjk matches on the
leading codepoint, but the tokenizer discards it and keeps
only the "report" token, so no run ever reaches cjk_terms
WHEN:
- The hook widens it
THEN:
- The alternative is the lone latin term itself, not wrapped
in And (a single piece collapses to itself), even though
the CJK side contributed nothing
"""
leaf = content("⺀report")
assert _widen_cjk_leaf(leaf) == _widened(leaf, content("report"))
def test_new_nodes_carry_the_leaf_span(self) -> None:
"""
GIVEN:
- A CJK term with a source span, split into two pieces
WHEN:
- The hook widens it
THEN:
- The Or, the alternative and each piece carry the leaf's
span, so an emit-time diagnostic still points into the
query text
"""
leaf = wc_ast.Term(field=CONTENT, text="東京・大阪", startchar=3, endchar=8)
widened = _widen_cjk_leaf(leaf)
assert isinstance(widened, wc_ast.Or)
alternative = widened.children[1]
assert isinstance(alternative, wc_ast.Or)
spans = {
(node.startchar, node.endchar)
for node in (widened, alternative, *alternative.children)
}
assert spans == {(3, 8)}
class TestAnalyzedTree:
@pytest.mark.parametrize(
"node",
[
pytest.param(wc_ast.Prefix(field=CONTENT, text="東京"), id="cjk_prefix"),
pytest.param(
wc_ast.Wildcard(field=CONTENT, pattern="東*"),
id="cjk_wildcard",
),
],
)
def test_patterns_are_never_widened(self, node: wc_ast.Node) -> None:
"""
GIVEN:
- A CJK prefix or wildcard pattern on content
WHEN:
- The tree is analyzed with the hook
THEN:
- It comes back unchanged: analyze() only offers Term and
Phrase leaves to the hook
"""
assert _analyze(node) == node
@pytest.mark.parametrize(
"build",
[
pytest.param(lambda node: node, id="bare"),
pytest.param(lambda node: wc_ast.Not(child=node), id="negated"),
],
)
def test_a_run_past_the_length_limit_keeps_its_bigram_side(
self,
build: Callable[[wc_ast.Node], wc_ast.Node],
long_cjk_run: str,
) -> None:
"""
GIVEN:
- A CJK run longer than the content analyzer's remove_long
limit, bare or under NOT
WHEN:
- The tree is analyzed with the hook
THEN:
- The content side drops out and the bigram side stands alone,
still negated under NOT. A NOT left holding nothing would
turn into "match everything" instead
"""
resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)
assert resolved is not None
bigrams = resolved.spec.analyzer(long_cjk_run)
assert _analyze(build(content(long_cjk_run))) == build(
wc_ast.And(
children=tuple(bigram(token) for token in dict.fromkeys(bigrams)),
),
)
def test_a_one_character_run_drops_out_of_the_alternative(self) -> None:
"""
GIVEN:
- A term whose runs are 東 (one character) and 大阪
WHEN:
- The tree is analyzed with the hook
THEN:
- Only 大阪 is required on the bigram side: a one-character run
has no bigram. An accepted gap, the same on the positive and
the negated side
"""
analyzed = _analyze(content("東・大阪"))
assert analyzed == _widened(
wc_ast.And(children=(content("東"), content("大阪"))),
bigram("大阪"),
)
def test_the_title_boost_wraps_the_widened_title_leaf(self) -> None:
"""
GIVEN:
- An unfielded CJK query parsed the way parse_user_query parses
it: copied onto every default field, title boosted
WHEN:
- The tree is analyzed with the hook
THEN:
- Each field copy is widened on its own bigram field, and the
title boost wraps the title copy's whole widened Or, so a
title bigram match scores like a title match
"""
parsed = wc.parse(
"東京",
registry=get_field_registry(None),
default_fields=_DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS,
).ast
analyzed = _analyze(parsed)
assert isinstance(analyzed, wc_ast.Or)
assert (
wc_ast.Boosted(
child=_widened(
wc_ast.Term(field=TITLE, text="東京"),
wc_ast.Term(field=BIGRAM_TITLE, text="東京"),
),
boost=_FIELD_BOOSTS["title"],
)
in analyzed.children
)
assert {BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {
child.field for child in analyzed.children if isinstance(child, wc_ast.Term)
}
def test_under_and_the_original_side_keeps_and(self) -> None:
"""
GIVEN:
- A term the content analyzer splits into two tokens
(東京・大阪), inside an And group
WHEN:
- The tree is analyzed with the hook
THEN:
- The original side still ANDs its two tokens. Wrapping the
leaf in the hook's Or must not turn that into an Or of them.
The cross-run Or flattens into the widening Or, so the
alternative is not a single child here and _widened() does
not fit; the tree is written out literally
"""
tree = wc_ast.And(children=(content("東京・大阪"), content("report")))
assert _analyze(tree) == wc_ast.And(
children=(
wc_ast.Or(
children=(
wc_ast.And(children=(content("東京"), content("大阪"))),
bigram("東京"),
bigram("大阪"),
),
),
content("report"),
),
)
def test_under_or_the_original_side_keeps_or(self) -> None:
"""
GIVEN:
- The same two-token term inside an Or group
WHEN:
- The tree is analyzed with the hook
THEN:
- The original side ORs its tokens, as the group says, and the
hook's Or flattens into the group
"""
tree = wc_ast.Or(children=(content("東京・大阪"), content("report")))
assert _analyze(tree) == wc_ast.Or(
children=(
content("東京"),
content("大阪"),
bigram("東京"),
bigram("大阪"),
content("report"),
),
)
@pytest.mark.parametrize(
"build",
[
pytest.param(lambda leaf: wc_ast.Not(child=leaf), id="not"),
pytest.param(
lambda leaf: wc_ast.AndNot(positive=content("invoice"), negative=leaf),
id="andnot",
),
pytest.param(
lambda leaf: wc_ast.AndMaybe(
required=content("invoice"),
optional=leaf,
),
id="andmaybe",
),
pytest.param(
lambda leaf: wc_ast.Require(
scored=leaf,
filter_only=content("invoice"),
),
id="require",
),
pytest.param(
lambda leaf: wc_ast.Boosted(child=leaf, boost=2.0),
id="boosted",
),
],
)
def test_the_widened_leaf_stays_where_it_was(
self,
build: Callable[[wc_ast.Node], wc_ast.Node],
long_cjk_run: str,
) -> None:
"""
GIVEN:
- A CJK term under Not, AndNot, AndMaybe, Require or Boosted,
beside a latin term
WHEN:
- The tree is analyzed with the hook
THEN:
- The node keeps its type, its latin operand is untouched, and
only the CJK leaf is replaced by its widened Or, in the same
position. A negated CJK leaf is widened too, so a negation
excludes exactly what the positive search would match
"""
assert _analyze(build(content("東京"))) == build(
_widened(content("東京"), bigram("東京")),
)
@@ -0,0 +1,200 @@
"""Which characters count as part of a CJK run.
_CJK_RE decides both what is indexed into the bigram fields
(extract_cjk_text) and how a query's CJK text is cut into runs
(_widen_leaf). A character it misses splits a word in two on both
sides. For the katakana prolonged sound mark that leaves one-character
runs, which have no bigrams, so the word could not be found through the
bigram fields at all.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.search._query import extract_cjk_text
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
pytestmark = pytest.mark.search
class TestRunExtraction:
@pytest.mark.parametrize(
("text", "expected"),
[
pytest.param("コーヒーを飲む", "コーヒーを飲む", id="prolonged_sound_mark"),
pytest.param("コーヒー", "コーヒー", id="halfwidth_prolonged_sound_mark"),
pytest.param("ゴルフ", "ゴルフ", id="halfwidth_voiced_mark"),
pytest.param("パン", "パン", id="halfwidth_semi_voiced_mark"),
pytest.param("締め切り〆日", "締め切り〆日", id="closing_mark"),
pytest.param("人々の生活", "人々の生活", id="iteration_mark"),
pytest.param("東京・大阪", "東京 大阪", id="interpunct_still_splits"),
pytest.param("東京、大阪", "東京 大阪", id="comma_still_splits"),
],
)
def test_japanese_word_marks_stay_inside_the_run(
self,
text: str,
expected: str,
) -> None:
"""
GIVEN:
- Japanese text containing ー (U+30FC), its halfwidth form
(U+FF70), the halfwidth voiced and semi-voiced marks
(U+FF9E, U+FF9F), 〆 or 々, or a separator between two runs
WHEN:
- Its CJK runs are extracted for the bigram fields
THEN:
- The marks stay inside their word's run. Every one of them
has Unicode script Common, so the script classes alone miss
them. Punctuation such as ・ and 、 still separates runs
"""
assert extract_cjk_text(text) == expected
@pytest.mark.django_db
class TestProlongedSoundMarkSearch:
@pytest.mark.parametrize(
("query", "content"),
[
pytest.param("コーヒー", "コーヒーを飲む", id="fullwidth"),
pytest.param("サーバー", "サーバーの設定を変更", id="fullwidth_two_marks"),
pytest.param("コーヒー", "コーヒーを飲む", id="halfwidth"),
],
)
def test_a_word_with_the_mark_is_found_inside_running_text(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
query: str,
content: str,
) -> None:
"""
GIVEN:
- A document containing a katakana word with ー inside a
longer unspaced run, and an unrelated latin document
WHEN:
- The word is searched
THEN:
- The document matches through the bigram fields. Without the
mark in _CJK_RE, the word splits into one-character runs
with no bigrams, and only a standalone content token could
match
"""
match = index_document(title="A", content=content)
index_document(title="B", content="invoice only")
assert matched_ids(query) == {match.pk}
def test_not_excludes_a_word_with_the_mark(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- An "invoice" document containing コーヒー inside a longer
run, and a latin-only "invoice" document
WHEN:
- "invoice NOT コーヒー" is searched
THEN:
- Only the latin document matches: the negation excludes what
the positive search finds
"""
index_document(
title="A",
content="コーヒーを飲む invoice",
)
latin = index_document(title="B", content="invoice only")
assert matched_ids("invoice NOT コーヒー") == {latin.pk}
def test_a_different_word_sharing_only_the_edges_does_not_match(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with コード and ヒント, two separate words
WHEN:
- "コーヒー" is searched
THEN:
- Nothing matches. With ー outside _CJK_RE, both sides reduce
to runs like コ and ヒ joined by spaces, and the space
bigrams that produces could match unrelated words
"""
index_document(title="A", content="コード ヒント")
assert matched_ids("コーヒー") == set()
@pytest.mark.django_db
class TestHalfwidthVoicedMarkSearch:
"""Halfwidth katakana, as legacy systems and bank statements write it.
Its voiced marks are separate codepoints with no precomposed form, so
NFC leaves them where they are and the character class has to cover
them. A word like パン is three codepoints, and splitting it at the mark
leaves two one-character runs with no bigrams, so it is not merely
imprecise but unfindable.
"""
@pytest.mark.parametrize(
("query", "content"),
[
pytest.param("ゴルフ", "ゴルフ場の利用料金", id="voiced_mark"),
pytest.param("パン", "パンと牛乳の購入", id="semi_voiced_mark"),
],
)
def test_a_word_with_the_mark_is_found_inside_running_text(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
query: str,
content: str,
) -> None:
"""
GIVEN:
- A document holding a halfwidth katakana word inside a longer
unspaced run, and an unrelated latin document
WHEN:
- The word is searched
THEN:
- The document matches. Without the mark in _CJK_RE the word
splits at it, and パン in particular loses both halves to the
one-character rule and can never be found
"""
match = index_document(title="A", content=content)
index_document(title="B", content="invoice only")
assert matched_ids(query) == {match.pk}
def test_the_whole_word_is_searched_not_just_the_tail(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document holding ゴルフ, and one holding ルフナ゙ー , which
shares the ルフ pair that splitting ゴルフ at its mark leaves
behind
WHEN:
- "ゴルフ" is searched
THEN:
- Only the first matches. Splitting at the mark would search
the ルフ fragment alone, which silently degrades the query
into a two-character substring search
"""
golf = index_document(title="A", content="ゴルフ場の利用料金")
index_document(title="B", content="ルフナ゙ーの修理")
assert matched_ids("ゴルフ") == {golf.pk}
@@ -0,0 +1,622 @@
"""CJK terms in QUERY-mode searches, matched through their bigram fields
in place inside the parsed query.
The content analyzer keeps an unspaced CJK run as one token, so a CJK
term is only findable through the bigram fields. Each CJK leaf is widened
where it sits, so everything around it (AND, NOT, REQUIRE, boosts,
fielding) constrains the bigram match exactly as it constrains the
original term.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Correspondent
from documents.models import Document
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestNegationSymmetry:
def test_not_excludes_what_the_positive_term_matches(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose 東京 sits inside a longer unspaced run
(indexed as one content token), and a latin-only document,
both matching "invoice"
WHEN:
- "東京" and "invoice NOT 東京" are searched
THEN:
- 東京 finds the CJK document, and NOT 東京 excludes that same
document. A content-only negation never fires here, since
the content field holds the whole run, not 東京
"""
cjk = index_document(
title="A",
content="東京都の公共文書について invoice",
)
latin = index_document(title="B", content="invoice only")
assert matched_ids("東京") == {cjk.pk}
assert matched_ids("invoice NOT 東京") == {latin.pk}
def test_bigram_approximation_is_the_same_on_both_sides(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document mentioning 東京 and 京都 separately, never 東京都
WHEN:
- "東京都" and "invoice NOT 東京都" are searched
THEN:
- 東京都 matches it (both of its bigrams are present, and the
bigram match ignores position), and NOT 東京都 excludes it.
An accepted approximation: a query and its negation agree
"""
separate = index_document(
title="A",
content="東京 と 京都 invoice",
)
latin = index_document(title="B", content="invoice only")
assert matched_ids("東京都") == {separate.pk}
assert matched_ids("invoice NOT 東京都") == {latin.pk}
@pytest.mark.parametrize(
"query",
[
pytest.param("invoice NOT 東京都", id="term"),
pytest.param('invoice NOT "東京都"', id="phrase"),
],
)
def test_a_negated_multi_bigram_run_needs_every_bigram_to_exclude(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
query: str,
) -> None:
"""
GIVEN:
- A document containing 東京都, and one containing only 京都
(one of 東京都's two bigrams)
WHEN:
- 東京都 is excluded, as a bare term or a quoted phrase
THEN:
- Only the 東京都 document is excluded. Sharing one bigram
with the excluded run is not enough to be excluded
"""
full = index_document(
title="A",
content="東京都の公共文書について invoice",
)
partial = index_document(
title="B",
content="京都の invoice",
)
assert matched_ids("invoice") == {full.pk, partial.pk}
assert matched_ids(query) == {partial.pk}
def test_a_run_past_the_length_limit_is_matched_and_excluded(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
long_cjk_run: str,
) -> None:
"""
GIVEN:
- A document holding a CJK run longer than the content
analyzer's remove_long limit, and a latin-only document
WHEN:
- The long run is searched bare, and excluded with NOT
THEN:
- Bare, it matches the document; negated, it excludes that
document and nothing else. The content side of the run
analyzes to nothing, which must not erase the NOT
"""
long = index_document(
title="A",
content=f"{long_cjk_run} invoice",
)
latin = index_document(title="B", content="invoice only")
assert matched_ids(long_cjk_run) == {long.pk}
assert matched_ids(f"invoice NOT {long_cjk_run}") == {latin.pk}
class TestTokenizerDroppedCodepoints:
@pytest.fixture
def invoice(self, index_document: Callable[..., Document]) -> Document:
"""One latin "invoice" document, with no U+2E80 anywhere in it."""
return index_document(title="A", content="invoice only")
def test_a_regex_only_match_finds_nothing(
self,
invoice: Document,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose content has nothing resembling U+2E80 (a
CJK Radicals Supplement codepoint that matches _CJK_RE but
that the content analyzer's simple tokenizer yields no
token for at all)
WHEN:
- "⺀" is searched
THEN:
- Nothing matches: _widen_leaf finds no CJK pieces to widen
with and returns the leaf as it is, which analyzes to the
same nothing it always did
"""
assert matched_ids("⺀") == set()
def test_not_a_regex_only_match_excludes_nothing(
self,
invoice: Document,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- An "invoice" document with no U+2E80 in it
WHEN:
- "invoice NOT ⺀" is searched
THEN:
- The document still matches: the negated leaf widens to
nothing extra, exactly as an unwidened NOT of a term with
no matches would behave
"""
assert matched_ids("invoice NOT ⺀") == {invoice.pk}
class TestStructureConstrainsTheCjkMatch:
@pytest.fixture
def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
"""One document with both 東京 and "invoice", one with only 東京."""
return {
"both": index_document(title="A", content="東京都の請求書 invoice").pk,
"cjk_only": index_document(title="B", content="東京 report").pk,
}
@pytest.mark.parametrize(
("query", "expected"),
[
pytest.param("東京 AND invoice", {"both"}, id="and"),
pytest.param("東京 REQUIRE invoice", {"both"}, id="require"),
pytest.param("東京^2 AND invoice", {"both"}, id="boosted"),
pytest.param("東京 ANDMAYBE invoice", {"both", "cjk_only"}, id="andmaybe"),
],
)
def test_a_latin_operand_still_constrains_the_cjk_term(
self,
matched_ids: Callable[[str], set[int]],
pks: dict[str, int],
query: str,
expected: set[str],
) -> None:
"""
GIVEN:
- One document with both 東京 and "invoice", one with only 東京
WHEN:
- 東京 is combined with "invoice" by AND, REQUIRE, a boosted
AND, or ANDMAYBE
THEN:
- AND and REQUIRE (boosted or not) need "invoice" too, so only
the document with both matches; ANDMAYBE leaves "invoice"
optional, so both match
"""
assert matched_ids(query) == {pks[label] for label in expected}
class TestFieldedTerms:
def test_a_fielded_multi_run_term_matches_through_either_run(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with 東京 and 大阪, and one with only 東京, both
containing "report"
WHEN:
- "content:東京・大阪 AND report" is searched (the content
analyzer splits 東京・大阪 into two tokens)
THEN:
- Both match. The original side still ANDs its two content
tokens, pinned in test_cjk_leaf_rewrite.py, but the bigram
side accepts either run, so the 東京-only document comes
back through it. The AND still binds: "report" is required
of both. This is what the separate bigram clause returned
before this change
"""
both = index_document(title="A", content="東京 大阪 report")
one = index_document(title="B", content="東京 report")
assert matched_ids("content:東京・大阪 AND report") == {
both.pk,
one.pk,
}
def test_a_run_is_matched_wherever_it_appears_in_the_text(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document holding 大阪 and 東京 apart, inside one unspaced
run (大阪府と東京都), a document with only 東京, and one
with neither
WHEN:
- "content:東京・大阪" is searched
THEN:
- The first two match and the third does not. Each run is
searched on its own, so no bigram spanning the gap between
the runs is required, and either run on its own is enough
"""
apart = index_document(title="A", content="大阪府と東京都")
one = index_document(title="B", content="東京 report")
index_document(title="C", content="report only")
assert matched_ids("content:東京・大阪") == {apart.pk, one.pk}
def test_an_unfielded_multi_run_term_matches_on_either_run(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with both CJK runs, one with only the first run,
and one with neither, all containing "invoice"
WHEN:
- The two runs are searched as one unfielded term (東京・大阪)
THEN:
- Both CJK documents match. An unfielded term is copied across
the default fields inside an Or, so the leaf's own semantics
are "either"; the bigram side matches that, and returns what
the separate bigram clause returned before this change
"""
both = index_document(
title="A",
content="大阪府と東京都の報告 invoice",
)
one = index_document(
title="B",
content="東京都の報告書 invoice",
)
index_document(title="C", content="invoice only")
assert matched_ids("東京・大阪") == {both.pk, one.pk}
def test_a_negated_multi_run_term_excludes_on_either_run(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Documents containing both runs of a glued mixed term, only
its first run, only its second run, and neither, all
containing "invoice"
WHEN:
- "invoice NOT 東京report資料" is searched
THEN:
- Only the document with neither run survives. The negation
excludes exactly what the positive term matches, and the
positive term matches on either run, so the exclusion is
wider than an "only documents holding both" reading. Today
this query excludes nothing at all
"""
index_document(
title="A",
content="東京都のreport資料です invoice",
)
index_document(title="B", content="東京都の報告 invoice")
index_document(title="C", content="参考資料の一覧 invoice")
neither = index_document(title="D", content="invoice only")
assert matched_ids("invoice NOT 東京report資料") == {neither.pk}
def test_a_bigram_field_name_is_not_query_syntax(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document holding 東京 in its content
WHEN:
- A query naming an internal bigram field as a field prefix is
searched end to end (bigram_content:東京)
THEN:
- Nothing matches. The prefix is not query syntax, so the
whole thing is plain text: the literal token
"bigram_content" is required on the leaf's own field, and no
document holds it. Asserted through the real search path, so
that parsing with the emit registry instead of the public
one would fail this test. A unit-level wc.parse() assertion
would not: it pins whoosh-compat's handling of an unknown
prefix rather than our choice of parse registry
"""
index_document(title="A", content="東京都の報告書")
assert matched_ids("bigram_content:東京") == set()
class TestMixedScript:
def test_separated_latin_is_required(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with 東京 and "report" as separate words, and one
with 東京 but no "report"
WHEN:
- "東京-report" is searched (the content analyzer splits it into
東京 and report)
THEN:
- Only the document with both matches: the latin piece is
required beside the CJK run's bigrams
"""
both = index_document(title="A", content="東京都の report")
index_document(title="B", content="東京都の invoice")
assert matched_ids("東京-report") == {both.pk}
def test_glued_latin_is_not_required(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document where the latin text is glued to other CJK text
(東京都のreport資料, one content token), and one with 東京
but no "report" at all
WHEN:
- "東京report" is searched (one token: latin glued to CJK)
THEN:
- Both match through 東京's bigrams. Glued latin is only ever
indexed inside a whole unspaced token, so requiring it would
have lost the first document
"""
glued = index_document(
title="A",
content="東京都のreport資料",
)
other = index_document(
title="B",
content="東京都の invoice",
)
assert matched_ids("東京report") == {glued.pk, other.pk}
def test_not_a_separated_mixed_term_excludes_only_documents_with_both(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Three "invoice" documents: with 東京 and "report", with 東京
only, and latin only
WHEN:
- "invoice NOT 東京-report" is searched
THEN:
- Only the document with both is excluded, not every document
mentioning 東京
"""
index_document(
title="A",
content="東京都の report invoice",
)
cjk_only = index_document(
title="B",
content="東京都の invoice",
)
latin = index_document(title="C", content="invoice only")
assert matched_ids("invoice NOT 東京-report") == {
cjk_only.pk,
latin.pk,
}
def test_a_one_character_run_is_not_required(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document containing 大阪 but not 東
WHEN:
- "content:東・大阪" is searched
THEN:
- It matches: a one-character run has no bigram, so only 大阪
is required. An accepted gap, the same under NOT
"""
osaka = index_document(title="A", content="大阪府の報告書")
assert matched_ids("content:東・大阪") == {osaka.pk}
class TestQuotedPhrase:
"""A quoted CJK phrase must not match more than the same words unquoted.
The bigram fields put every token at position 0, so no alternative
built from them can enforce adjacency. Requiring both runs is the
tightest thing available, and the floor: the parser's default group is
And, so anything looser would make quoting widen the search.
"""
@pytest.fixture
def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
return {
"tokyo": index_document(
title="A",
content="東京都の報告書 invoice",
).pk,
"osaka": index_document(
title="B",
content="大阪府の報告書 invoice",
).pk,
"both": index_document(
title="C",
content="東京都と大阪府の報告書 invoice",
).pk,
}
@pytest.mark.parametrize(
"query",
[
pytest.param("東京都 大阪府", id="unquoted"),
pytest.param('"東京都 大阪府"', id="quoted"),
],
)
def test_both_words_are_required_either_way(
self,
matched_ids: Callable[[str], set[int]],
pks: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- Three documents, holding only the first word, only the
second, and both
WHEN:
- The two words are searched unquoted and as a quoted phrase
THEN:
- Only the document holding both matches, either way. Quoting
asks for more than the bare words, never less
"""
assert matched_ids(query) == {pks["both"]}
@pytest.mark.parametrize(
"query",
[
pytest.param("invoice NOT (東京都 大阪府)", id="unquoted"),
pytest.param('invoice NOT "東京都 大阪府"', id="quoted"),
],
)
def test_the_negation_excludes_only_what_the_phrase_matches(
self,
matched_ids: Callable[[str], set[int]],
pks: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- The same three documents, all matching "invoice"
WHEN:
- The two words are excluded, grouped and as a quoted phrase
THEN:
- Only the document holding both words is excluded. The
negation is the mirror of the positive side, so a phrase
that required either word would take the two single-word
documents out with it
"""
assert matched_ids(query) == {pks["tokyo"], pks["osaka"]}
class TestOtherDefaultFields:
@pytest.mark.parametrize("where", ["title", "correspondent"])
def test_not_excludes_a_cjk_match_outside_content(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
where: str,
) -> None:
"""
GIVEN:
- An "invoice" document whose only 東京 is inside a longer run
in its title or its correspondent, and a latin-only one
WHEN:
- "invoice NOT 東京" is searched
THEN:
- The CJK document is excluded: every default field's bigram
companion is widened, not only content's
"""
kwargs: dict[str, object] = {
"title": "A",
"content": "invoice",
}
if where == "title":
kwargs["title"] = "東京都の報告書"
else:
kwargs["correspondent"] = Correspondent.objects.create(name="東京電力")
index_document(**kwargs)
latin = index_document(
title="B",
content="invoice only",
)
assert matched_ids("invoice NOT 東京") == {latin.pk}
class TestFuzzyDoesNotReadmit:
@pytest.mark.parametrize(
("query", "threshold", "expected"),
[
pytest.param("invoice NOT 東京", None, {"latin"}, id="not_fuzzy_off"),
pytest.param("invoice NOT 東京", 0.0, {"latin"}, id="not_fuzzy_on"),
pytest.param("東京 AND invoice", None, {"cjk"}, id="and_fuzzy_off"),
pytest.param(
"東京 AND invoice",
0.0,
{"cjk"},
id="and_fuzzy_on",
),
],
)
def test_fuzzy_on_matches_fuzzy_off_for_structure(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings: SettingsWrapper,
query: str,
threshold: float | None,
expected: set[str],
) -> None:
"""
GIVEN:
- An "invoice" document whose 東京 is inside a longer unspaced
run, a 東京-only document and a latin-only "invoice"
document, with fuzzy search off or on
WHEN:
- "invoice NOT 東京" or "東京 AND invoice" is searched
THEN:
- The result is the same whether fuzzy is on or off: fuzzy
widening happens inside the tree now, so a CJK NOT or AND
still constrains both the exact and the fuzzy side, and
neither re-admits what the structure excludes
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
cjk = index_document(
title="A",
content="東京都の公共文書について invoice",
)
cjk_only = index_document(
title="B",
content="東京都の報告書",
)
latin = index_document(title="C", content="invoice only")
pks = {"cjk": cjk.pk, "cjk_only": cjk_only.pk, "latin": latin.pk}
assert matched_ids(query) == {pks[label] for label in expected}
@@ -0,0 +1,72 @@
"""Whoosh's compact, separator-free date spelling, resolved end to end.
whoosh-compat owns both widths of this spelling and asserts both forms'
bounds directly in its own test suite: the 8-digit form as a whole calendar
day (lower bound, upper bound and exclusivity), and the 14-digit form as a
single instant. The 14-digit form is kept here as the single representative
because it is the one that exercises paperless's ``added`` DATETIME fast
field at full precision: the corpus separates a document at the named
instant from one on the same calendar day at another hour and one on the
next day at the same hour, so a query that degrades into a whole-day
window, or drops the time of day, matches the wrong set rather than passing
on a corpus that could not tell the difference.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@pytest.fixture
def docs(index_document: Callable[..., Document]) -> dict[str, int]:
return {
"instant": index_document(
title="On the instant",
content="x",
added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
).pk,
"same_day": index_document(
title="Same day, other hour",
content="x",
added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
).pk,
"next_day": index_document(
title="Next day, same hour",
content="x",
added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
).pk,
}
class TestCompactDateForms:
def test_fourteen_digits_is_a_single_instant(
self,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents indexed on the ``added`` DATETIME fast field:
one at 2005-03-04T15:30:00, one on the same calendar day at a
different hour, and one on the next day at the same hour
WHEN:
- Searching with the 14-digit compact date form
``added:20050304153000``
THEN:
- Only the document at that exact instant matches; the same-day
document is what tells this apart from the 8-digit day-window
form, and the next-day document from a form that ignored the
time of day altogether
"""
assert matched_ids("added:20050304153000") == {docs["instant"]}
@@ -0,0 +1,78 @@
"""Pins the correctness gained by deleting the pre-parse
_quote_date_keyword_phrases rewrite.
That rewrite matched date-keyword phrases (e.g. "previous month" after a
date field) anywhere in the raw query string, including inside an
unrelated quoted string, and inserted quotes mid-phrase there too. Its
own docstring gave ``title:"see added:previous month notes"`` as the
example of what it corrupted. whoosh-compat's grammar accepts the same
phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases
in test_acceptance.py), so the rewrite was redundant everywhere it was
safe and actively wrong everywhere it was not. This is the one case that
tells the two apart: a literal title phrase that happens to contain
"added:previous month" as running text.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestQuotedStringContainingDateKeywordText:
"""A quoted title phrase containing the literal text
"added:previous month" as running words must match on that literal
text alone, never spill into an unfielded search for "previous" and
"month" across the default search fields the way the deleted rewrite
would have decomposed it into."""
def test_matches_only_the_literal_phrase(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose title literally contains "see
added:previous month notes", and a decoy document whose
title/content carry the individual fragments the deleted
_quote_date_keyword_phrases rewrite would have decomposed
the phrase into (the decoy would incorrectly match under
the deleted rewrite: its title contains the "see added:"
and " notes" fragments the corrupted parse required as
title phrases, and its content supplies "previous" and
"month" as the decomposed word-match clauses the rewrite
turned the middle of the phrase into)
WHEN:
- 'title:"see added:previous month notes"' is searched
THEN:
- Only the document with the literal phrase matches; it must
never spill into an unfielded search for "previous" and
"month" across the default search fields
"""
literal = index_document(
title="see added:previous month notes",
content="quarterly filing",
archive_serial_number=920,
)
# Under the deleted rewrite, this decoy would incorrectly match:
# its title contains the "see added:" and " notes" fragments the
# corrupted parse required as title phrases, and its content
# supplies "previous" and "month" as the decomposed word-match
# clauses the rewrite turned the middle of the phrase into.
decoy = index_document(
title="see added: quarterly report notes",
content="we reviewed the previous statement about month end",
archive_serial_number=921,
)
query = 'title:"see added:previous month notes"'
assert matched_ids(query) == {literal.pk}
assert decoy.pk not in matched_ids(query)
@@ -0,0 +1,87 @@
"""Date keyword phrases (``today``, etc.) resolved in a non-UTC timezone,
end to end.
paperless's own ``tz=get_current_timezone()`` plumbing
(``TantivyBackend._parse_query``) is exercised elsewhere only for
relative *ranges* (``added:[-1 week to now]``, in
documents/tests/test_api_search.py). This covers a date *keyword*
(``today``), whose day boundary depends on the active timezone the same
way but goes through whoosh-compat's DateParserPlugin resolution instead
of an explicit range.
Discriminating shape: frozen at 2026-06-15T02:00 UTC, which is
2026-06-14T22:00 in America/New_York -- still "today" (06-14) there, but
already "today" (06-15) in UTC. Two documents pin both directions of the
mistake a hardcoded-UTC bug would make:
- ``in_ny_today`` (added 2026-06-14T20:00 UTC = 2026-06-14T16:00 NY) is
inside New York's "today" window and outside a naive UTC-calendar-day
window. A ``tz``-ignoring bug would miss it.
- ``in_utc_calendar_day_only`` (added 2026-06-15T10:00 UTC =
2026-06-15T06:00 NY) is inside a naive UTC-calendar-day window but
outside New York's actual "today" window. A ``tz``-ignoring bug would
wrongly match it.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)
class TestDateKeywordUsesTheActiveTimezone:
def test_today_matches_the_new_york_calendar_day_not_the_utc_one(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings: SettingsWrapper,
) -> None:
"""
GIVEN:
- TIME_ZONE set to America/New_York, time frozen at
2026-06-15T02:00 UTC (2026-06-14T22:00 NY -- still "today"
there, but already "today" in UTC), and two documents: one
added inside New York's "today" window but outside a naive
UTC-calendar-day window, the other the reverse (inside a
naive UTC-calendar-day window but outside New York's actual
"today")
WHEN:
- "added:today" is searched
THEN:
- Only the document inside New York's actual "today" window
matches, proving our tz=get_current_timezone() plumbing
resolves the date keyword in the active timezone rather
than a hardcoded UTC calendar day
"""
settings.TIME_ZONE = "America/New_York"
with time_machine.travel(FROZEN_NOW, tick=False):
in_ny_today = index_document(
title="NY today",
content="x",
added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),
)
# Not captured: the exact-set assertion below already proves
# this document (inside a naive UTC-calendar-day window, but
# outside New York's actual "today") does not match.
index_document(
title="UTC calendar day only",
content="x",
added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),
)
assert matched_ids("added:today") == {in_ny_today.pk}
@@ -0,0 +1,32 @@
"""``_DEFAULT_SEARCH_FIELDS`` must stay a subset of the registered public
field names.
Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in
``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every
unfielded search at request time, since ``index.parse_query``, the
fuzzy clause builder and the CJK bigram widening are handed a field name
the schema no longer has.
"""
from __future__ import annotations
from documents.search._fields import PUBLIC_FIELDS
from documents.search._query import _DEFAULT_SEARCH_FIELDS
class TestDefaultSearchFieldsAreRegistered:
def test_every_default_search_field_is_a_public_field(self) -> None:
"""
GIVEN:
- PUBLIC_FIELDS and _DEFAULT_SEARCH_FIELDS, our own field
tables
WHEN:
- Every name in _DEFAULT_SEARCH_FIELDS is checked against the
registered public field names
THEN:
- Every one is present; a rename in PUBLIC_FIELDS not
mirrored here would 400 every unfielded search at request
time
"""
public_field_names = {f.name for f in PUBLIC_FIELDS}
assert set(_DEFAULT_SEARCH_FIELDS) <= public_field_names
@@ -0,0 +1,453 @@
"""Pins the search syntax that ``docs/usage.md`` promises users.
Every query here is syntax the "Document searches" section of
``docs/usage.md`` documents, either spelled as the docs spell it or as a
concrete instance of a form the docs describe. Each case indexes real
documents and asserts on matched document IDs rather than on the parsed
query, because a query that parses cleanly is not necessarily a query that
means what the documentation says it means: ``added:now`` parses without a
single diagnostic and then matches nothing, because it resolves to an
instant rather than to a span.
The negative cases matter as much as the positive ones. They pin the
behaviours the docs explicitly warn about, so that if any of them ever
starts working the warning can be removed deliberately rather than being
left standing as a lie.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import Note
from documents.models import Tag
from documents.search._errors import InvalidDateQuery
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Generator
from django.contrib.auth.models import User
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# A Monday, so that "next monday"/"last monday" land a clean week either side.
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
# The checksum used in the docs' `checksum:` example.
DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
class TestLogicalExpressions:
@pytest.fixture
def docs(self, index_document: Callable[..., Document]) -> dict[str, int]:
return {
"secret": index_document(
title="Invoice one",
content="invoice secret contents",
).pk,
"plain": index_document(
title="Invoice two",
content="invoice ordinary contents",
).pk,
}
def test_not_excludes_a_term(
self,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one not
WHEN:
- "invoice NOT secret" is searched, as docs/usage.md documents
THEN:
- Only the document without "secret" matches
"""
assert matched_ids("invoice NOT secret") == {docs["plain"]}
def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
self,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one not
WHEN:
- "invoice -secret" is searched (a leading hyphen, not "NOT")
THEN:
- Only the document containing "secret" matches, because
separators are stripped at index time, so "-secret" is
indexed as the plain term "secret" and the query becomes an
AND rather than an exclusion, exactly as the docs warn
"""
assert matched_ids("invoice -secret") == {docs["secret"]}
def test_or_inside_parentheses_matches_either_branch(
self,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one
containing "ordinary"
WHEN:
- "invoice AND (secret OR ordinary)" is searched
THEN:
- Both documents match
"""
matched = matched_ids("invoice AND (secret OR ordinary)")
assert matched == {docs["secret"], docs["plain"]}
class TestPhraseSearch:
def test_quoted_phrase_requires_the_words_in_order(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose content contains "the quick brown fox jumps"
WHEN:
- A quoted phrase is searched, in order and out of order
THEN:
- The in-order phrase matches, and the same words reordered do
not
"""
doc = index_document(
title="Phrase",
content="the quick brown fox jumps",
)
assert matched_ids('"quick brown fox"') == {doc.pk}
assert matched_ids('"brown quick fox"') == set()
class TestTagCommaList:
"""``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks
that the documented spelling still returns what the docs promise: only the
document carrying every listed tag.
It is deliberately not proof of paperless's field configuration, and must
not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec
leaves this test passing, because paperless's analyzer splits the literal
value "bills,unpaid" into the same two tokens the value-list reading
produces, so the two readings select the same documents. The registry fact
-- that ``tag`` opts in and no other field does -- is observable only at
the registry, and is owned by test_registry.py's
``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``.
"""
def test_comma_list_requires_every_listed_tag(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document carrying both "bills" and "unpaid" tags, and a
second document carrying only "bills" (plus "archived")
WHEN:
- "tag:bills,unpaid" is searched
THEN:
- Only the document carrying every listed tag matches, and a
single-tag "tag:bills" search still matches both documents
"""
bills = Tag.objects.create(name="bills")
unpaid = Tag.objects.create(name="unpaid")
archived = Tag.objects.create(name="archived")
both = DocumentFactory(title="Both tags", content="body")
both.tags.add(bills, unpaid)
backend.add_or_update(both)
one = DocumentFactory(title="One tag", content="body")
one.tags.add(bills, archived)
backend.add_or_update(one)
assert matched_ids("tag:bills,unpaid") == {both.pk}
assert matched_ids("tag:bills") == {both.pk, one.pk}
class TestArchiveMetadataFields:
@pytest.fixture
def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
doc = DocumentFactory(
title="Metadata",
content="body",
checksum=DOC_CHECKSUM,
archive_serial_number=100,
page_count=12,
original_filename="invoice.pdf",
)
Note.objects.create(document=doc, user=admin_user, note="a note")
backend.add_or_update(doc)
return doc
@pytest.mark.parametrize(
"query",
[
"asn:100",
"asn:[50 to 150]",
"page_count:12",
"page_count:[10 to 20]",
"num_notes:1",
"num_notes:[1 to 5]",
"original_filename:invoice.pdf",
f"checksum:{DOC_CHECKSUM}",
"checksum:9f86d081*",
# A checksum term is stored verbatim, but a checksum *pattern* is
# lowercased before it is matched, so an uppercase prefix pattern
# still matches even though the uppercase term in the negative
# list below does not.
"checksum:9F86D081*",
],
)
def test_documented_metadata_query_matches(
self,
matched_ids: Callable[[str], set[int]],
doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document with an ASN, page count, a note, an original
filename and a known checksum
WHEN:
- Every documented metadata-field spelling (exact value,
range, and, for checksum, a lowercase prefix pattern
regardless of the case the pattern itself is typed in) is
searched
THEN:
- Each one matches the document
"""
assert matched_ids(query) == {doc.pk}
@pytest.mark.parametrize(
"query",
[
# The docs say only a complete, lowercase checksum matches.
"checksum:9f86d081",
f"checksum:{DOC_CHECKSUM.upper()}",
],
)
def test_partial_or_uppercase_checksum_matches_nothing(
self,
matched_ids: Callable[[str], set[int]],
doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document with a known, complete, lowercase checksum
WHEN:
- An exact-value search is run with a partial or uppercase
spelling of that checksum
THEN:
- Nothing matches, as the docs say only a complete, lowercase
checksum matches as an exact value
"""
assert matched_ids(query) == set()
class TestDocumentedDateForms:
@pytest.fixture(autouse=True)
def frozen_now(self) -> Generator[None, None, None]:
with time_machine.travel(FROZEN_NOW, tick=False):
yield
@pytest.fixture
def dated(self, backend: TantivyBackend) -> dict[str, int]:
stamps = {
"today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
"yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
"tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC),
"next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC),
"last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC),
"january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
"old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
}
docs = {
label: DocumentFactory(title=label, content="dated body", added=stamp)
for label, stamp in stamps.items()
}
with backend.batch_update() as batch:
for doc in docs.values():
batch.add_or_update(doc)
return {label: doc.pk for label, doc in docs.items()}
@pytest.mark.parametrize(
("query", "label"),
[
("added:today", "today"),
("added:yesterday", "yesterday"),
("added:tomorrow", "tomorrow"),
('added:"next monday"', "next_monday"),
('added:"last monday"', "last_monday"),
("added:january", "january"),
("added:2005-03-04", "old"),
("added:2005-03", "old"),
("added:[2005-01-01 to 2005-12-31]", "old"),
("added:[2005 to 2009]", "old"),
# A full timestamp works, but only quoted when it stands alone,
# and only unquoted when it is a range bound. The bare standalone
# spelling is pinned as a non-match below.
('added:"2005-03-04T15:30:00Z"', "old"),
("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"),
# A quoted range bound works when the quotes are single ones; the
# double-quoted spelling is pinned as an error below.
("added:['2005-03-04' to 2005-03-05]", "old"),
],
)
def test_documented_date_form_matches_its_day_or_month(
self,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
query: str,
label: str,
) -> None:
"""
GIVEN:
- Documents dated today, yesterday, tomorrow, next/last
Monday, in January, and on an old fixed date, indexed
against a frozen "now" (a Monday)
WHEN:
- Every documented date-form spelling is searched: relative
keywords, quoted multi-word phrases, a bare year-month, an
explicit range, a quoted full timestamp standing alone, an
unquoted full timestamp as a range bound, and a
single-quoted range bound
THEN:
- Each form matches exactly the document dated on its day or
within its month
"""
assert matched_ids(query) == {dated[label]}
@pytest.mark.parametrize(
"query",
[
# Zero-width: these resolve to a single instant, not a span, so
# nothing in a realistic corpus lands on them. The docs warn
# about them rather than presenting them as usable.
"added:now",
"added:noon",
"added:midnight",
# Quoting is what rescues the other multi-word date expressions,
# so pin that it does not rescue these: the problem is the width
# of the resulting range, not the way the value is delimited.
# One quoted spelling is enough for that; which keyword sits
# inside the quotes is grammar whoosh-compat owns.
'added:"now"',
# A relative offset, which the warning in the docs names by this
# exact spelling. Standing alone it is an instant like the rest of
# this list; the same offset used as a range bound is a real
# window, pinned by the test below.
'added:"-1 week"',
],
)
def test_forms_the_docs_warn_about_match_nothing(
self,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- A realistic dated corpus (see the `dated` fixture)
WHEN:
- A zero-width date form ("now", "noon", "midnight", a quoted
"now") or a standalone relative offset ("-1 week") is
searched: each resolves to a single instant rather than a
span, and quoting does not rescue them the way it rescues
other multi-word date expressions, since the problem is the
width of the resulting range, not how the value is
delimited
THEN:
- Nothing matches, exactly as the docs warn, rather than
presenting these as usable spellings
"""
assert matched_ids(query) == set()
def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
self,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
) -> None:
"""
GIVEN:
- A realistic dated corpus, including a document dated at a
known full timestamp
WHEN:
- The bare, unquoted spelling of that full timestamp is
searched (the quoted and range-bound spellings pinned above
do work and match this fixture's document)
THEN:
- `InvalidDateQuery` is raised rather than the query silently
matching nothing, since this is a user-fixable error the
docs tell the user to quote, and the reported value is the
whole contiguous fragment the user typed, not just the
prefix the date grammar's tokenizer first split on
"""
with pytest.raises(InvalidDateQuery) as exc_info:
matched_ids("added:2005-03-04T15:30:00Z")
assert exc_info.value.field == "added"
assert exc_info.value.value == "2005-03-04T15:30:00Z"
def test_relative_offset_as_a_range_bound_is_a_real_window(
self,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
) -> None:
"""
GIVEN:
- A realistic dated corpus, including a document dated two
hours before a "last Monday to now" window opens, and
documents dated today and yesterday, inside that window
WHEN:
- "added:['-1 week' to now]" is searched: the same offset
that matches nothing standing alone (see the test above),
used here as a range bound instead
THEN:
- The window matches today and yesterday but excludes the
document two hours before it opens, showing the bound is
the offset itself and not a whole-day rounding of it, as
the docs say next to the warning about the standalone form
"""
assert matched_ids("added:['-1 week' to now]") == {
dated["today"],
dated["yesterday"],
}
def test_double_quoted_range_bound_is_rejected(
self,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
) -> None:
"""
GIVEN:
- A realistic dated corpus
WHEN:
- A range bound is double-quoted rather than single-quoted
("added:[\"2005-03-04\" to 2005-03-05]")
THEN:
- `InvalidDateQuery` is raised, pinning which of the two
quote characters fails: quoting a range bound is allowed,
but only with single quotes, since the double-quoted
spelling reaches the date grammar with its quotes still
attached and is not a recognizable date
"""
with pytest.raises(InvalidDateQuery) as exc_info:
matched_ids('added:["2005-03-04" to 2005-03-05]')
assert exc_info.value.value == '"2005-03-04"'
@@ -0,0 +1,358 @@
"""Diagnostics route by Cause, and user-facing messages are host-owned.
whoosh-compat documents ``Diagnostic.message`` as developer output with no
stability guarantee, so it must never reach an HTTP response body.
"""
from __future__ import annotations
import logging
from datetime import UTC
from typing import TYPE_CHECKING
import pytest
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from whoosh_compat.errors import cause_for
from whoosh_compat.fields import FieldKind
from whoosh_compat.fields import FieldRef
from documents.search._errors import SearchQueryError
from documents.search._query import _map_emit_error
from documents.search._query import _single_diagnostic_to_error
from documents.search._query import parse_user_query
if TYPE_CHECKING:
import tantivy
pytestmark = pytest.mark.search
_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"
def _diagnostic(
kind: DiagnosticKind,
*,
field: FieldRef | None = FieldRef("title"),
field_kind: FieldKind | None = FieldKind.TEXT,
) -> Diagnostic:
"""A Diagnostic shaped like the emitter's, with the library's own
kind -> cause mapping rather than a hand-picked cause."""
return Diagnostic(
kind=kind,
cause=cause_for(kind),
message=_LIBRARY_PROSE,
field=field,
field_kind=field_kind,
)
class TestEmitErrorRouting:
"""Every Cause gets a distinguishable treatment, not just "a 400"."""
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.BACKEND_REJECTED,
DiagnosticKind.AST_INVALID_SHAPE,
DiagnosticKind.AST_UNKNOWN_FIELD,
],
)
def test_internal_cause_is_not_converted(self, kind: DiagnosticKind) -> None:
"""
GIVEN:
- A QueryError wrapping a Diagnostic whose Cause is INTERNAL
(BACKEND_REJECTED/AST_INVALID_SHAPE/AST_UNKNOWN_FIELD)
WHEN:
- _map_emit_error processes it
THEN:
- The original QueryError propagates unchanged, so it surfaces
as a 500 monitoring can see, never a 400 blaming the user
"""
error = QueryError(_diagnostic(kind))
with pytest.raises(QueryError) as excinfo:
_map_emit_error(error)
assert excinfo.value is error
def test_misconfigured_cause_is_logged_and_reraised(
self,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A QueryError for SCHEMA_FIELD_MISSING naming field "asn"
WHEN:
- _map_emit_error processes it
THEN:
- Exactly one ERROR log record is emitted naming the field and
the diagnostic kind, and the original QueryError propagates
unchanged: a registry/schema disagreement is transient (the
exact same query succeeds once the index is rebuilt), so it
surfaces as a 500 an operator can see rather than a 400
telling the client their query is permanently invalid
"""
kind = DiagnosticKind.SCHEMA_FIELD_MISSING
error = QueryError(_diagnostic(kind, field=FieldRef("asn")))
with (
caplog.at_level(logging.ERROR, logger="paperless.search"),
pytest.raises(QueryError) as excinfo,
):
_map_emit_error(error)
assert excinfo.value is error
errors = [r for r in caplog.records if r.levelno == logging.ERROR]
assert len(errors) == 1
assert "asn" in errors[0].getMessage()
assert kind.name in errors[0].getMessage()
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.TEXT_RANGE,
DiagnosticKind.PATTERN_TOO_COMPLEX,
DiagnosticKind.EXISTS_REQUIRES_FAST,
],
)
def test_unsupported_cause_is_a_400_with_no_operator_log(
self,
kind: DiagnosticKind,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A QueryError for a query tantivy cannot run
(TEXT_RANGE/PATTERN_TOO_COMPLEX/EXISTS_REQUIRES_FAST)
WHEN:
- _map_emit_error processes it
THEN:
- It becomes a SearchQueryError with no log record at WARNING
or above; a query tantivy cannot run is the user's to fix,
not an operator alert. EXISTS_REQUIRES_FAST is nominally
MISCONFIGURED but belongs here: it is decided from the
registry's own FieldSpec, so it never reports a disagreement
anyone could resolve
"""
with caplog.at_level(logging.WARNING, logger="paperless.search"):
error = _map_emit_error(QueryError(_diagnostic(kind)))
assert isinstance(error, SearchQueryError)
assert caplog.records == []
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.TEXT_RANGE,
DiagnosticKind.PATTERN_TOO_COMPLEX,
DiagnosticKind.EXISTS_REQUIRES_FAST,
],
)
def test_user_facing_message_never_echoes_library_prose(
self,
kind: DiagnosticKind,
) -> None:
"""
GIVEN:
- A QueryError carrying whoosh-compat's own developer-facing
message text (SCHEMA_FIELD_MISSING excluded: it is now
re-raised rather than converted, so it never produces a
user-facing message at all, see
test_misconfigured_cause_is_logged_and_reraised)
WHEN:
- _map_emit_error processes it
THEN:
- The resulting error's string never contains that library
prose
"""
error = _map_emit_error(QueryError(_diagnostic(kind)))
assert _LIBRARY_PROSE not in str(error)
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.TEXT_RANGE,
DiagnosticKind.PATTERN_TOO_COMPLEX,
DiagnosticKind.EXISTS_REQUIRES_FAST,
],
)
def test_user_facing_message_names_the_field(
self,
kind: DiagnosticKind,
) -> None:
"""
GIVEN:
- A QueryError for a JSON subpath field (custom_fields.value)
WHEN:
- _map_emit_error processes it
THEN:
- The resulting error names the field using its canonical
dotted form, including the subpath (FieldRef.__str__ yields
this dotted name, so every user-reachable emit kind can name
it)
"""
diagnostic = _diagnostic(
kind,
field=FieldRef("custom_fields", "value"),
field_kind=FieldKind.JSON,
)
error = _map_emit_error(QueryError(diagnostic))
assert "custom_fields.value" in str(error)
class TestParseDiagnosticMessages:
"""Parse-time diagnostics are host-worded too, off field_kind."""
def test_too_deep_is_a_400_without_library_prose(self) -> None:
"""
GIVEN:
- A parse-time Diagnostic for TOO_DEEP with no field
WHEN:
- _single_diagnostic_to_error processes it
THEN:
- It becomes a SearchQueryError with no library prose in its
message
"""
error = _single_diagnostic_to_error(
_diagnostic(DiagnosticKind.TOO_DEEP, field=None, field_kind=None),
)
assert isinstance(error, SearchQueryError)
assert _LIBRARY_PROSE not in str(error)
@pytest.mark.parametrize(
("kind", "field_kind"),
[
(DiagnosticKind.PATTERN_ON_NUMERIC, FieldKind.U64),
(DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS, FieldKind.BOOLEAN_EXISTS),
(DiagnosticKind.PATTERN_ON_SUBPATH, FieldKind.JSON),
],
)
def test_pattern_on_kinds_name_the_field_and_its_kind(
self,
kind: DiagnosticKind,
field_kind: FieldKind,
) -> None:
"""
GIVEN:
- A parse-time Diagnostic for a pattern used against a kind
that cannot take one
(PATTERN_ON_NUMERIC/PATTERN_ON_BOOLEAN_EXISTS/PATTERN_ON_SUBPATH)
WHEN:
- _single_diagnostic_to_error processes it
THEN:
- The message names both the field and its kind, with no
library prose
"""
error = _single_diagnostic_to_error(
_diagnostic(kind, field=FieldRef("asn"), field_kind=field_kind),
)
message = str(error)
assert _LIBRARY_PROSE not in message
assert "asn" in message
assert field_kind.name.lower() in message
def test_single_char_bracket_range_names_the_field_and_the_value(self) -> None:
"""
GIVEN:
- A SINGLE_CHAR_BRACKET_RANGE diagnostic for "title" with
raw_value "200[1-9]"
WHEN:
- _single_diagnostic_to_error processes it
THEN:
- The resulting SearchQueryError names both the field and the
offending value, with no library prose
"""
diagnostic = Diagnostic(
kind=DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE,
cause=cause_for(DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE),
message=_LIBRARY_PROSE,
field=FieldRef("title"),
field_kind=FieldKind.TEXT,
raw_value="200[1-9]",
)
error = _single_diagnostic_to_error(diagnostic)
message = str(error)
assert isinstance(error, SearchQueryError)
assert _LIBRARY_PROSE not in message
assert "title" in message
assert "200[1-9]" in message
class TestRealQueriesRouteCorrectly:
"""The routing table against diagnostics emit() really produces."""
def test_text_range_is_a_400_naming_the_field(
self,
query_index: tantivy.Index,
) -> None:
"""
GIVEN:
- A real query index
WHEN:
- parse_user_query is called with a text-range query
("title:[a to b]")
THEN:
- It raises SearchQueryError naming "title"
"""
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, "title:[a to b]", UTC)
assert "title" in str(excinfo.value)
def test_wildcard_on_a_numeric_field_is_a_400_naming_the_field(
self,
query_index: tantivy.Index,
) -> None:
"""
GIVEN:
- A real query index
WHEN:
- parse_user_query is called with a wildcard on a numeric
field ("asn:12*")
THEN:
- It raises SearchQueryError naming "asn"
"""
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, "asn:12*", UTC)
assert "asn" in str(excinfo.value)
def test_single_char_bracket_range_is_a_400_naming_field_and_value(
self,
query_index: tantivy.Index,
) -> None:
"""
GIVEN:
- A real query index
WHEN:
- parse_user_query is called with "title:200[1-9]"
THEN:
- It raises SearchQueryError naming both "title" and
"200[1-9]"
"""
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, "title:200[1-9]", UTC)
message = str(excinfo.value)
assert "title" in message
assert "200[1-9]" in message
def test_internal_diagnostic_escapes_as_a_query_error(
self,
query_index: tantivy.Index,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- tantivy_emit monkeypatched to raise a QueryError with an
INTERNAL-cause diagnostic (BACKEND_REJECTED), the one case
with no query text of its own involved
WHEN:
- parse_user_query runs a normal query ("invoice")
THEN:
- The QueryError propagates unconverted; emit() reporting a
defect in itself must not become a user-facing 400
"""
import documents.search._query as query_mod
def raise_internal(*args: object, **kwargs: object) -> None:
raise QueryError(_diagnostic(DiagnosticKind.BACKEND_REJECTED))
monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
with pytest.raises(QueryError):
parse_user_query(query_index, "invoice", UTC)
@@ -0,0 +1,119 @@
"""``field:*`` on a JSON field is user error, not an operator alert.
whoosh-compat classifies EXISTS_REQUIRES_FAST as MISCONFIGURED, and
_map_emit_error used to route every MISCONFIGURED diagnostic to an ERROR log.
But the kind is decided from the registry's own FieldSpec (kind plus fast)
without consulting the index schema, and field_descriptors() builds the JSON
fields non-fast deliberately, so nothing is misconfigured and no operator
action can clear the condition. Any authenticated user could otherwise emit
ERROR lines in a loop by repeating ``notes:*``.
SCHEMA_FIELD_MISSING, the other MISCONFIGURED kind, does compare the registry
against the live schema, so it stays an ERROR log. But it is not a 400
either: the exact same query would succeed once the index is rebuilt, so it
is a transient server-side condition, not a permanently bad request, and is
re-raised the same way an INTERNAL cause is.
"""
from __future__ import annotations
import logging
from datetime import UTC
from typing import TYPE_CHECKING
import pytest
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from whoosh_compat.errors import cause_for
from whoosh_compat.fields import FieldKind
from whoosh_compat.fields import FieldRef
from documents.search._errors import SearchQueryError
from documents.search._query import _map_emit_error
from documents.search._query import parse_user_query
if TYPE_CHECKING:
import tantivy
pytestmark = pytest.mark.search
# Every spelling of "does this JSON field have a value" a user can type.
EXISTS_QUERIES = [
"notes:*",
"notes.note:*",
"notes.user:*",
"custom_fields:*",
"custom_fields.name:*",
"custom_fields.value:*",
]
class TestJsonExistsIsUserError:
@pytest.mark.parametrize("query", EXISTS_QUERIES)
def test_query_is_a_400_that_emits_no_error_log(
self,
query_index: tantivy.Index,
caplog: pytest.LogCaptureFixture,
query: str,
) -> None:
"""
GIVEN:
- A real query index, and every spelling of "does this JSON
field have a value" (notes:*, notes.note:*, custom_fields:*,
etc.)
WHEN:
- parse_user_query runs the query
THEN:
- It raises SearchQueryError naming the field, and no
ERROR-level log record is emitted; EXISTS_REQUIRES_FAST on a
JSON field is by design, not a misconfiguration an operator
could act on
"""
with caplog.at_level(logging.WARNING, logger="paperless.search"):
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, query, UTC)
assert query.split(":", maxsplit=1)[0] in str(excinfo.value)
assert [r for r in caplog.records if r.levelno >= logging.ERROR] == []
class TestGenuineMisconfigurationStillLogs:
def test_schema_field_missing_is_an_error_log_and_reraised(
self,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A QueryError for SCHEMA_FIELD_MISSING: the registry naming a
field the index schema does not have
WHEN:
- _map_emit_error processes it
THEN:
- It logs exactly one ERROR record naming the diagnostic kind
(a real mismatch an operator can fix, so it keeps the
alert), and the original QueryError propagates unchanged
rather than becoming a SearchQueryError: the exact same
query would succeed once the index is rebuilt, so this is a
transient server-side condition, not a permanently bad
request, and surfaces as a 500 rather than a 400
"""
kind = DiagnosticKind.SCHEMA_FIELD_MISSING
error = QueryError(
Diagnostic(
kind=kind,
cause=cause_for(kind),
message="field 'asn' is not defined in the index schema",
field=FieldRef("asn"),
field_kind=FieldKind.U64,
),
)
with (
caplog.at_level(logging.ERROR, logger="paperless.search"),
pytest.raises(QueryError) as excinfo,
):
_map_emit_error(error)
assert excinfo.value is error
records = [r for r in caplog.records if r.levelno == logging.ERROR]
assert len(records) == 1
assert kind.name in records[0].getMessage()
@@ -0,0 +1,223 @@
"""_fuzzy_alternative, the near-match side of the widening hook.
A leaf's words each become a Fuzzy leaf on the leaf's own field, so a
typo in one word of a term still finds the document while everything
around the term keeps constraining it. A Term's words are OR'd for
per-word recall; a Phrase's are AND-ed, because quoting must not widen a
search. CJK words get no fuzzy side at all.
"""
from __future__ import annotations
import pytest
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
from documents.search._query import _fuzzy_alternative
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import TITLE
from documents.tests.search._ast_helpers import content
pytestmark = pytest.mark.search
# 130 characters, past the analyzer's 129-byte remove_long limit (128 is
# kept, 129 is dropped), so the index never holds it and neither side
# should search for it.
_TOO_LONG = "x" * 130
def _fuzzy(field: wc.FieldRef, text: str) -> wc_ast.Fuzzy:
return wc_ast.Fuzzy(field=field, text=text, distance=1, prefix=True)
class TestTheAlternative:
def test_a_single_word_becomes_one_fuzzy_leaf(self) -> None:
"""
GIVEN:
- A one-word term
WHEN:
- The fuzzy alternative is built
THEN:
- It is a single Fuzzy leaf on the same field, distance 1 and
prefix matching, which is what the old clause used
"""
assert _fuzzy_alternative(content("invoice")) == _fuzzy(CONTENT, "invoice")
def test_the_leafs_own_field_is_used(self) -> None:
"""
GIVEN:
- A term fielded on title
WHEN:
- The fuzzy alternative is built
THEN:
- The Fuzzy leaf is on title, not spread across the default
fields. This is the fielding fix: today's clause searches a
fielded word everywhere
"""
assert _fuzzy_alternative(wc_ast.Term(field=TITLE, text="invoice")) == _fuzzy(
TITLE,
"invoice",
)
def test_several_words_are_ored(self) -> None:
"""
GIVEN:
- A hyphenated term the analyzer splits into two words
WHEN:
- The fuzzy alternative is built
THEN:
- The words are OR'd. AND would be stricter than the exact
side, which is OR'd for an unfielded term, and would lose
today's per-word recall inside a term
"""
assert _fuzzy_alternative(content("COVID-19")) == wc_ast.Or(
children=(_fuzzy(CONTENT, "covid"), _fuzzy(CONTENT, "19")),
)
def test_words_are_split_where_the_index_splits_them(self) -> None:
"""
GIVEN:
- A term the analyzer lowercases and folds
WHEN:
- The fuzzy alternative is built
THEN:
- The Fuzzy text is the analyzer's output, not the raw text,
so it is in the same shape as the index terms
"""
assert _fuzzy_alternative(content("Éclair")) == _fuzzy(CONTENT, "eclair")
def test_the_splitter_does_not_stem(self) -> None:
"""
GIVEN:
- A word the stemmer would change
WHEN:
- The fuzzy alternative is built
THEN:
- The word is unstemmed. Fuzzy text goes through the field's
pattern_normalizer, which stems it once; stemming here too
would search for a term the index does not hold
"""
assert _fuzzy_alternative(content("universities")) == _fuzzy(
CONTENT,
"universities",
)
@pytest.mark.parametrize(
"text",
[
pytest.param("x", id="one_character"),
pytest.param("__", id="only_dropped_characters"),
pytest.param(_TOO_LONG, id="past_remove_long"),
pytest.param("", id="empty"),
pytest.param(" ", id="whitespace"),
],
)
def test_text_with_no_usable_word_gets_no_alternative(self, text: str) -> None:
"""
GIVEN:
- Text with no word the index could hold: a single character,
characters the tokenizer drops, a word past remove_long, or
nothing at all
WHEN:
- The fuzzy alternative is built
THEN:
- None, so the leaf is left alone. Returning an alternative
here would collapse Or(leaf, fuzzy) to a required clause
that can never match, making a query return LESS with fuzzy
on than off
"""
assert _fuzzy_alternative(content(text)) is None
def test_a_one_character_word_is_dropped_from_a_longer_term(self) -> None:
"""
GIVEN:
- A term mixing a one-character word with a real one
WHEN:
- The fuzzy alternative is built
THEN:
- Only the real word survives. A one-character prefix fuzzy
term matches every term in the field
"""
assert _fuzzy_alternative(content("h52.1")) == _fuzzy(CONTENT, "h52")
def test_the_leaf_span_is_copied(self) -> None:
"""
GIVEN:
- A leaf with a source span, whose words both survive the
one-character filter so there is an Or to inspect
WHEN:
- The fuzzy alternative is built
THEN:
- Every node it builds carries that span, so an emit-time
diagnostic still points into the query text
"""
leaf = wc_ast.Term(field=CONTENT, text="ab-cd", startchar=4, endchar=9)
alternative = _fuzzy_alternative(leaf)
assert isinstance(alternative, wc_ast.Or)
spans = {
(node.startchar, node.endchar)
for node in (alternative, *alternative.children)
}
assert spans == {(4, 9)}
class TestPhrasesAreNotWidened:
def test_a_multi_word_phrase_requires_every_word(self) -> None:
"""
GIVEN:
- A quoted two-word phrase
WHEN:
- Its fuzzy alternative is built
THEN:
- The words are And-combined, where a Term's are Or-combined
(test_a_hyphenated_term_matches_on_one_word pins that). The
parser's default group is And, so an Or here would let the
quoted phrase match strictly more than the same two words
unquoted
"""
leaf = wc_ast.Phrase(field=CONTENT, text="tax report")
assert _fuzzy_alternative(leaf) == wc_ast.And(
children=(_fuzzy(CONTENT, "tax"), _fuzzy(CONTENT, "report")),
)
class TestCjkGetsNoFuzzySide:
@pytest.mark.parametrize(
"text",
[
pytest.param("東京", id="cjk_only"),
pytest.param("東京都の報告書", id="longer_run"),
pytest.param("서울", id="hangul"),
],
)
def test_a_cjk_word_is_skipped(self, text: str) -> None:
"""
GIVEN:
- A CJK term
WHEN:
- Its fuzzy alternative is built
THEN:
- There is none. The content analyzer keeps the run as one
token, so a prefix Fuzzy over it would match any run within
one edit of its start, which is the "東京都 matches 京都"
failure the bigram fields exist to avoid
"""
assert _fuzzy_alternative(content(text)) is None
def test_latin_beside_cjk_still_gets_its_fuzzy_side(self) -> None:
"""
GIVEN:
- A term mixing a CJK run and a latin word
WHEN:
- Its fuzzy alternative is built
THEN:
- Only the latin word is fuzzed. Skipping CJK words must not
cost the latin half its near-match
"""
leaf = content("東京 report")
assert _fuzzy_alternative(leaf) == _fuzzy(CONTENT, "report")
@@ -0,0 +1,134 @@
"""What survives ADVANCED_FUZZY_SEARCH_THRESHOLD.
The rest of the fuzzy tests run at threshold 0.0 so they see matching
behavior rather than the filter. These run at 0.5, the value the docs
recommend, because the filter has properties of its own that nothing else
covers: a near-miss must not be cut according to how well the query's
correctly spelled words matched, and an exact match must outrank one that
is only partly exact.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# Long enough that BM25 scores it well below a short document holding the
# same word, which is what makes the spread these tests are about.
_PADDING = "lorem ipsum dolor sit amet " * 40
@pytest.fixture(autouse=True)
def _threshold(settings: SettingsWrapper) -> None:
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
class TestNearMissesAreNotCutByTheSpread:
def test_two_equal_near_misses_both_survive(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two documents that both contain a rare correctly spelled
word and both near-match a misspelled one. One has the rare
word in its title, the other buried in a long body, so BM25
scores them very differently
WHEN:
- Both words are searched at threshold 0.5
THEN:
- Both survive. Neither is a better answer than the other:
they differ only in where the correctly spelled word sits.
If the widened tree were scored normally, the fuzzy side
would inherit the BM25 spread of the correct word and the
weaker document would be cut
"""
titled = index_document(
title="Zarquon",
content=f"invoices {_PADDING}",
)
buried = index_document(
title="Nothing",
content=f"{_PADDING} zarquon invoicing",
)
assert matched_ids("zarquon invoce") == {titled.pk, buried.pk}
def test_near_misses_still_rank_among_themselves(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
) -> None:
"""
GIVEN:
- A document near-matching the query in several fields and one
near-matching it in a single field
WHEN:
- The misspelled word is searched
THEN:
- Both survive, and the multi-field one ranks first. Flat
scoring alone would tie every near-miss and lose relevance
ordering entirely, which is what the tiebreak clause exists
to prevent
"""
strong = index_document(
title="Invoices",
content="invoices for invoicing",
)
weak = index_document(title="Nothing", content="invoices")
hits = backend.search_ids("invoce", user=None)
assert set(hits) == {strong.pk, weak.pk}
assert hits[0] == strong.pk
class TestExactOutranksNearMiss:
def test_a_fully_exact_document_outranks_a_partly_exact_one(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
) -> None:
"""
GIVEN:
- A document matching both query words exactly but buried in a
long body, and one matching the first word exactly several
times while only near-matching the second
WHEN:
- Both words are searched, spelled correctly
THEN:
- The fully exact document ranks first. docs/configuration.md
promises fuzzy results rank below exact matches, and the
widened tree alone would break it: a partly exact document
collects real BM25 for what it did match plus a constant for
what it only near-matched
Only the ordering is asserted, not that the partly exact document
survives the threshold. Spec 4.2a accepts that a large enough BM25
spread still cuts a near-miss, so whether it survives depends on
the corpus and is not a property to pin.
"""
index_document(
title="Invoice Invoice Invoice",
content="reportage weekly",
)
fully = index_document(
title="Nothing",
content=f"{_PADDING} invoice report",
)
hits = backend.search_ids("invoice report", user=None)
assert hits[0] == fully.pk
@@ -0,0 +1,211 @@
"""The words a leaf contributes to its fuzzy alternative.
Each leaf is widened in the tree now, so nothing is re-parsed as a string
and a boolean keyword can no longer be read as grammar. What still has to
hold is that a word is stemmed exactly once (analysis is not idempotent)
and that hyphenated, dotted and quoted terms keep contributing their
words.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
pytestmark = [
pytest.mark.search,
pytest.mark.django_db,
pytest.mark.usefixtures("fuzzy_enabled"),
]
class TestFuzzyClauseWords:
def test_a_stemmed_word_is_not_stemmed_a_second_time(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Documents whose content contains "universities", a
one-transposition typo of it ("universties"), and two
unrelated words that share its stem prefix ("univalent",
"unicycle")
WHEN:
- Searching for "universities" with the fuzzy blend enabled
THEN:
- Only the correctly-spelled document and its typo match; the
clause does not widen far enough to reach the unrelated
words. 'universities' stems to 'univers'; feeding that back
to tantivy would stem it again to 'univ', whose fuzzy prefix
reaches unrelated words - the clause must stay wide enough
for a typo and no wider
"""
wanted = index_document(
title="A",
content="universities of europe",
)
typo = index_document(
title="B",
content="universties of europe",
)
index_document(
title="C",
content="univalent chemical bonds",
)
index_document(
title="D",
content="unicycle repair manual",
)
assert matched_ids("universities") == {wanted.pk, typo.pk}
def test_a_hyphenated_term_still_reaches_the_clause(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose content contains a near-miss of "COVID-19"
("covidx")
WHEN:
- Searching for "COVID-19" with the fuzzy blend enabled
THEN:
- The document matches; 'COVID-19' is one raw token, so unless
it is split into words, it carries characters the re-parse
would read as grammar, is dropped, and the whole query loses
its fuzzy clause
"""
misspelled = index_document(
title="A",
content="covidx testing results",
)
assert matched_ids("COVID-19") == {misspelled.pk}
def test_a_phrase_still_reaches_the_clause(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose content near-misses a quoted phrase
WHEN:
- Searching for the quoted phrase '"tax reports"' with the
fuzzy blend enabled
THEN:
- The document matches; a phrase is one raw token carrying a
space, and is the whole query's only free text here, so it
must still reach the clause
"""
near_miss = index_document(
title="A",
content="taxation reportage weekly",
)
assert matched_ids('"tax reports"') == {near_miss.pk}
class TestBooleanKeywordsInRawText:
"""Tantivy's boolean keywords used to reach its parser with their case
intact, through the word string the old clause was re-parsed from, so a
quoted phrase could smuggle grammar in. Leaves are built as AST nodes
now, which closes that off structurally; these pin it shut."""
@pytest.fixture
def corpus(self, index_document: Callable[..., Document]) -> dict[str, int]:
both = index_document(
title="A",
content="taxation reportage weekly",
)
tax_only = index_document(
title="B",
content="taxation only here",
)
report_only = index_document(
title="C",
content="reportage only here",
)
return {
"both": both.pk,
"tax_only": tax_only.pk,
"report_only": report_only.pk,
}
@pytest.mark.parametrize(
("keyword_spelling", "ordinary_spelling"),
[
pytest.param('"tax AND reports"', '"tax and reports"', id="and"),
pytest.param('"tax OR reports"', '"tax or reports"', id="or"),
pytest.param('"tax NOT reports"', '"tax not reports"', id="not"),
pytest.param('"tax IN reports"', '"tax in reports"', id="in"),
],
)
def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(
self,
matched_ids: Callable[[str], set[int]],
corpus: dict[str, int],
keyword_spelling: str,
ordinary_spelling: str,
) -> None:
"""
GIVEN:
- Three documents: one with both "taxation" and "reportage",
one with only "taxation", one with only "reportage"
WHEN:
- A quoted phrase carries a tantivy boolean keyword as one of
its words, spelled in upper case and in lower case
THEN:
- Both spellings match the same documents, so the keyword is
an ordinary word of the phrase rather than grammar: AND does
not make it a conjunction, NOT does not give it its own
exclusion, IN does not fail the parse. Only the upper-case
spelling was ever grammar
"""
assert matched_ids(keyword_spelling) == matched_ids(ordinary_spelling)
def test_a_phrase_needs_a_near_match_for_every_word(
self,
matched_ids: Callable[[str], set[int]],
corpus: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents: one with both "taxation" and "reportage",
one with only "taxation", one with only "reportage"
WHEN:
- '"tax reports"' is searched, both words misspelled
THEN:
- Only the document near-matching both words comes back. A
quoted phrase asks for more than the bare words, so its
fuzzy side requires every one of them
"""
assert matched_ids('"tax reports"') == {corpus["both"]}
def test_a_trailing_keyword_is_just_a_word(
self,
matched_ids: Callable[[str], set[int]],
corpus: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents: one with both "taxation" and "reportage",
one with only "taxation", one with only "reportage"
WHEN:
- '"tax AND"' is searched, a phrase that used to be a tantivy
syntax error once the clause was re-parsed as a string
- The same phrase is searched with the keyword in lower case
THEN:
- Both match the same documents, and neither raises. Nothing
is re-parsed any more, so a trailing keyword cannot cost the
query its fuzzy side
"""
assert matched_ids('"tax AND"') == matched_ids('"tax and"')
@@ -0,0 +1,467 @@
"""Fuzzy matching applied inside the parsed query rather than beside it.
With a threshold set, every leaf gains a near-match alternative where it
sits, so fielding, negation, AND and positive filters all constrain the
fuzzy match exactly as they constrain the exact one. The old clause was a
flat bag of words OR'd in at the top level, which none of them reached.
Threshold is 0.0 here so these tests see the matching behavior, not the
score filter. The filter has its own file, test_fuzzy_scoring.py.
"""
from __future__ import annotations
import datetime
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
from documents.models import StoragePath
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@pytest.fixture(autouse=True)
def _fuzzy_on(settings: SettingsWrapper) -> None:
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
class TestStructureIsHonoured:
@pytest.fixture
def near_matches_both(
self,
index_document: Callable[..., Document],
) -> Document:
"""One document near-matching both query words, one only the first."""
both = index_document(title="A", content="invoices report")
index_document(title="B", content="invoices only")
return both
def test_a_fielded_term_fuzzes_only_that_field(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- One document with a near-miss of the word in its title, one
with a near-miss in its content only
WHEN:
- "title:invoce" is searched
THEN:
- Only the title document matches. The old clause searched a
fielded word across every default field
"""
titled = index_document(title="Invoces", content="nothing")
index_document(title="Nothing", content="invoces here")
assert matched_ids("title:invoce") == {titled.pk}
def test_every_word_needs_a_near_match(
self,
near_matches_both: Document,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document near-matching both words, and one near-matching
only the first
WHEN:
- "invoce reprot" is searched (both words misspelled)
THEN:
- Only the document near-matching both survives. The old
clause OR'd the words, so anything near one of them matched
"""
assert matched_ids("invoce reprot") == {near_matches_both.pk}
def test_a_negated_word_is_not_fuzzed(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document containing "taxi" and one containing "tax"
WHEN:
- "invoice NOT tax" is searched
THEN:
- The "taxi" document survives and the "tax" one does not. A
negated leaf keeps its exact side only: fuzzing it with
prefix matching would exclude every word starting near it
"""
taxi = index_document(title="A", content="invoice taxi fare")
index_document(title="B", content="invoice tax return")
assert matched_ids("invoice NOT tax") == {taxi.pk}
def test_a_negation_inside_a_branch_still_binds(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- An "invoice" document containing "secret", one without, and
a "bill" document
WHEN:
- "(invoce AND NOT secret) OR bill" is searched
THEN:
- The secret document stays out. The old clause restated only
top-level conjunctive exclusions, so an Or branch's NOT was
never applied to the fuzzy side
"""
index_document(title="A", content="invoices secret")
clean = index_document(title="B", content="invoices only")
bill = index_document(title="C", content="bill")
assert matched_ids("(invoce AND NOT secret) OR bill") == {
clean.pk,
bill.pk,
}
def test_a_structured_filter_constrains_the_fuzzy_match(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two near-miss documents created in different years
WHEN:
- "created:2024 invoce" is searched
THEN:
- Only the 2024 one matches. The old blend restated negations
above the fuzzy clause but never positive constraints, so a
filter did not reach the fuzzy side at all.
created is the right field to test with: it is a DATE field,
so it has no fuzzy side of its own and cannot be widened.
type: would not test this, because it is an alias for
document_type, which is one of the five default search
fields and so gets widened like any other leaf
"""
matching = index_document(
title="A",
content="invoices",
created=datetime.date(2024, 6, 1),
)
index_document(
title="B",
content="invoices",
created=datetime.date(2023, 6, 1),
)
assert matched_ids("created:2024 invoce") == {matching.pk}
def test_a_filter_on_a_non_default_field_constrains_the_fuzzy_match(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two near-miss documents with different storage paths
WHEN:
- "path:archive invoce" is searched
THEN:
- Only the matching one comes back. storage_path is a TEXT
field that is NOT one of the default search fields, so it
gets no fuzzy side; the release note promises this case and
the date test above does not cover it
"""
archive = StoragePath.objects.create(name="archive", path="archive/{title}")
other = StoragePath.objects.create(name="misc", path="misc/{title}")
matching = index_document(
title="A",
content="invoices",
storage_path=archive,
)
index_document(
title="B",
content="invoices",
storage_path=other,
)
assert matched_ids("path:archive invoce") == {matching.pk}
def test_a_require_filters_on_the_widened_side_too(
self,
near_matches_both: Document,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document near-matching both words and one near-matching
only the scored side
WHEN:
- "invoce REQUIRE reprot" is searched
THEN:
- Only the document near-matching both comes back. The
filter-only side is unscored but still filters, and it is
widened like any other leaf, so a near miss satisfies it
"""
assert matched_ids("invoce REQUIRE reprot") == {near_matches_both.pk}
class TestRecallIsKept:
def test_a_typo_still_finds_its_document(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A correctly spelled document
WHEN:
- A one-edit misspelling is searched
THEN:
- It matches. This is what the whole feature is for
"""
doc = index_document(title="A", content="invoice total")
assert matched_ids("invoce") == {doc.pk}
def test_a_hyphenated_term_matches_on_one_word(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document holding a near-miss of one half of a hyphenated
term
WHEN:
- "COVID-19" is searched
THEN:
- It matches. The words inside a term are OR'd on the fuzzy
side, which is the per-word recall the old clause had
"""
doc = index_document(title="A", content="covidx cases")
assert matched_ids("COVID-19") == {doc.pk}
def test_a_word_the_index_cannot_hold_does_not_narrow_the_query(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- An "invoice" document
WHEN:
- "invoice __" is searched, where __ is characters the
tokenizer discards entirely
THEN:
- It still matches. If such a word got a fuzzy alternative,
the leaf would collapse to a required clause that can never
match and the query would return less with fuzzy on than off
"""
doc = index_document(title="A", content="invoice total")
assert matched_ids("invoice __") == {doc.pk}
def test_a_one_character_word_no_longer_matches_everything(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two unrelated documents
WHEN:
- "x invoice" is searched, where x is a one-character word no
document holds
THEN:
- Nothing matches. One-character words get no fuzzy side, and
the exact side requires a term no document has. Today this
query matches the whole corpus, because a one-character
prefix fuzzy term matches every term in the field
"""
index_document(title="A", content="invoice total")
index_document(title="B", content="unrelated")
assert matched_ids("x invoice") == set()
class TestCjkAndFuzzyTogether:
def test_a_cjk_term_keeps_its_bigram_side_with_fuzzy_on(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a CJK term inside an unspaced run
WHEN:
- That term is searched with fuzzy on
THEN:
- It matches through the bigram side, which sits in the same
Or as the fuzzy side and is unaffected by it
"""
doc = index_document(
title="A",
content="東京都の公共文書について",
)
assert matched_ids("東京") == {doc.pk}
def test_a_negated_cjk_term_keeps_its_bigram_side(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- An "invoice" document whose only 東京 is inside a run, and
one with no 東京 at all
WHEN:
- "invoice NOT 東京" is searched with fuzzy on
THEN:
- The 東京 document is excluded. A negated leaf loses its
fuzzy side but keeps its CJK one, so the exclusion still
reaches inside the run
"""
index_document(title="A", content="invoice 東京都の報告書")
clean = index_document(title="B", content="invoice only")
assert matched_ids("invoice NOT 東京") == {clean.pk}
class TestTheHookIsSkipped:
def test_a_plain_query_passes_no_hook(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings: SettingsWrapper,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- Fuzzy off and a query with no CJK in it
WHEN:
- It is parsed
THEN:
- emit() is called once, with rewrite_leaf=None. Nothing is
widened, so the query is exactly what it was before any of
this work
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = None
from documents.search import _query
calls: list[object] = []
real = _query.tantivy_emit
def spy(*args: object, **kwargs: object) -> object:
calls.append(kwargs.get("rewrite_leaf"))
return real(*args, **kwargs)
monkeypatch.setattr(_query, "tantivy_emit", spy)
index_document(title="A", content="invoice")
matched_ids("invoice")
assert calls == [None]
class TestTheEmitRegistryIsInvisible:
def test_a_non_cjk_query_gives_the_same_result_under_either_registry(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- A plain latin document and a non-CJK query with fuzzy on,
which now selects the registry carrying the internal bigram
fields even though nothing in the query is CJK
WHEN:
- The same query runs with that selection forced back to the
public registry
THEN:
- The same documents come back. The bigram fields are absent
from PUBLIC_FIELDS and the two registries agree on every
field a query can name, so adding them changes nothing for
a query that never reaches them
"""
from documents.search import _query
from documents.search._registry import get_field_registry
index_document(title="A", content="invoice total")
index_document(title="B", content="unrelated")
with_bigram_fields = matched_ids("invoce")
monkeypatch.setattr(
_query,
"_get_emit_field_registry",
lambda language: get_field_registry(language),
)
assert matched_ids("invoce") == with_bigram_fields
class TestQuotingDoesNotWiden:
@pytest.fixture
def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
"""One document per word, and one holding both."""
return {
"tax": index_document(title="A", content="tax invoice").pk,
"report": index_document(title="B", content="report invoice").pk,
"both": index_document(title="C", content="tax report").pk,
}
@pytest.mark.parametrize(
"query",
[
pytest.param("tax report", id="unquoted"),
pytest.param('"tax report"', id="quoted"),
],
)
def test_a_quoted_phrase_needs_every_word_like_the_bare_words(
self,
matched_ids: Callable[[str], set[int]],
pks: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- A document per word, and one holding both
WHEN:
- The words are searched unquoted and as a quoted phrase
THEN:
- Only the document holding both matches, either way. With
the phrase's words Or'd on the fuzzy side, the quoted form
matched every document, including near-misses of one word
"""
assert matched_ids(query) == {pks["both"]}
class TestCjkIsNotFuzzed:
def test_a_cjk_term_does_not_match_a_run_sharing_its_start(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document holding 東京都 and one holding only 京都, each
inside a longer unspaced run
WHEN:
- "東京" is searched with fuzzy on
THEN:
- Only the 東京都 document matches. A prefix Fuzzy over the
whole run would match the 京都 document too, undoing what
the bigram fields' multitoken=AND guarantees with fuzzy off
"""
tokyo = index_document(title="A", content="東京都の報告書")
index_document(title="B", content="京都の観光案内について")
assert matched_ids("東京") == {tokyo.pk}
@@ -0,0 +1,238 @@
"""Regression coverage for the unguarded TEXT-mode highlight query.
parse_simple_text_highlight_query re-parses simple-search tokens through
Tantivy's query-string parser to build a SnippetGenerator-compatible query.
Simple-search tokens keep arbitrary punctuation (quotes, colons, brackets,
slashes), so any token carrying Tantivy query grammar raised an unguarded
ValueError. The search itself had already succeeded by the time this ran:
only the highlight step failed, and with the DocumentViewSet.list
exception handler narrowed elsewhere on this branch, that ValueError now
reaches the client as a bare 500 rather than a 400.
Covers three angles:
- the query builder itself: quoting each token as its own escaped phrase
should let it parse instead of raising, for every failure mode a plain-
text query can trigger (syntax error, unknown field, unsupported regex).
- highlight_hits: even when a token still can't be expressed as a
highlight query, the guard must fall back to a query that still
produces usable highlight HTML, not silently empty ones.
- the real API endpoint: pinning the previously-500 status to 200.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
import tantivy
from rest_framework import status
from documents.search._backend import SearchMode
from documents.search._query import parse_simple_text_highlight_query
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# Each spelling below trips a different Tantivy parser failure mode:
# 'a"b' -> Syntax Error (unterminated quote)
# foo:bar -> unknown field
# (a -> Syntax Error (unbalanced group)
# [a -> Syntax Error (unbalanced range)
# /a/ -> Unsupported query (regex queries disallowed)
_MALFORMED_QUERIES = [
pytest.param('a"b', id="unterminated_quote"),
pytest.param("foo:bar", id="unknown_field"),
pytest.param("(a", id="unbalanced_group"),
pytest.param("[a", id="unbalanced_range"),
pytest.param("/a/", id="unsupported_regex"),
]
class TestParseSimpleTextHighlightQueryDoesNotRaise:
"""The query builder itself must tolerate Tantivy syntax in its tokens."""
@pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
def test_malformed_token_does_not_raise(
self,
query_index: tantivy.Index,
raw_query: str,
) -> None:
"""
GIVEN:
- A simple-search query token carrying Tantivy query grammar
(unterminated quote, unknown field, unbalanced group/range,
or unsupported regex)
WHEN:
- parse_simple_text_highlight_query builds a highlight query
from it
THEN:
- It returns a tantivy.Query instead of raising, since each
token is quoted as its own escaped phrase rather than fed
to the parser raw
"""
assert isinstance(
parse_simple_text_highlight_query(query_index, raw_query),
tantivy.Query,
)
class TestHighlightHitsProducesUsableHighlights:
"""highlight_hits must keep producing real <b>-wrapped snippet HTML for
these queries, not merely avoid raising."""
@pytest.mark.parametrize(
"raw_query",
[*_MALFORMED_QUERIES, pytest.param("plain text", id="plain_text_sanity")],
)
def test_highlight_still_contains_matched_text(
self,
backend: TantivyBackend,
raw_query: str,
) -> None:
"""
GIVEN:
- A document whose content contains the raw query text
verbatim
WHEN:
- backend.highlight_hits builds highlights for a TEXT-mode
search using that same (possibly Tantivy-grammar-carrying)
query text
THEN:
- The hit still carries a content highlight with real
<b>-wrapped matched-term markup, not an empty fallback
"""
doc = DocumentFactory.create(
title="probe",
content=f"needle content containing {raw_query} literally here",
)
backend.add_or_update(doc)
hits = backend.highlight_hits(
raw_query,
[doc.pk],
search_mode=SearchMode.TEXT,
)
assert len(hits) == 1
highlights = hits[0]["highlights"]
assert "content" in highlights, (
f"Expected a content highlight for {raw_query!r}, got: {highlights!r}"
)
assert "<b>" in highlights["content"], (
f"Highlight for {raw_query!r} carries no matched-term markup: "
f"{highlights['content']!r}"
)
class TestHighlightGuardDiscriminatesOnValueError:
"""The guard added to highlight_hits must catch exactly ValueError, the
same shape as the sibling notes_text guard, and let anything else
through -- so a real library defect is never mistaken for a harmless
syntax error."""
def test_non_value_error_is_not_swallowed(
self,
backend: TantivyBackend,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- parse_simple_text_highlight_query patched to raise
RuntimeError instead of a syntax-related ValueError
WHEN:
- backend.highlight_hits is called
THEN:
- The RuntimeError propagates unguarded; the highlight guard
must catch exactly ValueError, the same shape as the
sibling notes_text guard, never mistaking a real library
defect for a harmless syntax error
"""
import documents.search._backend as backend_mod
def raise_runtime_error(*args: object, **kwargs: object) -> object:
raise RuntimeError("synthetic bug, unrelated to query syntax")
monkeypatch.setattr(
backend_mod,
"parse_simple_text_highlight_query",
raise_runtime_error,
)
doc = DocumentFactory.create(title="probe", content="anything here")
backend.add_or_update(doc)
with pytest.raises(RuntimeError):
backend.highlight_hits(
"anything",
[doc.pk],
search_mode=SearchMode.TEXT,
)
@pytest.mark.usefixtures("_search_index")
class TestApiNoLongerReturns500:
"""Pins the actual regression: a matching TEXT-mode search whose query
string carries Tantivy syntax must return results, not a server error."""
@pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
def test_malformed_text_query_returns_200(
self,
admin_client: APIClient,
raw_query: str,
) -> None:
"""
GIVEN:
- A matching document whose content contains the raw query
text, indexed via the real search index fixture
WHEN:
- A TEXT-mode search is issued through the real API with a
query string carrying Tantivy syntax
THEN:
- The response is 200 with the expected result count, not a
500 (the regression this file exists to pin)
"""
from documents.search import get_backend
doc = DocumentFactory.create(
title="probe",
content=f"needle content containing {raw_query} literally here",
)
get_backend().add_or_update(doc)
response = admin_client.get(f"/api/documents/?text={raw_query}")
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
def test_plain_text_query_still_returns_200(
self,
admin_client: APIClient,
) -> None:
"""
GIVEN:
- A matching document indexed via the real search index
fixture
WHEN:
- An ordinary TEXT-mode search (no Tantivy syntax) is issued
THEN:
- The response is 200 with the expected result count; sanity
check that the guard does not mask a total failure of the
ordinary highlight path
"""
from documents.search import get_backend
doc = DocumentFactory.create(
title="probe",
content="needle content containing plain text literally here",
)
get_backend().add_or_update(doc)
response = admin_client.get("/api/documents/?text=plain text")
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
@@ -0,0 +1,180 @@
"""Bare notes:/custom_fields: prefix resolution.
"notes:foo"/"custom_fields:foo" were valid fielded searches before the
whoosh-compat migration. The registry only exposes them as JSON subpaths, so
each JSON FieldSpec declares a default subpath (SubpathSpec(default=True)):
notes: resolves to notes.note:, custom_fields: resolves to
custom_fields.value:. This replaced an earlier regex-based rewrite
(_rewrite_bare_json_field_prefixes) that ran on the raw query string before
parsing and was blind to quoting, so a phrase like
content:"payment notes: none" was silently corrupted into a notes-field
search and matched nothing. Resolving the default subpath inside the parser
instead means quoting is already understood by the time it happens.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Note
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestBareJsonFieldPrefixes:
def test_bare_notes_prefix_searches_note_text(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a note whose text contains a word, and a
decoy document whose content (not notes) contains the same
word
WHEN:
- A bare "notes:" prefix query is run (notes declares "note"
as its default subpath)
THEN:
- Only the document whose note matches is returned; the
decoy's content match does not resurface through a demoted
text search
"""
alice = UserFactory(username="alice")
with_note = DocumentFactory(title="Has note", content="x")
Note.objects.create(document=with_note, user=alice, note="crocodile")
backend.add_or_update(with_note)
# This document's CONTENT contains the words a demoted text search
# would match; it must NOT match once the prefix addresses notes.
index_document(
title="Notes about things",
content="notes crocodile mention",
)
assert matched_ids("notes:crocodile") == {with_note.pk}
def test_bare_custom_fields_prefix_searches_values(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a custom field instance whose value
contains a word, and a decoy document whose content (not a
custom field value) contains the same word
WHEN:
- A bare "custom_fields:" prefix query is run (custom_fields
declares "value" as its default subpath)
THEN:
- Only the document whose custom field value matches is
returned
"""
field = CustomField.objects.create(
name="Policy Number",
data_type=CustomField.FieldDataType.STRING,
)
with_value = DocumentFactory(title="Has field", content="x")
CustomFieldInstance.objects.create(
document=with_value,
field=field,
value_text="crocodile",
)
backend.add_or_update(with_value)
index_document(
title="Custom things",
content="custom fields crocodile",
)
assert matched_ids("custom_fields:crocodile") == {with_value.pk}
def test_subpath_spellings_are_untouched(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document with a note carrying both an author and note text
WHEN:
- The explicit subpath spellings "notes.user:" and
"notes.note:" are queried
THEN:
- Both resolve to their intended subpath and match the
document; the default-subpath resolution for the bare
prefix does not interfere with explicit subpath addressing
"""
bob = UserFactory(username="bob")
doc = DocumentFactory(title="Bob note", content="x")
Note.objects.create(document=doc, user=bob, note="remark")
backend.add_or_update(doc)
assert matched_ids("notes.user:bob") == {doc.pk}
assert matched_ids("notes.note:remark") == {doc.pk}
class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
"""The regex rewrite this migration removes was blind to quoting: it
matched "notes:" anywhere in the raw query string, including inside an
already-quoted phrase on an unrelated field, silently turning
content:"payment notes: none" into a notes-field search that matched
nothing. Resolving the default subpath during parsing (which is
quote-aware) fixes this."""
def test_quoted_phrase_with_notes_colon_matches_by_content(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose content literally contains the text
"payment notes: none" inside a quoted phrase
WHEN:
- A query quoting that exact phrase against the content
field is run
THEN:
- It matches by content, rather than the "notes:" substring
inside the quotes being corrupted into a notes-field search
that matches nothing (the bug the deleted regex rewrite
caused, since it was blind to quoting)
"""
target = index_document(
title="Statement",
content="payment notes: none",
)
assert matched_ids('content:"payment notes: none"') == {target.pk}
def test_quoted_phrase_matches_the_same_document_unquoted(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document whose content contains the same words as the
previous test's phrase, but without the colon
WHEN:
- A query quoting that phrase against the content field is
run
THEN:
- It matches by content, proving the earlier fix is about
quote-awareness specifically, not about the words
themselves being unsearchable
"""
target = index_document(
title="Statement",
content="payment notes none",
)
assert matched_ids('content:"payment notes none"') == {target.pk}
@@ -0,0 +1,95 @@
"""Every declared JSON subpath must actually be written to the index.
PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` ->
{"user", "note"}), but nothing coupled that declaration to what
``_backend.py``'s document builder actually writes into the JSON blob at
index time. A subpath declared but never written would be
queryable-but-always-empty -- syntactically valid, silently matching
nothing -- with no test failure anywhere.
This indexes one real document carrying values for every JSON field
(a Note, a CustomFieldInstance, a DocumentBarcode) and inspects the document's own stored
JSON payload, rather than running field-specific queries: that way a
future JSON field's subpaths are covered automatically, without a new
per-subpath query having to be added by hand each time.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
import tantivy
from whoosh_compat import FieldKind
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.search._fields import PUBLIC_FIELDS
from paperless_testing.factories import DocumentBarcodeFactory
from paperless_testing.factories import UserFactory
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestJsonSubpathsAreWrittenAtIndexTime:
def test_every_declared_json_subpath_appears_in_the_stored_document(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with a Note, a CustomFieldInstance and a
DocumentBarcode attached
WHEN:
- The document is indexed via TantivyBackend.add_or_update
THEN:
- Every subpath PUBLIC_FIELDS declares for notes/custom_fields/barcodes
is present as a key in the document's stored JSON payload
"""
user = UserFactory(username="completeness-user")
field = CustomField.objects.create(
name="Completeness Field",
data_type=CustomField.FieldDataType.STRING,
)
doc = Document.objects.create(
title="Completeness doc",
content="x",
checksum="json-subpath-completeness",
)
Note.objects.create(document=doc, user=user, note="a note")
CustomFieldInstance.objects.create(
document=doc,
field=field,
value_text="a value",
)
DocumentBarcodeFactory(document=doc, value="a barcode")
backend.add_or_update(doc)
index = backend._index
searcher = index.searcher()
hits = searcher.search(
tantivy.Query.term_query(index.schema, "id", doc.pk),
limit=1,
).hits
assert hits, "the document was not indexed"
stored = searcher.doc(hits[0][1]).to_dict()
json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON]
assert json_fields, "no JSON fields declared - fixture is stale"
for field_spec in json_fields:
stored_values = stored.get(field_spec.name)
assert stored_values, (
f"{field_spec.name} was not written to the index at all"
)
written_keys = stored_values[0].keys()
for subpath in field_spec.subpaths:
assert subpath in written_keys, (
f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS "
"but _backend.py's document builder never writes it - it "
"would be queryable but always empty"
)
@@ -0,0 +1,62 @@
"""Wildcard patterns on KEYWORD fields must stay literal.
``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer,
so its terms are never lowercased, folded or stemmed. Running its wildcard
patterns through the stemming normalizer rewrote hex prefixes ("ceded" ->
"cede") and returned documents whose checksum did not start with what the user
typed, which for an identity field is a wrong answer.
This covers only the registry-level normalizer, which is all that exists to
prove at this point in the stack: user queries are not yet routed through
whoosh-compat (that lands with the query-layer PR), so the same fact proven
end to end against real indexed documents lives in
``test_checksum_prefix_queries.py``.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.search._registry import get_field_registry
if TYPE_CHECKING:
from whoosh_compat import FieldRegistry
from whoosh_compat import PatternNormalizer
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _normalizer(registry: FieldRegistry, name: str) -> PatternNormalizer:
ref = registry.make_ref(name)
assert ref is not None
resolved = registry.resolve(ref)
assert resolved is not None
assert resolved.spec.pattern_normalizer is not None
return resolved.spec.pattern_normalizer
class TestKeywordPatternNormalizer:
@pytest.mark.parametrize(
"run",
[
pytest.param("ceded", id="stems_to_cede"),
pytest.param("added", id="stems_to_ad"),
pytest.param("cafed", id="stems_to_cafe"),
],
)
def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None:
"""
GIVEN:
- The "checksum" field's registered pattern normalizer
(KEYWORD kind, "en" registry)
WHEN:
- A wildcard pattern run is normalized
THEN:
- The run is returned unchanged, never widened to a stem (which
would return checksums that do not start with what the user
typed)
"""
normalize = _normalizer(get_field_registry("en"), "checksum")
assert normalize(run) == run
@@ -15,7 +15,7 @@ from documents.search._backend import SearchIndexLockError
from documents.search._backend import TantivyBackend
from documents.tasks import index_document
from documents.tasks import remove_document_from_index
from documents.tests.factories import DocumentFactory
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Generator
@@ -1,6 +1,6 @@
import pytest
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
pytestmark = pytest.mark.search
@@ -0,0 +1,203 @@
"""_negated_leaf_ids, the pre-scan that tells the widening hook which
leaves sit under a negation.
A negated leaf keeps its CJK alternative but gets no fuzzy one: with
prefix matching, NOT tax would otherwise exclude "taxi" and "taxonomy",
which is not what excluding a word means. The hook cannot see a leaf's
context, so the tree is walked once up front and the hook compares by
identity.
"""
from __future__ import annotations
import pytest
import whoosh_compat.ast as wc_ast
from documents.search._query import _negated_leaf_ids
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import content
pytestmark = pytest.mark.search
class TestCollection:
def test_a_bare_tree_has_no_negated_leaves(self) -> None:
"""
GIVEN:
- A tree with no negation in it
WHEN:
- It is pre-scanned
THEN:
- The set is empty
"""
tree = wc_ast.And(children=(content("invoice"), content("report")))
assert _negated_leaf_ids(tree) == frozenset()
def test_a_leaf_under_not_is_collected(self) -> None:
"""
GIVEN:
- A term under a Not
WHEN:
- The tree is pre-scanned
THEN:
- That leaf's id is collected, and the positive one is not
"""
positive = content("invoice")
negated = content("secret")
tree = wc_ast.And(children=(positive, wc_ast.Not(child=negated)))
ids = _negated_leaf_ids(tree)
assert id(negated) in ids
assert id(positive) not in ids
def test_the_negative_side_of_andnot_is_collected(self) -> None:
"""
GIVEN:
- An AndNot, whose positive and negative sides are both terms
WHEN:
- The tree is pre-scanned
THEN:
- Only the negative side is collected
"""
positive = content("invoice")
negated = content("secret")
tree = wc_ast.AndNot(positive=positive, negative=negated)
ids = _negated_leaf_ids(tree)
assert id(negated) in ids
assert id(positive) not in ids
def test_every_leaf_beneath_a_negation_is_collected(self) -> None:
"""
GIVEN:
- A Not whose child is a group of several leaves, one of them
a Phrase
WHEN:
- The tree is pre-scanned
THEN:
- All of them are collected, at any depth and whatever the
leaf type
"""
a = content("alpha")
b = content("beta")
phrase = wc_ast.Phrase(field=CONTENT, text="gamma delta")
tree = wc_ast.Not(
child=wc_ast.Or(
children=(a, wc_ast.And(children=(b, phrase))),
),
)
assert _negated_leaf_ids(tree) == frozenset({id(a), id(b), id(phrase)})
def test_a_double_negation_stays_collected(self) -> None:
"""
GIVEN:
- A term under two nested Nots
WHEN:
- The tree is pre-scanned
THEN:
- It is still collected. The rule is parity-blind on purpose:
the cost of over-collecting is a lost widening, while
under-collecting would make a negation exclude far more than
the user asked
"""
leaf = content("tax")
tree = wc_ast.Not(child=wc_ast.Not(child=leaf))
assert _negated_leaf_ids(tree) == frozenset({id(leaf)})
@pytest.mark.parametrize(
"build",
[
pytest.param(
lambda leaf: wc_ast.Boosted(child=leaf, boost=2.0),
id="boosted",
),
pytest.param(
lambda leaf: wc_ast.Require(scored=leaf, filter_only=leaf),
id="require",
),
pytest.param(
lambda leaf: wc_ast.AndMaybe(required=leaf, optional=leaf),
id="andmaybe",
),
],
)
def test_containers_that_are_not_negations_collect_nothing(
self,
build,
) -> None:
"""
GIVEN:
- A leaf inside a container that is not a negation
WHEN:
- The tree is pre-scanned
THEN:
- Nothing is collected. Only Not.child and AndNot.negative are
negative positions
"""
leaf = content("invoice")
assert _negated_leaf_ids(build(leaf)) == frozenset()
def test_a_bare_leaf_is_accepted(self) -> None:
"""
GIVEN:
- A tree that is a single leaf, with no container at all
WHEN:
- It is pre-scanned
THEN:
- The set is empty and nothing raises
"""
assert _negated_leaf_ids(content("invoice")) == frozenset()
class TestTotality:
@pytest.mark.parametrize(
"node",
[
pytest.param(wc_ast.Every(), id="every"),
pytest.param(wc_ast.Nothing(), id="nothing"),
pytest.param(
wc_ast.Wildcard(field=CONTENT, pattern="inv*"),
id="wildcard",
),
pytest.param(
wc_ast.Fuzzy(field=CONTENT, text="invoce", distance=1, prefix=True),
id="fuzzy",
),
],
)
def test_an_unrecognised_node_does_not_raise(self, node: wc_ast.Node) -> None:
"""
GIVEN:
- A node type the walk does not collect from, including one
(Fuzzy) that this code will later build itself
WHEN:
- It is pre-scanned, bare and under a Not
THEN:
- Nothing raises. This function runs outside emit()'s error
conversion, so an exception here is an unconverted 500
"""
assert _negated_leaf_ids(node) == frozenset()
assert _negated_leaf_ids(wc_ast.Not(child=node)) == frozenset()
def test_a_deep_tree_does_not_exhaust_the_stack(self) -> None:
"""
GIVEN:
- A tree nested far deeper than the parser's 200-group cap
WHEN:
- It is pre-scanned
THEN:
- It completes. The walk is iterative, so depth costs heap
rather than Python stack frames
"""
leaf = content("invoice")
node: wc_ast.Node = leaf
for _ in range(5000):
node = wc_ast.Not(child=node)
assert _negated_leaf_ids(node) == frozenset({id(leaf)})
@@ -0,0 +1,156 @@
"""The pattern normalizer's stem-alternates contract, and its consistency
with the index-side analyzer.
Query patterns are normalized but were not stemmed, while index terms are
stemmed, so the natural spelling of a prefix search matched nothing:
``invoice*`` found no document although ``invoic*`` did. v2's index was
UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
regressed against both baselines.
These are pure unit tests against ``_make_pattern_normalizer`` and
``stem_pattern_text`` directly, no query routing involved. The end-to-end
proof that a real wildcard query actually reaches a stemmed index term
lives in ``test_pattern_stemming.py``.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.search._registry import _make_pattern_normalizer
from documents.search._tokenizer import ascii_fold
from documents.search._tokenizer import paperless_text_analyzer
from documents.search._tokenizer import stem_pattern_text
if TYPE_CHECKING:
from whoosh_compat import PatternNormalizer
class TestStemsMatchTheIndexAnalyzer:
"""stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather
than sharing it, so a filter added to the index analyzer alone would silently
stop patterns from reaching the terms it produces.
"""
@pytest.mark.parametrize(
"language",
["en", "de", "fr", "es", "sv", None, "klingon"],
)
@pytest.mark.parametrize(
"word",
["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
)
def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
"""
GIVEN:
- A word, across several representative index languages
("en", "de", "fr", "es", "sv"), no language, and an
unsupported language ("klingon")
WHEN:
- `stem_pattern_text` (the pattern-side stemmer) processes the
folded word, and `paperless_text_analyzer` (the index-side
analyzer) independently processes the same word
THEN:
- The two produce the identical term. `stem_pattern_text`
rebuilds `paperless_text_analyzer`'s stemming tail rather
than sharing it, so a filter added to the index analyzer
alone would silently stop patterns from reaching the terms
it produces; this pins the two staying in sync
"""
indexed = paperless_text_analyzer(language).analyze(word)[0]
assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]:
"""The distinct forms a term may match, in order, the way the emitter reads
the normalizer's answer (see whoosh_compat.PatternNormalizer)."""
result = normalize(text)
if isinstance(result, str):
return (result,)
return tuple(dict.fromkeys(result))
class TestPatternNormalizer:
@pytest.mark.parametrize(
("text", "expected"),
[
("Invoice", ("invoice", "invoic")),
("companies", ("companies", "compani")),
# y -> i is a substitution, so both forms are needed: the index
# holds "librari" for "library" and "library" for "librarian".
("library", ("library", "librari")),
# A run the stemmer leaves alone collapses back to one form, so it
# costs exactly the one regex branch it did before.
("invoic", ("invoic",)),
("Universit", ("universit",)),
("Café", ("cafe",)),
],
)
def test_offers_the_typed_run_and_its_stem(
self,
text: str,
expected: tuple[str, ...],
) -> None:
"""
GIVEN:
- The "en" pattern normalizer
WHEN:
- It processes a literal run (e.g. "Invoice", "library",
"Café")
THEN:
- It returns the folded run and, where it differs, the
stemmed form, as distinct alternatives; a run the stemmer
leaves alone (e.g. "invoic") collapses back to the single
folded form. "library" needs both forms since y -> i is a
substitution: the index holds "librari" for "library" and
"library" for "librarian"
"""
assert _forms(_make_pattern_normalizer("en"), text) == expected
def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
"""
GIVEN:
- The "en" pattern normalizer
WHEN:
- It processes a run past the analyzer's remove_long limit
THEN:
- The run analyzes to zero tokens, so there is no stem to
offer, and only the folded run remains
"""
over_long = "invoices" * 20
assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
@pytest.mark.parametrize("language", [None, "klingon"])
def test_unstemmed_language_folds_only(self, language: str | None) -> None:
"""
GIVEN:
- A pattern normalizer with no language configured, or one
this build has no stemmer for ("klingon")
WHEN:
- It processes "Invoices"
THEN:
- Only the folded form ("invoices") is offered, since with no
stemmer configured the index holds surface forms and the
pattern must keep them too
"""
assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
@pytest.mark.parametrize("char", ["a", "Z", "é"])
def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
"""
GIVEN:
- The "en" pattern normalizer
WHEN:
- It processes a single character
THEN:
- Exactly one, one-character form is returned. A bracket
class body is normalized one character at a time and the
answer is used only when it is a single one-character
form, so a stemmer that changed a lone character would
silently disable folding inside classes
"""
forms = _forms(_make_pattern_normalizer("en"), char)
assert len(forms) == 1
assert len(forms[0]) == 1
@@ -0,0 +1,213 @@
"""Wildcard patterns must match a stemmed index, end to end.
Query patterns are normalized but were not stemmed, while index terms are
stemmed, so the natural spelling of a prefix search matched nothing:
``invoice*`` found no document although ``invoic*`` did. v2's index was
UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
regressed against both baselines.
These are end-to-end tests against a real indexed document and a real
query, proving the pattern normalizer's stem-alternates contract actually
reaches a stemmed index term. The pure unit tests against the normalizer
function itself live in ``test_pattern_normalizer.py``.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
CONTENT = (
"invoice total due for electricity from both companies, "
"payments made to the university library, copies attached"
)
@pytest.fixture
def indexed_doc(index_document: Callable[..., Document]) -> Document:
return index_document(
title="Invoice 2020 productname",
content=CONTENT,
archive_serial_number=900,
)
class TestPrefixStemming:
@pytest.mark.parametrize(
"query",
[
"invoice*",
"electricity*",
"companies*",
"payments*",
"library*",
"title:Invoice*",
],
)
def test_full_word_prefix_matches_its_stem(
self,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document indexed with content containing "invoice",
"electricity", "companies", "payments", "library" and title
"Invoice 2020 productname"
WHEN:
- A prefix wildcard on the full, unstemmed word is queried
(e.g. "invoice*", "title:Invoice*")
THEN:
- The document matches, since the pattern normalizer offers
the word's stem as an alternative alongside the typed run,
reaching the stemmed index term
"""
assert matched_ids(query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
def test_already_stemmed_prefix_still_matches(
self,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- The same indexed document
WHEN:
- A prefix wildcard is typed already in its stemmed spelling
(e.g. "invoic*")
THEN:
- The document still matches, since the typed-run alternative
is itself a prefix of the stored stemmed term
"""
assert matched_ids(query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["univers*", "librar*"])
def test_partial_prefix_reaches_the_stemmed_term(
self,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- The same indexed document
WHEN:
- A prefix shorter than a whole word is queried ("univers*",
"librar*")
THEN:
- It still matches, and neither case needs the two-alternative
path to do it: measured under "en", the stemmer leaves
"librar" alone, so it has one form, and that form is a
prefix of the "librari" the index holds for "library";
"univers" stems to the *shorter* "univ", and the run as
typed and its stem are both prefixes of the "univers" the
index holds for "university". The case where the two forms
genuinely diverge, and only one of them matches, is
test_stem_substitution_reaches_both_the_inflection_and_the_compound
"""
assert matched_ids(query) == {indexed_doc.id}
def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
self,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The same indexed document, storing "university" as "univers"
WHEN:
- "universities*", "universit*" and "univers*" are each queried
THEN:
- "universities*" matches, since the stem of "universities" is
that same "univers"; "universit*" matches nothing, since
"universit" is a prefix of neither its own stem nor the
stored term. The alternatives widen recall without turning
a wildcard into a prefix search over the original text.
usage.md tells a reader whose `universit*` finds nothing to
shorten it to `univers*`, which matches
"""
assert matched_ids("universities*") == {indexed_doc.id}
assert matched_ids("universit*") == set()
assert matched_ids("univers*") == {indexed_doc.id}
def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
self,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The same indexed document, with "productname" indexed as
"productnam"
WHEN:
- "produ*name" (a pattern straddling the stem boundary) is
queried
THEN:
- It matches nothing; produ*name cannot match a stemmed
index, and usage.md must not advertise it. Pinned so the
limitation is deliberate, not accidental
"""
assert matched_ids("produ*name") == set()
def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The indexed document (containing "copies") plus a second
document titled "Copyright notice" with content "copyright
notice for the work"
WHEN:
- "copy*" and "copyright*" are each queried
THEN:
- "copy*" matches both documents, and "copyright*" matches
only the compound one. English stemming substitutes as well
as truncates: "copy" and "copies" both index as "copi",
while "copyright" keeps its literal "y". Neither form is a
prefix of the other, so no single normalized string reaches
both; the run is therefore emitted as a disjunction of the
folded and stemmed forms, and "copy*" reaches the base
word, its inflections and the compound alike
"""
compound = index_document(
title="Copyright notice",
content="copyright notice for the work",
archive_serial_number=901,
)
assert matched_ids("copy*") == {indexed_doc.id, compound.id}
assert matched_ids("copyright*") == {compound.id}
class TestBracketClassStillFolds:
def test_class_body_matches_case_insensitively(
self,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The indexed document, titled "Invoice 2020 productname"
WHEN:
- A bracket-class pattern mixing case is queried
("title:[IP]nvoice*")
THEN:
- It matches: the class body is folded per character, which
the alternatives contract preserves only because a lone
character stems to itself
"""
assert matched_ids("title:[IP]nvoice*") == {indexed_doc.id}
@@ -0,0 +1,199 @@
"""Permission filtering must hold against the real indexed document shape.
Only three of the index's unsigned ``*_id`` columns are load-bearing:
``owner_id``, ``viewer_id`` and ``viewer_group_id``, all read by
build_permission_filter. The rest (correspondent/document_type/storage_path/tag
ids) were written on every document and read by nothing, and were dropped.
These tests index real Documents through the backend's own document builder and
assert result-level visibility per user, so a mistake about which columns are
load-bearing shows up as documents leaking across users rather than as a passing
unit test over a hand-built index.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_object
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@pytest.fixture
def owner() -> User:
return UserFactory(username="owner")
@pytest.fixture
def stranger() -> User:
return UserFactory(username="stranger")
@pytest.fixture
def viewer() -> User:
return UserFactory(username="viewer")
@pytest.fixture
def group_member() -> User:
user = UserFactory(username="group_member")
user.groups.add(Group.objects.create(name="accounting"))
return user
class TestPermissionFilteringOnIndexedDocuments:
def test_unowned_document_is_visible_to_everyone(
self,
backend: TantivyBackend,
stranger: User,
) -> None:
"""
GIVEN:
- A document with no owner, indexed via the backend's real
document builder
WHEN:
- A stranger (no relation to the document) searches
THEN:
- The document is visible to them
"""
doc = Document.objects.create(
title="Public Invoice",
content="invoice total due",
checksum="perm-unowned",
)
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=stranger) == [doc.pk]
def test_owned_document_is_visible_only_to_its_owner(
self,
backend: TantivyBackend,
owner: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user, indexed via the backend's
real document builder
WHEN:
- The owner and an unrelated stranger each search
THEN:
- The owner sees the document; the stranger does not
"""
doc = Document.objects.create(
title="Private Invoice",
content="invoice total due",
checksum="perm-owned",
owner=owner,
)
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=owner) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
def test_explicitly_shared_document_is_visible_to_the_viewer(
self,
backend: TantivyBackend,
owner: User,
viewer: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user and explicitly shared with a
second user via guardian's view_document permission,
indexed via the backend's real document builder
WHEN:
- The shared viewer and an unrelated stranger each search
THEN:
- The viewer sees the document; the stranger does not
"""
doc = Document.objects.create(
title="Shared Invoice",
content="invoice total due",
checksum="perm-shared-user",
owner=owner,
)
grant_object(viewer, doc, "view_document")
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=viewer) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
def test_group_shared_document_is_visible_to_group_members(
self,
backend: TantivyBackend,
owner: User,
group_member: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user and shared with a group via
guardian's view_document permission, indexed via the
backend's real document builder
WHEN:
- A member of that group and an unrelated stranger each
search
THEN:
- The group member sees the document; the stranger does not
"""
doc = Document.objects.create(
title="Group Invoice",
content="invoice total due",
checksum="perm-shared-group",
owner=owner,
)
grant_object(group_member.groups.first(), doc, "view_document")
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=group_member) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
def test_metadata_does_not_widen_visibility(
self,
backend: TantivyBackend,
owner: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user and carrying
correspondent/document_type/storage_path/tag metadata,
indexed via the backend's real document builder
WHEN:
- The owner and an unrelated stranger each search
THEN:
- The owner sees the document; the stranger does not, since
the dropped, non-load-bearing metadata *_id columns must
not widen visibility beyond the owner_id/viewer_id/
viewer_group_id filter
"""
doc = Document.objects.create(
title="Tagged Invoice",
content="invoice total due",
checksum="perm-metadata",
owner=owner,
correspondent=Correspondent.objects.create(name="ACME"),
document_type=DocumentType.objects.create(name="Bill"),
storage_path=StoragePath.objects.create(name="Archive", path="archive/"),
)
doc.tags.add(Tag.objects.create(name="paid"))
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=owner) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,154 @@
"""Negation must survive the blended query.
parse_user_query ORs an exact clause with an optional fuzzy clause. That
clause is built from positive terms only, so unless the query's exclusions
are applied to the blend as a whole, a document the exact clause excluded
is re-admitted by it. CJK terms are widened inside the exact clause itself,
so the query's own structure constrains them.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
class TestNegationConstrainsEveryClause:
@pytest.mark.usefixtures("fuzzy_enabled")
def test_fuzzy_clause_does_not_readmit_an_excluded_document(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two documents both matching a positive term, one of which
also contains a word the query excludes, with the fuzzy
blend clause enabled
WHEN:
- A query combining the positive term with a NOT exclusion is
run
THEN:
- Only the document without the excluded word is returned;
the fuzzy clause (built from positive terms only) does not
readmit the document the exact clause excluded
"""
secret = index_document(
title="Invoice A",
content="invoice total secret",
)
public = index_document(
title="Invoice B",
content="invoice total public",
)
assert matched_ids("invoice") == {secret.pk, public.pk}
assert matched_ids("invoice NOT secret") == {public.pk}
def test_an_excluded_document_stays_out_of_a_cjk_match(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two documents both containing a CJK run, one of which also
contains a word the query excludes
WHEN:
- A query combining the CJK term with a NOT exclusion is run
THEN:
- Only the document without the excluded word is returned;
the CJK term's bigram match sits beside the NOT inside the
same query, so the exclusion applies to it
"""
secret = index_document(
title="Tokyo A",
content="東京都の秘密です secret",
)
public = index_document(
title="Tokyo B",
content="東京都の報告書です public",
)
assert matched_ids("東京") == {secret.pk, public.pk}
assert matched_ids("東京 NOT secret") == {public.pk}
@pytest.mark.usefixtures("fuzzy_enabled")
def test_disjunctive_negation_still_admits_the_other_branch(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- A document matching a positive term and also containing a
word a disjunctive NOT branch excludes, plus an unrelated
document
WHEN:
- A query of the shape "term OR NOT excluded_word" is run
THEN:
- Both documents are returned; "invoice OR NOT secret"
excludes nothing on its own, so a document matching the
left branch stays in even though it contains the excluded
word
"""
secret_invoice = index_document(
title="Invoice A",
content="invoice total secret",
)
unrelated = index_document(
title="Recipe",
content="flour and water",
)
assert matched_ids("invoice OR NOT secret") == {
secret_invoice.pk,
unrelated.pk,
}
def test_a_negation_under_or_constrains_its_own_cjk_term(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
- Two CJK documents, one of which also contains a word an OR
branch's own NOT excludes, plus an unrelated latin document
WHEN:
- The exclusion is under a disjunctive OR branch, and in
conjunctive position
THEN:
- Either way the excluded document is left out. The CJK
term's bigram match is widened in place inside its own OR
branch, so that branch's NOT applies to it; the other
branch still admits the latin document
"""
secret = index_document(
title="Tokyo A",
content="東京都の秘密です secret",
)
public = index_document(
title="Tokyo B",
content="東京都の報告書です public",
)
bill = index_document(
title="Bill",
content="bill payment received",
)
assert matched_ids("(東京 AND NOT secret) OR bill") == {
bill.pk,
public.pk,
}
assert matched_ids("東京 AND NOT secret") == {public.pk}
assert secret.pk in matched_ids("東京")
+224
View File
@@ -0,0 +1,224 @@
from collections.abc import Sequence
import pytest
from whoosh_compat import FieldKind
from whoosh_compat import FieldRegistry
from whoosh_compat.fields import ResolvedField
from documents.search._fields import PUBLIC_FIELDS
from documents.search._registry import get_field_registry
@pytest.fixture
def registry() -> FieldRegistry:
return get_field_registry(None)
def _resolve(registry: FieldRegistry, name: str) -> ResolvedField:
ref = registry.make_ref(name)
assert ref is not None, f"{name} is not a valid field ref"
resolved = registry.resolve(ref)
assert resolved is not None, f"{name} did not resolve"
return resolved
def _distinct_forms(result: str | Sequence[str]) -> tuple[str, ...]:
"""The forms a term may match, in order, the way whoosh-compat's emitter
reads a pattern_normalizer's answer: a bare str is one form, a sequence is
several, deduplicated."""
if isinstance(result, str):
return (result,)
return tuple(dict.fromkeys(result))
class TestFieldRegistry:
def test_no_queryable_field_name_ends_in_id(self) -> None:
"""
GIVEN:
- PUBLIC_FIELDS, the canonical query-syntax field table
WHEN:
- Every declared field name is inspected
THEN:
- None of them end in "_id" (internal id columns, written for
permission filtering and joins, must never reach the query
surface; checked against PUBLIC_FIELDS rather than the
registry so a leak is caught where it is declared)
"""
leaked = [f.name for f in PUBLIC_FIELDS if f.name.endswith("_id")]
assert not leaked, f"internal id fields reached the query surface: {leaked}"
def test_type_alias_resolves_to_document_type(
self,
registry: FieldRegistry,
) -> None:
"""
GIVEN:
- The field registry
WHEN:
- The alias "type" is resolved
THEN:
- It resolves to the canonical "document_type" field
"""
assert _resolve(registry, "type").spec.name == "document_type"
def test_path_alias_resolves_to_storage_path(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry
WHEN:
- The alias "path" is resolved
THEN:
- It resolves to the canonical "storage_path" field
"""
assert _resolve(registry, "path").spec.name == "storage_path"
def test_notes_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry
WHEN:
- "notes.user" is resolved
THEN:
- It resolves to the "notes" field with json_path "user"
"""
resolved = _resolve(registry, "notes.user")
assert resolved.spec.name == "notes"
assert resolved.json_path == "user"
assert resolved.is_subpath is True
def test_custom_fields_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry
WHEN:
- "custom_fields.name" and "custom_fields.value" are resolved
THEN:
- Both resolve without error
"""
for raw in ("custom_fields.name", "custom_fields.value"):
_resolve(registry, raw)
def test_tag_is_comma_values(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry
WHEN:
- The "tag" field is resolved
THEN:
- It is marked comma_values=True
"""
assert _resolve(registry, "tag").spec.comma_values is True
def test_correspondent_is_not_comma_values(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry
WHEN:
- The "correspondent" field is resolved
THEN:
- It is not marked comma_values ("tag" is the only field that
opts in; end to end the two readings of
"correspondent:foo,bar" agree anyway, since the analyzer
splits the literal value on the comma regardless, so this is
only observable at the registry level)
"""
assert _resolve(registry, "correspondent").spec.comma_values is False
def test_created_is_date_kind(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry
WHEN:
- The "created" field is resolved
THEN:
- Its kind is DATE and date_only is True
"""
resolved = _resolve(registry, "created")
assert resolved.spec.kind is FieldKind.DATE
assert resolved.spec.date_only is True
def test_analyzer_lowercases_and_ascii_folds(self, registry: FieldRegistry) -> None:
"""
GIVEN:
- The field registry with no language configured (no stemmer
in the analyzer chain)
WHEN:
- The "title" field's analyzer processes "Café"
THEN:
- It is lowercased and ASCII-folded to the single token "cafe"
"""
resolved = _resolve(registry, "title")
assert resolved.spec.analyzer is not None
assert resolved.spec.analyzer("Café") == ["cafe"]
def test_checksum_analyzer_is_identity_single_token(
self,
registry: FieldRegistry,
) -> None:
"""
GIVEN:
- The field registry
WHEN:
- The "checksum" field's analyzer (raw tokenizer, no
splitting) processes "ABC-123"
THEN:
- It is returned unchanged as a single token
"""
resolved = _resolve(registry, "checksum")
assert resolved.spec.analyzer is not None
assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
def test_pattern_normalizer_follows_the_registry_language(
self,
registry: FieldRegistry,
) -> None:
"""
GIVEN:
- A registry with no language, and a registry built for "en"
WHEN:
- The "title" field's pattern normalizer processes "Running"
THEN:
- With no language, only the folded run is offered
("running"), since the index holds surface forms
- With "en", the stem is offered too ("run"), since indexed
terms are stemmed and the pattern has to reach them
"""
resolved = _resolve(registry, "title")
assert resolved.spec.pattern_normalizer is not None
assert _distinct_forms(resolved.spec.pattern_normalizer("Running")) == (
"running",
)
resolved_en = _resolve(get_field_registry("en"), "title")
assert resolved_en.spec.pattern_normalizer is not None
assert _distinct_forms(resolved_en.spec.pattern_normalizer("Running")) == (
"running",
"run",
)
def test_registry_is_cached_per_language(self) -> None:
"""
GIVEN:
- Two calls to get_field_registry("en")
WHEN:
- Both calls are made
THEN:
- They return the same registry instance
"""
a = get_field_registry("en")
b = get_field_registry("en")
assert a is b
def test_registry_rebuilds_on_language_change(self) -> None:
"""
GIVEN:
- A call to get_field_registry("en") and a call to
get_field_registry("de")
WHEN:
- Both calls are made
THEN:
- They return different registry instances
"""
a = get_field_registry("en")
b = get_field_registry("de")
assert a is not b
+92 -20
View File
@@ -5,14 +5,19 @@ from typing import TYPE_CHECKING
import pytest
from documents.search._fields import PUBLIC_FIELDS
from documents.search._schema import SCHEMA_VERSION
from documents.search._schema import build_schema
from documents.search._schema import field_descriptors
from documents.search._schema import needs_rebuild
from documents.search._schema import schema_fingerprint
if TYPE_CHECKING:
from pathlib import Path
import tantivy
from pytest_django.fixtures import Settings
from paperless_testing.dirs import PaperlessDirs
pytestmark = pytest.mark.search
@@ -20,60 +25,127 @@ pytestmark = pytest.mark.search
class TestNeedsRebuild:
"""needs_rebuild covers all sentinel-file states that require a full reindex."""
def test_returns_true_when_settings_file_missing(self, index_dir: Path) -> None:
assert needs_rebuild(index_dir) is True
def test_returns_true_when_settings_file_missing(
self,
paperless_dirs: PaperlessDirs,
) -> None:
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_returns_false_when_version_and_language_match(
self,
index_dir: Path,
paperless_dirs: PaperlessDirs,
settings: Settings,
) -> None:
settings.SEARCH_LANGUAGE = "en"
(index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
(paperless_dirs.index_dir / ".index_settings.json").write_text(
json.dumps(
{
"schema_version": SCHEMA_VERSION,
"language": "en",
"schema_fingerprint": schema_fingerprint(),
},
),
)
assert needs_rebuild(index_dir) is False
assert needs_rebuild(paperless_dirs.index_dir) is False
def test_returns_true_on_schema_version_mismatch(
self,
index_dir: Path,
paperless_dirs: PaperlessDirs,
settings: Settings,
) -> None:
settings.SEARCH_LANGUAGE = None
(index_dir / ".index_settings.json").write_text(
(paperless_dirs.index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": SCHEMA_VERSION - 1, "language": None}),
)
assert needs_rebuild(index_dir) is True
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_returns_true_when_version_is_not_an_integer(
self,
index_dir: Path,
paperless_dirs: PaperlessDirs,
settings: Settings,
) -> None:
settings.SEARCH_LANGUAGE = None
(index_dir / ".index_settings.json").write_text(
(paperless_dirs.index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": "not-a-number", "language": None}),
)
assert needs_rebuild(index_dir) is True
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_returns_true_when_language_key_missing(
self,
index_dir: Path,
paperless_dirs: PaperlessDirs,
settings: Settings,
) -> None:
settings.SEARCH_LANGUAGE = "en"
(index_dir / ".index_settings.json").write_text(
(paperless_dirs.index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": SCHEMA_VERSION}),
)
assert needs_rebuild(index_dir) is True
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_returns_true_when_language_differs(
self,
index_dir: Path,
paperless_dirs: PaperlessDirs,
settings: Settings,
) -> None:
settings.SEARCH_LANGUAGE = "de"
(index_dir / ".index_settings.json").write_text(
(paperless_dirs.index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
)
assert needs_rebuild(index_dir) is True
assert needs_rebuild(paperless_dirs.index_dir) is True
def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]:
"""{name: field-state} for every field declared on a tantivy Schema.
tantivy-py 0.26 exposes no public introspection API on Schema (no
__iter__, get_field, to_json, etc.) -- __reduce__() (used internally for
pickling) is the only way to recover the field list, so we lean on it
here for test assertions only.
"""
state = schema.__reduce__()[1][0]
return {field["name"]: field for field in state["inner"]}
class TestSchemaMatchesPublicFields:
def test_every_public_field_is_in_the_schema(self) -> None:
"""
GIVEN:
- PUBLIC_FIELDS and the tantivy schema built by build_schema()
WHEN:
- Every field declared in PUBLIC_FIELDS is checked against the
schema
THEN:
- Each one is present as a field in the built schema
"""
schema = build_schema()
schema_field_names = set(_schema_fields(schema))
for field in PUBLIC_FIELDS:
assert field.name in schema_field_names, (
f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()"
)
class TestFastFlagAgreement:
def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None:
"""
GIVEN:
- PUBLIC_FIELDS and field_descriptors() (the latter is exactly
the input build_schema()'s SchemaBuilder consumes for the
`fast` kwarg on every field kind, so it pins the agreement
without depending on a private tantivy-py pickled
representation)
WHEN:
- Every PUBLIC_FIELDS entry's fast flag is compared against
field_descriptors()' fast flag for the same field
THEN:
- They agree for every field, catching a fast=True
PUBLIC_FIELDS entry the builder silently ignores here
instead of at a user's field:* existence query, which
whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag to
resolve
"""
descriptor_fast = {d.name: d.fast for d in field_descriptors()}
for public_field in PUBLIC_FIELDS:
assert descriptor_fast[public_field.name] == public_field.fast, (
f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but"
f" field_descriptors() says fast={descriptor_fast[public_field.name]}"
)
@@ -0,0 +1,597 @@
"""The schema fingerprint stamped into .index_settings.json.
tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema,
path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION
is the manual guard against that, but build_schema() is edited for *parser*
reasons - adding an alias, flipping fast=True, adding a subpath - by people not
thinking about the on-disk index, and forgetting the bump is exactly how this
branch's bug happened.
The fingerprint is the automatic guard: it hashes the field descriptor list that
build_schema() itself iterates, so any change to a field's name, kind, options
or *position* forces a rebuild on its own.
"""
from __future__ import annotations
import hashlib
import json
from typing import TYPE_CHECKING
import pytest
import tantivy
from documents.search import _schema
from documents.search._schema import SCHEMA_VERSION
from documents.search._schema import FieldDescriptor
from documents.search._schema import _write_sentinels
from documents.search._schema import build_schema
from documents.search._schema import field_descriptors
from documents.search._schema import needs_rebuild
from documents.search._schema import schema_fingerprint
if TYPE_CHECKING:
from pathlib import Path
from pytest_django.fixtures import SettingsWrapper
from paperless_testing.dirs import PaperlessDirs
pytestmark = pytest.mark.search
# The on-disk field layout of a v2 index, pinned as data. Any edit here is an
# index-format change: it must come with a rebuild, which the fingerprint now
# forces automatically. Reproduced from build_schema()'s output as it stood
# before the descriptor refactor, so it also pins that the refactor changed
# nothing.
PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = (
FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
FieldDescriptor(
"title",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"content",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"correspondent",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"document_type",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"storage_path",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"original_filename",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"tag",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"checksum",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="raw",
),
FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
FieldDescriptor(
"page_count",
"u64",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"num_notes",
"u64",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"created",
"date",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"modified",
"date",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"added",
"date",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"notes",
"json",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"notes_text",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"custom_fields",
"json",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"barcodes",
"json",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"title_sort",
"text",
stored=False,
indexed=True,
fast=True,
tokenizer="simple_analyzer",
),
FieldDescriptor(
"correspondent_sort",
"text",
stored=False,
indexed=True,
fast=True,
tokenizer="simple_analyzer",
),
FieldDescriptor(
"type_sort",
"text",
stored=False,
indexed=True,
fast=True,
tokenizer="simple_analyzer",
),
FieldDescriptor(
"bigram_content",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_title",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_correspondent",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_document_type",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_tag",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"simple_title",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="simple_search_analyzer",
),
FieldDescriptor(
"simple_content",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="simple_search_analyzer",
),
FieldDescriptor(
"autocomplete_word",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="raw",
),
FieldDescriptor(
"owner_id",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"viewer_id",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"viewer_group_id",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
)
def _schema_fields(schema: tantivy.Schema) -> list[dict]:
"""The tantivy-level field list, in declaration order.
tantivy-py 0.26 exposes no public introspection API on Schema, so
__reduce__() (its pickling hook) is the only way to recover the field list.
It is used here, in a test, precisely because it is the representation the
persisted fingerprint must NOT depend on.
"""
return schema.__reduce__()[1][0]["inner"]
def _sentinels(index_dir: Path, **overrides: object) -> None:
data = {
"schema_version": SCHEMA_VERSION,
"language": None,
"schema_fingerprint": schema_fingerprint(),
}
data.update(overrides)
(index_dir / ".index_settings.json").write_text(json.dumps(data))
class TestDescriptorsDescribeTheBuiltSchema:
def test_descriptors_match_the_pinned_field_layout(self) -> None:
"""
GIVEN:
- PINNED_DESCRIPTORS, a frozen snapshot of the v2 on-disk field
layout, reproduced from build_schema()'s output as it stood
before the descriptor refactor
WHEN:
- field_descriptors() is called
THEN:
- It matches the pinned layout exactly, in the same order,
pinning that the refactor changed nothing
"""
assert tuple(field_descriptors()) == PINNED_DESCRIPTORS
def test_built_schema_matches_the_descriptors(self) -> None:
"""
GIVEN:
- The schema built by build_schema()
WHEN:
- Its fields are read back via __reduce__() (schema.__reduce__(),
tantivy-py's pickling hook)
THEN:
- Every field's name, kind, stored/fast flags and tokenizer
match what field_descriptors() declared as input; the
descriptors are not a parallel description, they are the
input, so a descriptor edit cannot claim a shape the
SchemaBuilder did not actually build
"""
kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"}
built = [
(
field["name"],
field["type"],
field["options"]["stored"],
bool(field["options"].get("fast")),
(field["options"].get("indexing") or {}).get("tokenizer"),
)
for field in _schema_fields(build_schema())
]
expected = [
(
descriptor.name,
kinds[descriptor.kind],
descriptor.stored,
descriptor.fast,
descriptor.tokenizer,
)
for descriptor in field_descriptors()
]
assert built == expected
class TestFingerprintSensitivity:
def test_a_field_option_change_moves_the_fingerprint(
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- The current schema fingerprint
WHEN:
- A single field descriptor's "fast" option is changed, with
no other change
THEN:
- The fingerprint changes
"""
before = schema_fingerprint()
changed = field_descriptors()
changed[1] = changed[1]._replace(fast=True)
monkeypatch.setattr(_schema, "field_descriptors", lambda: changed)
assert schema_fingerprint() != before
def test_reordering_alone_moves_the_fingerprint(
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- The current schema fingerprint
WHEN:
- Two field descriptors are swapped, with no other change (the
original bug: same fields, different declaration order)
THEN:
- The fingerprint changes; a set- or dict-based fingerprint
would be blind to this, and tantivy would reject every write
against the existing index
"""
before = schema_fingerprint()
swapped = field_descriptors()
swapped[1], swapped[2] = swapped[2], swapped[1]
monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped)
assert schema_fingerprint() != before
class TestFingerprintIsIndependentOfTantivy:
def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None:
"""
GIVEN:
- The built schema's raw field list, and the same list with a
new tantivy-internal option key added (simulating a
tantivy-py upgrade)
WHEN:
- Both raw lists are hashed directly, and schema_fingerprint()
is compared against a hash of field_descriptors()
THEN:
- The raw hashes differ (hashing schema.__reduce__() would
force a global reindex on every tantivy-py upgrade), but
schema_fingerprint() is unaffected, since it hashes
field_descriptors(), never tantivy's own representation
"""
fields = _schema_fields(build_schema())
upgraded = [
{**field, "options": {**field["options"], "coerce": True}}
for field in fields
]
assert _hash(upgraded) != _hash(fields)
assert schema_fingerprint() == _fingerprint_of(field_descriptors())
def test_fingerprint_never_touches_the_schema_builder(
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- tantivy.SchemaBuilder replaced with a stand-in that raises if
constructed
WHEN:
- build_schema() is called (and raises), then
schema_fingerprint() is called again
THEN:
- schema_fingerprint() still matches its earlier value,
proving it never consults SchemaBuilder
"""
before = schema_fingerprint()
class _RemovedSchemaBuilder:
def __init__(self) -> None:
raise AssertionError("tantivy.SchemaBuilder was consulted")
monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder)
with pytest.raises(AssertionError):
build_schema()
assert schema_fingerprint() == before
def _hash(payload: object) -> str:
return hashlib.blake2b(json.dumps(payload).encode()).hexdigest()
def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
return _hash([list(descriptor) for descriptor in descriptors])
class TestNeedsRebuildOnFingerprint:
def test_matching_fingerprint_does_not_rebuild(
self,
paperless_dirs: PaperlessDirs,
settings: SettingsWrapper,
) -> None:
"""
GIVEN:
- An index directory whose sentinel file records the current
schema_fingerprint()
WHEN:
- needs_rebuild() is called
THEN:
- It returns False
"""
settings.SEARCH_LANGUAGE = None
_sentinels(paperless_dirs.index_dir)
assert needs_rebuild(paperless_dirs.index_dir) is False
def test_stale_fingerprint_rebuilds_despite_a_matching_version(
self,
paperless_dirs: PaperlessDirs,
settings: SettingsWrapper,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- An index directory whose sentinel matches SCHEMA_VERSION,
but field_descriptors() is patched to add a field the
fingerprint never saw (schema edited, version not bumped)
WHEN:
- needs_rebuild() is called
THEN:
- It returns True; without the fingerprint check,
`reindex --if-needed` would report the index up to date and
every subsequent write would raise
"""
settings.SEARCH_LANGUAGE = None
_sentinels(paperless_dirs.index_dir)
extended = [
*field_descriptors(),
FieldDescriptor(
"new_field",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
]
monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_reordered_schema_rebuilds(
self,
paperless_dirs: PaperlessDirs,
settings: SettingsWrapper,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- An index directory whose sentinel matches the current
fingerprint, but field_descriptors() is patched to swap two
fields' order
WHEN:
- needs_rebuild() is called
THEN:
- It returns True
"""
settings.SEARCH_LANGUAGE = None
_sentinels(paperless_dirs.index_dir)
reordered = field_descriptors()
reordered[1], reordered[2] = reordered[2], reordered[1]
monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_missing_fingerprint_rebuilds(
self,
paperless_dirs: PaperlessDirs,
settings: SettingsWrapper,
) -> None:
"""
GIVEN:
- An index directory whose sentinel has no "schema_fingerprint"
key at all
WHEN:
- needs_rebuild() is called
THEN:
- It returns True; an index whose schema shape nobody recorded
is rebuilt rather than trusted
"""
settings.SEARCH_LANGUAGE = None
(paperless_dirs.index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
)
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_written_sentinels_satisfy_the_check(
self,
paperless_dirs: PaperlessDirs,
settings: SettingsWrapper,
) -> None:
"""
GIVEN:
- An index directory whose sentinels are written by
_write_sentinels() itself
WHEN:
- needs_rebuild() is called
THEN:
- It returns False
"""
settings.SEARCH_LANGUAGE = "en"
_write_sentinels(paperless_dirs.index_dir)
assert needs_rebuild(paperless_dirs.index_dir) is False
@@ -0,0 +1,227 @@
"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does.
tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the
schema from the index's own ``meta.json``, so reads against an index built by an
older release keep working after a field reorder. Writes do not:
``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an
open-or-create that raises ``ValueError`` on any schema difference. Nothing
catches that ValueError, so consumption, index_document and bulk edit all
hard-fail while ``/api/status/`` still reports the index healthy.
The only thing that saves such an install is ``needs_rebuild()`` noticing the
version stamped in ``.index_settings.json`` is stale.
"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING
import pytest
import tantivy
from django.conf import settings as django_settings
from documents.search._schema import SCHEMA_VERSION
from documents.search._schema import build_schema
from documents.search._schema import needs_rebuild
from documents.search._schema import open_or_rebuild_index
from documents.search._schema import schema_fingerprint
if TYPE_CHECKING:
from pathlib import Path
pytestmark = [pytest.mark.search]
RELEASED_V1_SCHEMA_VERSION = 1
def _build_released_v1_schema() -> tantivy.Schema:
"""Frozen copy of build_schema() as shipped in v3.0.x (schema version 1).
Deliberately duplicated rather than imported: it must keep describing the
on-disk layout of already-deployed indexes even as build_schema() evolves.
"""
sb = tantivy.SchemaBuilder()
sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
for field in (
"title",
"correspondent",
"document_type",
"storage_path",
"original_filename",
"content",
):
sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
for field in ("title_sort", "correspondent_sort", "type_sort"):
sb.add_text_field(
field,
stored=False,
tokenizer_name="simple_analyzer",
fast=True,
)
for field in (
"bigram_content",
"bigram_title",
"bigram_correspondent",
"bigram_document_type",
"bigram_tag",
):
sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer")
for field in ("simple_title", "simple_content"):
sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer")
sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
for field in (
"correspondent_id",
"document_type_id",
"storage_path_id",
"tag_id",
"owner_id",
"viewer_id",
"viewer_group_id",
):
sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
for field in ("created", "modified", "added"):
sb.add_date_field(field, stored=True, indexed=True, fast=True)
for field in ("asn", "page_count", "num_notes"):
sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
return sb.build()
@pytest.fixture
def released_v1_index(tmp_path: Path) -> Path:
"""An index directory as a v3.0.x install would leave it on disk."""
index_dir = tmp_path / "index"
index_dir.mkdir()
tantivy.Index(_build_released_v1_schema(), path=str(index_dir))
(index_dir / ".index_settings.json").write_text(
json.dumps(
{
"schema_version": RELEASED_V1_SCHEMA_VERSION,
"language": django_settings.SEARCH_LANGUAGE,
},
),
)
return index_dir
class TestUpgradeFromReleasedV1Index:
def test_released_v1_index_is_flagged_for_rebuild(
self,
released_v1_index: Path,
) -> None:
"""
GIVEN:
- An index directory laid out exactly as a v3.0.x (schema
version 1) install would leave it
WHEN:
- needs_rebuild() is called
THEN:
- It returns True; if this fails,
`document_index reindex --if-needed` prints "Search index is
up to date" and skips, leaving the mismatched index in place
"""
assert needs_rebuild(released_v1_index) is True
def test_opening_a_v1_index_leaves_it_writable(
self,
released_v1_index: Path,
) -> None:
"""
GIVEN:
- A v1 index directory
WHEN:
- open_or_rebuild_index() is called against it
THEN:
- The directory can be reopened with the current schema
without raising; end to end, open_or_rebuild_index must
hand back an index the write path can reopen. Before the
version bump, needs_rebuild() returned False here, and the
stale directory survived untouched, so every subsequent
write against it raised tantivy's own schema-mismatch
ValueError
"""
open_or_rebuild_index(released_v1_index)
tantivy.Index(build_schema(), path=str(released_v1_index))
def test_rebuilt_index_is_not_rebuilt_again(
self,
released_v1_index: Path,
) -> None:
"""
GIVEN:
- A v1 index directory that has just been rebuilt by
open_or_rebuild_index()
WHEN:
- needs_rebuild() is called again
THEN:
- It returns False; the rebuild must stamp the version it
actually wrote, otherwise every startup wipes and reindexes
the whole corpus
"""
open_or_rebuild_index(released_v1_index)
assert needs_rebuild(released_v1_index) is False
class TestCorruptedIndex:
@pytest.fixture
def current_index_missing_meta_json(self, tmp_path: Path) -> Path:
"""A current-schema index directory whose meta.json has gone missing.
The sentinel matches the current schema, so needs_rebuild() reports
False and open_or_rebuild_index() takes the plain Index.open() path.
"""
index_dir = tmp_path / "index"
index_dir.mkdir()
tantivy.Index(build_schema(), path=str(index_dir))
(index_dir / ".index_settings.json").write_text(
json.dumps(
{
"schema_version": SCHEMA_VERSION,
"language": django_settings.SEARCH_LANGUAGE,
"schema_fingerprint": schema_fingerprint(),
},
),
)
(index_dir / "meta.json").unlink()
return index_dir
def test_missing_meta_json_is_rebuilt_instead_of_raising(
self,
current_index_missing_meta_json: Path,
) -> None:
"""
GIVEN:
- An index directory with a sentinel matching the current
schema, but a missing/corrupted meta.json (e.g. an
interrupted write left the directory in a torn state)
WHEN:
- open_or_rebuild_index() is called against it
THEN:
- It rebuilds the index instead of letting tantivy's
ValueError propagate; before this, a corrupted index left
every read, write, and `document_index reindex --if-needed`
hard-failing indefinitely, since the sentinel already
matched and reported the index as up to date
"""
open_or_rebuild_index(current_index_missing_meta_json)
assert (current_index_missing_meta_json / "meta.json").exists()
tantivy.Index.open(str(current_index_missing_meta_json))
@@ -0,0 +1,37 @@
from __future__ import annotations
import pytest
from documents.search._tokenizer import stem_pattern_text
pytestmark = pytest.mark.search
class TestStemPatternText:
def test_unsupported_language_returns_text_unchanged(self) -> None:
"""
GIVEN:
- A language code with no Snowball stemmer mapping
WHEN:
- A pattern run is stemmed for that language
THEN:
- The run is returned unchanged, since the stemming gate that
disables stemming for an unsupported language also disables
the pattern-side stemmer
"""
assert stem_pattern_text("running", "klingon") == "running"
def test_run_past_remove_long_limit_returns_text_unchanged(self) -> None:
"""
GIVEN:
- A supported language and a run longer than the remove_long
filter's limit (129 characters, matching Document.title's
max_length)
WHEN:
- The over-long run is stemmed
THEN:
- The remove_long filter drops the token entirely, leaving no
stem to substitute, so the run is returned unchanged
"""
long_run = "a" * 130
assert stem_pattern_text(long_run, "en") == long_run
+2 -2
View File
@@ -7,8 +7,8 @@ import pytest
import tantivy
from documents.search._tokenizer import _bigram_analyzer
from documents.search._tokenizer import _paperless_text
from documents.search._tokenizer import _simple_search_analyzer
from documents.search._tokenizer import paperless_text_analyzer
from documents.search._tokenizer import register_tokenizers
if TYPE_CHECKING:
@@ -25,7 +25,7 @@ class TestTokenizers:
sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
schema = sb.build()
idx = tantivy.Index(schema, path=None)
idx.register_tokenizer("paperless_text", _paperless_text(""))
idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
return idx
@pytest.fixture
@@ -1,810 +0,0 @@
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
from zoneinfo import ZoneInfo
import pytest
import time_machine
from documents.search._dates import _precision_bounds
if TYPE_CHECKING:
import tantivy
from documents.search._query import _FIELD_BOOSTS
from documents.search._query import DEFAULT_SEARCH_FIELDS
from documents.search._translate import OPEN_HI
from documents.search._translate import OPEN_LO
from documents.search._translate import Comma
from documents.search._translate import FieldRange
from documents.search._translate import FieldValue
from documents.search._translate import FieldValueList
from documents.search._translate import InvalidDateQuery
from documents.search._translate import Passthrough
from documents.search._translate import resolve_commas
from documents.search._translate import scan
from documents.search._translate import translate_query
from documents.search._translate import translate_range
from documents.search._translate import translate_scalar
@pytest.mark.search
class TestPrecisionBounds:
@pytest.mark.parametrize(
("digits", "expected"),
[
("2020", ((2020, 1, 1), (2021, 1, 1))),
("202003", ((2020, 3, 1), (2020, 4, 1))),
("202012", ((2020, 12, 1), (2021, 1, 1))),
("20200115", ((2020, 1, 15), (2020, 1, 16))),
("20201231", ((2020, 12, 31), (2021, 1, 1))),
],
)
def test_valid(self, digits, expected):
lo, hi = _precision_bounds(digits)
assert (lo.year, lo.month, lo.day) == expected[0]
assert (hi.year, hi.month, hi.day) == expected[1]
@pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"])
def test_invalid_returns_none(self, digits):
assert _precision_bounds(digits) is None
@pytest.mark.search
class TestScan:
def test_plain_words_are_passthrough(self):
assert scan("bank statement") == [Passthrough("bank statement")]
def test_field_value(self):
assert scan("created:2020") == [FieldValue("created", "2020")]
def test_field_value_in_boolean(self):
toks = scan("created:2020 OR foo")
assert toks == [
FieldValue("created", "2020"),
Passthrough(" OR foo"),
]
def test_field_value_in_parens(self):
toks = scan("(created:2020 OR foo)")
assert toks == [
Passthrough("("),
FieldValue("created", "2020"),
Passthrough(" OR foo)"),
]
def test_quoted_value(self):
assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')]
def test_field_range(self):
assert scan("created:[2020 TO 2021]") == [
FieldRange("created", "[", "2020", "2021", "]"),
]
@pytest.mark.parametrize(
("query", "expected"),
[
pytest.param(
"created:[2020 to]",
FieldRange("created", "[", "2020", "", "]"),
id="open_upper",
),
pytest.param(
"created:[to 2020]",
FieldRange("created", "[", "", "2020", "]"),
id="open_lower",
),
],
)
def test_open_range(self, query, expected):
assert scan(query) == [expected]
def test_comma_inside_range_not_split(self):
# No depth-0 comma here; the whole thing is one range token.
toks = scan("created:[2020 TO 2021]")
assert len(toks) == 1
# --- Edge-case / regression tests (scan must never raise) ---
def test_url_is_passthrough(self):
# "http" is not a known field; the whole URL must pass through verbatim.
assert scan("http://example.com") == [Passthrough("http://example.com")]
def test_unterminated_quote_is_passthrough(self):
# title is a known field but the quoted value has no closing quote;
# _consume_value returns None so the whole string falls into passthrough.
assert scan('title:"abc') == [Passthrough('title:"abc')]
def test_unterminated_bracket_is_passthrough(self):
# created is a known field but the range bracket is never closed;
# _consume_range returns None so the whole string falls into passthrough.
assert scan("created:[2020") == [Passthrough("created:[2020")]
def test_empty_value_at_end_is_passthrough(self):
# created is a known field but there is no value after the colon
# (_consume_value returns None for start >= n), so passthrough.
assert scan("created:") == [Passthrough("created:")]
def test_value_containing_colon(self):
# The bare-word value reader stops at whitespace/paren, not at colon,
# so "2020:30" is consumed as a single value token.
assert scan("created:2020:30") == [FieldValue("created", "2020:30")]
def test_comma_followed_by_unconsumable_value_stops(self):
# A comma followed by whitespace is neither a value-list continuation nor a
# clause separator: the value stops and the comma stays as passthrough.
assert scan("tag:foo, bar") == [
FieldValue("tag", "foo"),
Passthrough(", bar"),
]
def test_bracket_without_to_is_open_upper_bound(self):
# A bracketed value with no TO falls back to (value, "") -> open upper bound.
assert scan("created:[2020]") == [
FieldRange("created", "[", "2020", "", "]"),
]
def test_known_field_name_midword_is_passthrough(self):
# A known field name embedded mid-word is not a field token (the
# word-boundary guard); the whole run stays passthrough.
assert scan("xtag:foo") == [Passthrough("xtag:foo")]
@pytest.mark.search
class TestCommaResolution:
def test_value_list_multi_value_field(self):
toks = resolve_commas(scan("tag:foo,bar"))
assert toks == [FieldValueList("tag", ("foo", "bar"))]
def test_value_list_three(self):
toks = resolve_commas(scan("tag_id:1,2,3"))
assert toks == [FieldValueList("tag_id", ("1", "2", "3"))]
def test_text_field_comma_is_literal(self):
# correspondent is not multi-value: comma stays inside the value.
toks = resolve_commas(scan("correspondent:foo,bar"))
assert toks == [FieldValue("correspondent", "foo,bar")]
def test_clause_separator_before_known_field(self):
toks = resolve_commas(scan("tag:foo,type:bar"))
assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")]
def test_clause_separator_after_range(self):
toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]"))
assert toks == [
FieldRange("created", "[", "2020", "2021", "]"),
Comma(),
FieldRange("added", "[", "2022", "2023", "]"),
]
def test_clause_separator_after_quote(self):
toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]'))
assert toks == [
FieldValue("correspondent", '"A B"'),
Comma(),
FieldRange("created", "[", "2020", "2021", "]"),
]
def test_url_comma_is_literal_passthrough(self):
toks = resolve_commas(scan("http://example.com/a,b"))
assert toks == [Passthrough("http://example.com/a,b")]
def test_non_multi_value_comma_is_literal(self):
# title is not in MULTI_VALUE_FIELDS: comma stays inside the value.
toks = resolve_commas(scan("title:10,20"))
assert toks == [FieldValue("title", "10,20")]
def test_clause_separator_before_known_date_field(self):
# The comma between a bare value and a known date field acts as a
# clause separator; both sides survive as distinct tokens.
toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]"))
assert toks == [
FieldValue("correspondent", "foo"),
Comma(),
FieldRange("created", "[", "2020", "2021", "]"),
]
@pytest.mark.search
class TestTranslateScalar:
@pytest.mark.parametrize(
("field", "value", "expected"),
[
(
"created",
"2020",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
(
"created",
"202003",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
),
(
"created",
"20200115",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
),
(
"created",
"2020-01-15",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
),
(
"created",
"2020-03",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
),
],
)
def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None:
assert translate_scalar(field, value, UTC) == expected
def test_invalid_date_raises(self) -> None:
with pytest.raises(InvalidDateQuery) as exc_info:
translate_scalar("created", "202023", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "202023"
def test_keyword_delegates(self) -> None:
# keyword path produces a half-open range; just assert it is a created range
out = translate_scalar("created", "today", UTC)
assert out.startswith("created:[") and out.endswith("}")
def test_14digit_compact_datetime(self) -> None:
out = translate_scalar("created", "20240115120000", UTC)
assert "20240115120000" not in out
assert out.startswith("created:")
assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]"
def test_14digit_invalid_month_raises(self) -> None:
with pytest.raises(InvalidDateQuery) as exc_info:
translate_scalar("created", "20231300120000", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "20231300120000"
def test_unrecognized_value_raises(self) -> None:
# A value that is not a keyword, digits, ISO date, or compact timestamp
# raises rather than producing invalid Tantivy syntax or silently matching
# nothing.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_scalar("created", "garbage", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "garbage"
@pytest.mark.search
class TestTranslateRange:
@pytest.mark.parametrize(
("lo", "hi", "expected"),
[
("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),
(
"202001",
"202006",
"created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",
),
(
"20200101",
"20201231",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
(
"2020-01-01",
"2020-12-31",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
],
)
def test_absolute_ranges(self, lo, hi, expected):
assert translate_range("created", lo, hi, UTC) == expected
def test_reversed_swaps(self):
assert translate_range("created", "2009", "2005", UTC) == (
"created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"
)
def test_open_upper(self):
out = translate_range("created", "2020", "", UTC)
assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]"
def test_open_lower(self):
out = translate_range("created", "", "2020", UTC)
assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"
def test_invalid_bound_raises(self):
with pytest.raises(InvalidDateQuery) as exc_info:
translate_range("created", "202023", "2025", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "202023"
def test_invalid_high_bound_raises(self):
# Low bound parses, high bound does not -> raise on the high bound.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_range("created", "2020", "garbage", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "garbage"
@pytest.mark.search
class TestTranslateQuery:
@pytest.mark.parametrize(
("raw", "expected"),
[
(
"created:2020",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
("tag:foo,bar", "tag:foo AND tag:bar"),
# 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
("tag:foo,type:bar", "tag:foo AND document_type:bar"),
(
"created:[2020 TO 2021],added:[2022 TO 2023]",
(
"created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
" AND "
"added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}"
),
),
# correspondent is not multi-value: comma stays literal inside the value
("correspondent:foo,bar", "correspondent:foo,bar"),
],
)
def test_golden(self, raw: str, expected: str) -> None:
assert translate_query(raw, UTC) == expected
@pytest.mark.parametrize(
"raw",
[
"created:2020",
"created:202003",
"created:[20200101 TO 20201231]",
"created:[2020-01-01 TO 2020-12-31]",
"created:[2020 to]",
"created:[to 2020]",
"title:x,created:[2020 TO 2021]",
"created:2020 OR foo",
"(created:2020 OR invoice)",
"tag:foo,type:bar",
"bank statement",
],
)
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
translated = translate_query(raw, UTC)
# Must not raise:
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestFieldAliasing:
"""Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path)."""
def test_type_alias(self) -> None:
assert translate_query("type:invoice", UTC) == "document_type:invoice"
def test_path_alias(self) -> None:
assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar"
def test_type_id_alias(self) -> None:
assert translate_query("type_id:5", UTC) == "document_type_id:5"
def test_path_id_alias(self) -> None:
assert translate_query("path_id:7", UTC) == "storage_path_id:7"
def test_clause_separator_plus_alias(self) -> None:
# Comma between known fields acts as AND separator; alias still applied.
assert (
translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar"
)
def test_type_range_alias(self) -> None:
# type is not a date field; range passes through verbatim with alias applied.
assert (
translate_query("type:[2020 TO 2021]", UTC)
== "document_type:[2020 TO 2021]"
)
def test_parse_acceptance_type(self, index: tantivy.Index) -> None:
# Translated output must be accepted by the real Tantivy parser.
translated = translate_query("type:invoice", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
def test_parse_acceptance_path(self, index: tantivy.Index) -> None:
translated = translate_query("path:foo", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
# Freeze time so relative-date tests are deterministic.
_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC)
@pytest.mark.search
class TestRelativeRanges:
"""Relative date-range tokens resolved against a frozen clock."""
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_7_days_to_now(self) -> None:
assert translate_query("added:[-7 days to now]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_1_week_to_now(self) -> None:
assert translate_query("added:[-1 week to now]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_1_month_to_now(self) -> None:
assert translate_query("created:[-1 month to now]", UTC) == (
"created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_1_year_to_now(self) -> None:
assert translate_query("modified:[-1 year to now]", UTC) == (
"modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_3_hours_to_now(self) -> None:
assert translate_query("added:[-3 hours to now]", UTC) == (
"added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_uppercase_units(self) -> None:
assert translate_query("added:[-1 WEEK TO NOW]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_now_minus_7d_compact(self) -> None:
assert translate_query("added:[now-7d TO now]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_reversed_range_swapped(self) -> None:
# now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h
assert translate_query("added:[now+1h TO now-1h]", UTC) == (
"added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]"
)
@pytest.mark.parametrize(
"raw",
[
"added:[-7 days to now]",
"added:[-1 week to now]",
"created:[-1 month to now]",
"modified:[-1 year to now]",
"added:[-3 hours to now]",
"added:[now-7d TO now]",
"added:[now+1h TO now-1h]",
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
translated = translate_query(raw, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestWhooshUnitAbbreviations:
"""
Whoosh's PlusMinus date grammar accepted abbreviated unit spellings
(e.g. "yrs", "mos", "wks", "hrs", "mins", "secs"); saved views/searches
created under the old Whoosh backend can contain those tokens (see
https://github.com/paperless-ngx/paperless-ngx/issues/13482), so the
Tantivy translator must still accept them.
"""
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_999_yrs(self) -> None:
assert translate_query("created:[-999yrs to now]", UTC) == (
"created:[1027-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@pytest.mark.parametrize(
("token", "expected_lo"),
[
("-1y", "2025-03-28T12:00:00Z"),
("-1yr", "2025-03-28T12:00:00Z"),
("-3mos", "2025-12-28T12:00:00Z"),
("-3mo", "2025-12-28T12:00:00Z"),
("-2wks", "2026-03-14T12:00:00Z"),
("-2wk", "2026-03-14T12:00:00Z"),
("-5dys", "2026-03-23T12:00:00Z"),
("-5dy", "2026-03-23T12:00:00Z"),
("-1hrs", "2026-03-28T11:00:00Z"),
("-1hr", "2026-03-28T11:00:00Z"),
("-10mins", "2026-03-28T11:50:00Z"),
("-10min", "2026-03-28T11:50:00Z"),
("-30secs", "2026-03-28T11:59:30Z"),
("-30sec", "2026-03-28T11:59:30Z"),
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_abbreviated_units(self, token: str, expected_lo: str) -> None:
assert translate_query(f"added:[{token} to now]", UTC) == (
f"added:[{expected_lo} TO 2026-03-28T12:00:00Z]"
)
@pytest.mark.parametrize(
"raw",
[
"created:[-999yrs to now]",
"added:[-1y to now]",
"created:[-3mos to now]",
"added:[-2wks to now]",
"added:[-5dys to now]",
"added:[-1hrs to now]",
"added:[-10mins to now]",
"added:[-30secs to now]",
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
translated = translate_query(raw, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestOperatorNormalization:
"""Post-render operator normalization in translate_query."""
def test_spaced_dash_removed(self) -> None:
assert (
translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit"
)
def test_spaced_dash_simple(self) -> None:
assert translate_query("bar - baz", UTC) == "bar baz"
def test_trailing_operator_stripped(self) -> None:
assert translate_query("foo -", UTC) == "foo"
def test_date_range_preserved(self) -> None:
out = translate_query("created:[2020 TO 2021]", UTC)
# Must not corrupt the ISO range
assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
def test_date_scalar_with_or(self) -> None:
out = translate_query("created:2020 OR foo", UTC)
# The created scalar becomes a range; " OR foo" passes through verbatim.
assert out.startswith("created:[")
assert "OR foo" in out
def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None:
translated = translate_query("H52.1 - Kurzsichtigkeit", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None:
translated = translate_query("foo -", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestMultiWordDateKeywords:
"""scan() must consume multi-word date keywords as a single value."""
def test_scan_previous_week_as_single_token(self) -> None:
# "created:previous week" must produce one FieldValue with value "previous week",
# not FieldValue("created","previous") + Passthrough(" week").
toks = scan("created:previous week")
assert toks == [FieldValue("created", "previous week")]
def test_scan_this_month_as_single_token(self) -> None:
toks = scan("added:this month")
assert toks == [FieldValue("added", "this month")]
def test_scan_previous_month_as_single_token(self) -> None:
toks = scan("created:previous month")
assert toks == [FieldValue("created", "previous month")]
def test_scan_this_year_as_single_token(self) -> None:
toks = scan("added:this year")
assert toks == [FieldValue("added", "this year")]
def test_scan_previous_year_as_single_token(self) -> None:
toks = scan("created:previous year")
assert toks == [FieldValue("created", "previous year")]
def test_scan_previous_quarter_as_single_token(self) -> None:
toks = scan("created:previous quarter")
assert toks == [FieldValue("created", "previous quarter")]
def test_quoted_multi_word_keyword_still_works(self) -> None:
# The quoted form must continue to work as before.
toks = scan('created:"previous week"')
assert toks == [FieldValue("created", '"previous week"')]
def test_non_date_field_not_affected(self) -> None:
# "previous" stops at the space for non-date fields; " week" passes through.
toks = scan("correspondent:previous week")
assert toks == [
FieldValue("correspondent", "previous"),
Passthrough(" week"),
]
@pytest.mark.search
class TestKeywordDateResolution:
"""Relative date keywords resolve to exact ISO ranges against a frozen clock.
Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month,
quarter and year rollovers are all exercised by a single anchor.
"""
# created is a DateField: bounds are UTC midnight, no timezone offset.
@pytest.mark.parametrize(
("keyword", "expected"),
[
pytest.param(
"today",
"created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",
id="today",
),
pytest.param(
"yesterday",
"created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",
id="yesterday",
),
pytest.param(
"previous week",
"created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",
id="previous-week",
),
pytest.param(
"this month",
"created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",
id="this-month",
),
pytest.param(
"previous month",
"created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",
id="previous-month",
),
pytest.param(
"this year",
"created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",
id="this-year",
),
pytest.param(
"previous year",
"created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
id="previous-year",
),
pytest.param(
"previous quarter",
"created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
id="previous-quarter",
),
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_date_only_field_keyword_ranges(
self,
keyword: str,
expected: str,
) -> None:
assert translate_query(f"created:{keyword}", UTC) == expected
# added is a DateTimeField: local-tz midnight converted to UTC. Tokyo
# (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day
# before, so this also exercises the local-midnight offset path.
@pytest.mark.parametrize(
("keyword", "expected"),
[
pytest.param(
"today",
"added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",
id="today",
),
pytest.param(
"yesterday",
"added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",
id="yesterday",
),
pytest.param(
"previous week",
"added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",
id="previous-week",
),
pytest.param(
"this month",
"added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",
id="this-month",
),
pytest.param(
"previous month",
"added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",
id="previous-month",
),
pytest.param(
"this year",
"added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",
id="this-year",
),
pytest.param(
"previous year",
"added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",
id="previous-year",
),
pytest.param(
"previous quarter",
"added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",
id="previous-quarter",
),
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_datetime_field_keyword_ranges_local_tz(
self,
keyword: str,
expected: str,
) -> None:
assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected
@pytest.mark.search
class TestISODatetimeBounds:
"""Full ISO datetime tokens in range bounds must be parsed directly."""
def test_translate_range_iso_bounds_passthrough(self) -> None:
# Already-ISO datetime bounds must pass through as-is (exact instant).
result = translate_range(
"created",
"2020-01-01T00:00:00Z",
"2021-01-01T00:00:00Z",
UTC,
)
assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
def test_translate_query_iso_range_preserved(self) -> None:
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
assert translate_query(q, UTC) == q
def test_translate_query_comma_separated_iso_ranges(self) -> None:
q = (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
result = translate_query(q, UTC)
assert result == (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
" AND "
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
result = translate_query("schäfersee,created:previous year", UTC)
assert result == (
"schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}"
)
def test_invalid_iso_datetime_raises(self) -> None:
# A token with "T" that is not valid ISO datetime -> raise.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_range(
"created",
"2020-01-01T99:00:00Z",
"2021-01-01T00:00:00Z",
UTC,
)
assert exc_info.value.field == "created"
assert exc_info.value.value == "2020-01-01T99:00:00Z"
def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None:
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
translated = translate_query(q, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None:
q = (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
translated = translate_query(q, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@@ -0,0 +1,184 @@
"""Search text is held in one Unicode normal form on both sides.
A bigram is a pair of codepoints, so decomposed and composed spellings of
the same Japanese word produce different bigrams. Unless the indexed text
and the query string are normalized the same way, a document written one
way is invisible to a query written the other.
"""
from __future__ import annotations
import unicodedata
from typing import TYPE_CHECKING
import pytest
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.search._query import normalize_search_text
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# がっこうの書類 ("school documents"). The が is precomposed in NFC and
# か + U+3099 in NFD, so the two spellings differ by one codepoint.
_NFC = "がっこうの書類"
_NFD = unicodedata.normalize("NFD", _NFC)
# The bare word, for fielded queries against a name or filename.
_NFC_WORD = "がっこう"
_NFD_WORD = unicodedata.normalize("NFD", _NFC_WORD)
class TestTheNormalizer:
def test_it_composes_decomposed_kana(self) -> None:
"""
GIVEN:
- The same word spelled decomposed and composed
WHEN:
- Each is normalized
THEN:
- Both become the composed spelling. The inputs really do
differ, so the fixture is not vacuous
"""
assert _NFD != _NFC
assert normalize_search_text(_NFD) == _NFC
assert normalize_search_text(_NFC) == _NFC
def test_it_leaves_halfwidth_katakana_alone(self) -> None:
"""
GIVEN:
- A halfwidth katakana word carrying a voiced sound mark
WHEN:
- It is normalized
THEN:
- It is unchanged. Halfwidth katakana has no precomposed
voiced form, which is why _CJK_RE still has to list the
marks rather than rely on this
"""
assert normalize_search_text("パン") == "パン"
class TestEitherSpellingFindsEither:
@pytest.mark.parametrize(
"content",
[
pytest.param(_NFC, id="composed_document"),
pytest.param(_NFD, id="decomposed_document"),
],
)
@pytest.mark.parametrize(
"query",
[
pytest.param("がっこう", id="composed_query"),
pytest.param(
unicodedata.normalize("NFD", "がっこう"),
id="decomposed_query",
),
],
)
def test_a_document_is_found_whichever_way_each_side_is_spelled(
self,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
content: str,
query: str,
) -> None:
"""
GIVEN:
- A document holding the word inside a longer unspaced run,
spelled composed or decomposed, and an unrelated document
WHEN:
- The word is searched, spelled composed or decomposed
THEN:
- It matches in all four combinations. Without normalizing
both sides, the decomposed run splits at the combining mark
and the two spellings produce different bigrams
"""
match = index_document(title="A", content=content)
index_document(title="B", content="invoice only")
assert matched_ids(query) == {match.pk}
class TestEverySearchableFieldIsNormalized:
"""The query side is normalized in _parse_query, so every searchable
field has to be normalized on the way in as well.
A field left out is worse than normalizing nothing: both sides used to
be decomposed and matched each other, so normalizing only the query
turns a working search into no results. original_filename is the one
most likely to hold NFD in practice, since macOS stores filenames
decomposed.
"""
@pytest.mark.parametrize(
"field",
[
pytest.param("title", id="title"),
pytest.param("content", id="content"),
pytest.param("original_filename", id="original_filename"),
pytest.param("correspondent", id="correspondent"),
pytest.param("document_type", id="document_type"),
pytest.param("storage_path", id="storage_path"),
pytest.param("tag", id="tag"),
pytest.param("custom_fields.value", id="custom_field"),
],
)
def test_a_composed_query_finds_a_decomposed_value(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
field: str,
) -> None:
"""
GIVEN:
- A document carrying a decomposed Japanese word in one
searchable field, and an unrelated document
WHEN:
- The composed spelling is searched, fielded to that field
THEN:
- The document matches. The query is normalized either way, so
a field left unnormalized on the way in can never be found
"""
kwargs: dict[str, object] = {"title": "A", "content": "invoice"}
if field == "correspondent":
kwargs["correspondent"] = Correspondent.objects.create(name=_NFD_WORD)
elif field == "document_type":
kwargs["document_type"] = DocumentType.objects.create(name=_NFD_WORD)
elif field == "storage_path":
kwargs["storage_path"] = StoragePath.objects.create(
name=_NFD_WORD,
path="archive/",
)
elif field in {"title", "content", "original_filename"}:
kwargs[field] = _NFD_WORD
doc = index_document(**kwargs)
if field == "tag":
doc.tags.add(Tag.objects.create(name=_NFD_WORD))
elif field == "custom_fields.value":
CustomFieldInstance.objects.create(
document=doc,
field=CustomField.objects.create(
name="Note",
data_type=CustomField.FieldDataType.STRING,
),
value_text=_NFD_WORD,
)
# The relations above are attached after the factory built the
# document, so the index needs the newer state.
backend.add_or_update(doc)
index_document(title="B", content="invoice only")
assert matched_ids(f"{field}:{_NFC_WORD}") == {doc.pk}
@@ -0,0 +1,127 @@
"""_widen_leaf, the hook that adds a CJK alternative, a fuzzy one, or both.
The alternatives are independent: a leaf can qualify for either, and a
negated leaf keeps its CJK side while losing its fuzzy one.
"""
from __future__ import annotations
import pytest
import whoosh_compat.ast as wc_ast
from documents.search._query import _cjk_alternative
from documents.search._query import _fuzzy_alternative
from documents.search._query import _widen_leaf
from documents.tests.search._ast_helpers import NOTES
from documents.tests.search._ast_helpers import content
pytestmark = pytest.mark.search
def _widen(
leaf: wc_ast.Term,
*,
fuzzy: bool = True,
negated: frozenset[int] = frozenset(),
) -> wc_ast.Node:
return _widen_leaf(leaf, fuzzy=fuzzy, negated=negated)
class TestWhichAlternativesAreAdded:
def test_a_latin_leaf_gains_only_the_fuzzy_side(self) -> None:
"""
GIVEN:
- A latin term, which has no CJK in it
WHEN:
- The hook runs with fuzzy on
THEN:
- The Or holds the leaf and the boosted fuzzy alternative,
and nothing else
"""
leaf = content("invoice")
assert _widen(leaf) == wc_ast.Or(
children=(
leaf,
wc_ast.Boosted(child=_fuzzy_alternative(leaf), boost=0.1),
),
)
def test_a_cjk_leaf_gains_only_the_cjk_side_when_fuzzy_is_off(self) -> None:
"""
GIVEN:
- A CJK term
WHEN:
- The hook runs with fuzzy off
THEN:
- Only the bigram alternative is added, which is exactly what
the CJK work shipped
"""
leaf = content("東京")
assert _widen(leaf, fuzzy=False) == wc_ast.Or(
children=(leaf, _cjk_alternative(leaf)),
)
def test_a_cjk_leaf_gains_only_its_bigram_side(self) -> None:
"""
GIVEN:
- A CJK term
WHEN:
- The hook runs with fuzzy on
THEN:
- The Or holds the leaf and the bigram alternative, and no
fuzzy one. A prefix Fuzzy over a whole unspaced run matches
any run within one edit of its start, and _cjk_alternative
already supplies the in-run recall
"""
leaf = content("東京")
assert _fuzzy_alternative(leaf) is None
assert _widen(leaf) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf)))
def test_a_negated_leaf_keeps_cjk_and_loses_fuzzy(self) -> None:
"""
GIVEN:
- A CJK term whose id is in the negated set
WHEN:
- The hook runs with fuzzy on
THEN:
- The bigram alternative survives and the fuzzy one does not.
NOT X should exclude what X matches, which needs the bigram
side, but prefix fuzzy matching would exclude far more
"""
leaf = content("東京")
assert _widen(leaf, negated=frozenset({id(leaf)})) == wc_ast.Or(
children=(leaf, _cjk_alternative(leaf)),
)
def test_a_negated_latin_leaf_is_returned_unchanged(self) -> None:
"""
GIVEN:
- A latin term whose id is in the negated set
WHEN:
- The hook runs with fuzzy on
THEN:
- The leaf itself comes back. It qualifies for no alternative
at all, so there is no Or to build
"""
leaf = content("tax")
assert _widen(leaf, negated=frozenset({id(leaf)})) is leaf
def test_a_leaf_outside_the_default_fields_is_returned_unchanged(self) -> None:
"""
GIVEN:
- A term on a JSON subpath field, which is not one of the five
default search fields
WHEN:
- The hook runs with fuzzy on
THEN:
- The leaf comes back untouched. Widening is scoped to the
default search fields, as it was for CJK
"""
leaf = wc_ast.Term(field=NOTES, text="invoice")
assert _widen(leaf) is leaf
+4 -4
View File
@@ -16,11 +16,11 @@ from documents.models import Document
from documents.models import Tag
from documents.search import get_backend
from documents.search import reset_backend
from documents.tests.factories import DocumentFactory
from documents.tests.factories import TagFactory
from documents.tests.factories import UserFactory
from documents.tests.utils import DirectoriesMixin
from paperless.admin import PaperlessUserAdmin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
@pytest.fixture
+114 -5
View File
@@ -3,7 +3,6 @@ from io import BytesIO
from pathlib import Path
from unittest.mock import patch
from django.contrib.auth.models import User
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import override_settings
from PIL import Image
@@ -11,10 +10,11 @@ from PIL.PngImagePlugin import PngInfo
from rest_framework import status
from rest_framework.test import APITestCase
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response
from paperless.models import ApplicationConfiguration
from paperless.models import ColorConvertChoices
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
class TestApiAppConfig(DirectoriesMixin, APITestCase):
@@ -23,7 +23,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=user)
def test_api_get_config(self) -> None:
@@ -74,6 +74,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
"barcode_enable_tag": None,
"barcode_tag_mapping": None,
"barcode_tag_split": None,
"barcode_store_values": None,
"remote_ocr_engine": None,
"remote_ocr_api_key": None,
"remote_ocr_endpoint": None,
@@ -81,6 +82,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
"ai_enabled": None,
"llm_embedding_backend": None,
"llm_embedding_model": None,
"llm_embedding_api_key": None,
"llm_embedding_endpoint": None,
"llm_embedding_chunk_size": None,
"llm_context_size": None,
@@ -194,6 +196,70 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
self.assertEqual(config.language, None)
self.assertEqual(config.barcode_tag_mapping, None)
def test_api_update_config_json_objects(self) -> None:
"""
GIVEN:
- API request to update app config with JSON objects for the
user_args and barcode_tag_mapping JSONFields
WHEN:
- API is called
THEN:
- Correct HTTP response
- Both objects are stored as sent
"""
user_args = {"unpaper_args": "--pre-rotate 90", "jobs": 2}
barcode_tag_mapping = {"TAG:(.*)": "\\g<1>", "ASN12.*": ""}
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps(
{
"user_args": json.dumps(user_args),
"barcode_tag_mapping": json.dumps(barcode_tag_mapping),
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
config = ApplicationConfiguration.objects.first()
assert config is not None
self.assertEqual(config.user_args, user_args)
self.assertEqual(config.barcode_tag_mapping, barcode_tag_mapping)
def test_api_update_config_rejects_invalid_json_objects(self) -> None:
"""
GIVEN:
- API request to update app config with a JSON-encoded value that
is not an object for user_args or barcode_tag_mapping, or a
barcode_tag_mapping with a non-string substitute
WHEN:
- API is called
THEN:
- Request is rejected with a 400 naming the problem
- Config is not updated
"""
not_objects = (True, 1, [1, 2, 3], "not a dict")
cases = [
(field, value, b"must be a JSON object")
for field in ("user_args", "barcode_tag_mapping")
for value in not_objects
]
cases += [
("barcode_tag_mapping", {"TAG:(.*)": 5}, b"values must be strings"),
("barcode_tag_mapping", {"TAG:(.*)": None}, b"values must be strings"),
]
for field, value, expected_message in cases:
with self.subTest(field=field, value=value):
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps({field: json.dumps(value)}),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
config = ApplicationConfiguration.objects.first()
assert config is not None
self.assertIsNone(getattr(config, field))
def test_api_replace_app_logo(self) -> None:
"""
GIVEN:
@@ -203,7 +269,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
THEN:
- old app_logo file is deleted
"""
admin = User.objects.create_superuser(username="admin")
admin = UserFactory(username="admin", superuser=True)
self.client.force_login(user=admin)
response = self.client.get("/logo/")
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
@@ -858,6 +924,49 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
self.assertEqual(ApplicationConfiguration.objects.count(), 1)
def test_update_llm_embedding_api_key(self) -> None:
"""
GIVEN:
- Existing config with llm_embedding_api_key specified
WHEN:
- API to update llm_embedding_api_key is called with all *s
- API to update llm_embedding_api_key is called with empty string
THEN:
- llm_embedding_api_key is unchanged
- llm_embedding_api_key is set to None
"""
config = ApplicationConfiguration.objects.first()
assert config is not None
config.llm_embedding_api_key = "1234567890"
config.save()
# Test with all *
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps(
{
"llm_embedding_api_key": "*" * 32,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
config.refresh_from_db()
self.assertEqual(config.llm_embedding_api_key, "1234567890")
# Test with empty string
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps(
{
"llm_embedding_api_key": "",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
config.refresh_from_db()
self.assertEqual(config.llm_embedding_api_key, None)
def test_update_llm_api_key(self) -> None:
"""
GIVEN:
+42 -6
View File
@@ -4,7 +4,6 @@ import json
import shutil
import zipfile
from django.contrib.auth.models import User
from django.test import override_settings
from django.utils import timezone
from rest_framework import status
@@ -13,9 +12,11 @@ from rest_framework.test import APITestCase
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import SampleDirMixin
from documents.tests.utils import read_streaming_response
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_global
class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -24,7 +25,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
self.doc1 = Document.objects.create(title="unrelated", checksum="A")
@@ -165,7 +166,15 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
),
content_type="application/json",
)
response.close()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response["Content-Type"], "application/zip")
with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
self.assertEqual(zipf.infolist()[0].compress_type, zipfile.ZIP_LZMA)
with self.doc2.source_file as f:
self.assertEqual(f.read(), zipf.read("2021-01-01 document A.pdf"))
@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
def test_formatted_download_originals(self) -> None:
@@ -325,7 +334,8 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
)
def test_download_insufficient_permissions(self) -> None:
user = User.objects.create_user(username="temp_user")
user = UserFactory(username="temp_user")
grant_global(user, "view_document")
self.client.force_authenticate(user=user)
self.doc2.owner = self.user
@@ -339,3 +349,29 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.content, b"Insufficient permissions")
def test_bad_search_query_returns_400(self) -> None:
"""
GIVEN:
- Bulk download request selects documents via a saved-search
query filter
WHEN:
- The query contains a malformed field value (an invalid date)
THEN:
- The response is a 400 naming the bad value, exactly like the
search list endpoint, never a 500
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"all": True,
"filters": {"query": "added:notadate"},
"content": "originals",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"notadate", response.content)
+641 -47
View File
@@ -2,27 +2,30 @@ import json
from unittest import mock
from auditlog.models import LogEntry
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from guardian.shortcuts import assign_perm
from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
class TestBulkEditAPI(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
user = UserFactory(username="temp_admin", superuser=True)
self.user = user
self.client.force_authenticate(user=user)
@@ -284,9 +287,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
m,
) -> None:
self.setup_mock(m, "modify_custom_fields")
user = User.objects.create_user(username="doc-owner")
user.user_permissions.add(Permission.objects.get(codename="change_document"))
other_user = User.objects.create_user(username="other-user")
user = UserFactory(username="doc-owner")
grant_global(user, "change_document")
other_user = UserFactory(username="other-user")
source_doc = Document.objects.create(
checksum="source",
title="Source",
@@ -520,6 +523,30 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(len(kwargs), 0)
@mock.patch("documents.views.bulk_edit.delete")
def test_delete_documents_endpoint_with_excluded_documents(self, m) -> None:
self.setup_mock(m, "delete")
response = self.client.post(
"/api/documents/delete/",
json.dumps(
{
"all": True,
"excluded_documents": [
self.doc2.id,
self.doc3.id,
self.doc4.id,
self.doc5.id,
],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(len(kwargs), 0)
@mock.patch("documents.views.bulk_edit.reprocess")
def test_reprocess_documents_endpoint(self, m) -> None:
self.setup_mock(m, "reprocess")
@@ -691,6 +718,26 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"documents is required unless all is true", response.content)
def test_api_rejects_excluded_documents_unless_all_is_true(self) -> None:
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id],
"excluded_documents": [self.doc2.id],
"method": "set_storage_path",
"parameters": {"storage_path": self.sp1.id},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(
b"excluded_documents is only supported when all is true",
response.content,
)
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
def test_api_bulk_edit_with_all_true_resolves_documents_from_filters(
self,
@@ -717,13 +764,34 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
def test_api_bulk_edit_with_all_true_excludes_documents(self, m) -> None:
self.setup_mock(m, "set_storage_path")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"all": True,
"excluded_documents": [self.doc2.id, self.doc4.id],
"method": "set_storage_path",
"parameters": {"storage_path": self.sp1.id},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
self.assertCountEqual(args[0], [self.doc1.id, self.doc3.id, self.doc5.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
def test_api_bulk_edit_with_all_true_resolves_owned_duplicates(self, m) -> None:
self.setup_mock(m, "set_storage_path")
user = User.objects.create_user(username="duplicate-owner")
user.user_permissions.add(
Permission.objects.get(codename="change_document"),
)
user = UserFactory(username="duplicate-owner")
grant_global(user, "change_document")
first_duplicate = Document.objects.create(
checksum="owned-duplicate",
title="First duplicate",
@@ -767,6 +835,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
for filters in (
{"text": "new doc 2017-03-16"},
{"title_search": "apple"},
{"title_search": "apple", "query": "created:[-1 year to now]"},
):
with self.subTest(filters=filters):
get_backend.return_value.search_ids.return_value = [self.doc2.id]
@@ -818,7 +887,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
"all": True,
"filters": {
"text": "B",
"query": "c1",
"title_search": "c1",
},
"method": "set_storage_path",
"parameters": {"storage_path": self.sp1.id},
@@ -1077,6 +1146,33 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
],
)
def test_api_selection_data_with_excluded_documents(self) -> None:
response = self.client.post(
"/api/documents/selection_data/",
json.dumps(
{
"all": True,
"excluded_documents": [self.doc2.id],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertCountEqual(
response.data["selected_correspondents"],
[
{"id": self.c1.id, "document_count": 0},
{"id": self.c2.id, "document_count": 1},
],
)
self.assertCountEqual(
response.data["selected_tags"],
[
{"id": self.t1.id, "document_count": 1},
{"id": self.t2.id, "document_count": 2},
],
)
def test_api_selection_data_requires_view_permission(self) -> None:
self.doc2.owner = self.user
self.doc2.save()
@@ -1084,6 +1180,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
user1 = User.objects.create(username="user1")
self.client.force_authenticate(user=user1)
grant_object(user1, self.doc2, "view_document")
response = self.client.post(
"/api/documents/selection_data/",
json.dumps({"documents": [self.doc2.id]}),
@@ -1091,7 +1189,16 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.content, b"Insufficient permissions")
grant_global(user1, "view_document")
user1 = User.objects.get(pk=user1.pk)
self.client.force_authenticate(user=user1)
response = self.client.post(
"/api/documents/selection_data/",
json.dumps({"documents": [self.doc2.id]}),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions(self, m) -> None:
@@ -1152,6 +1259,150 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertIn(b"set_permissions not specified", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_rejects_malformed_set_permissions(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit where set_permissions is not an
object, has a non-list users value, or has an unknown action
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400 describing the problem
- set_permissions is not called
"""
self.setup_mock(m, "set_permissions")
for set_permissions, expected_message in (
(False, b"Expected a dictionary"),
({"view": {"users": False}}, b"Expected a list"),
({"not_a_real_action": {"users": [1]}}, b"Unknown permission action"),
):
with self.subTest(set_permissions=set_permissions):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {"set_permissions": set_permissions},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_rejects_invalid_owner(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit with an owner that is a nonexistent
id, a boolean, a list, a dict, or a non-numeric string
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- set_permissions is not called
"""
self.setup_mock(m, "set_permissions")
for bad_owner in (
999999,
True,
["not", "an", "id"],
{"nested": "dict"},
"not-a-number",
):
with self.subTest(owner=bad_owner):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {
"set_permissions": {
"view": {"users": [self.user.id]},
},
"owner": bad_owner,
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"Specified owner cannot be found", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_null_is_a_noop(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit with set_permissions null and an
owner, i.e. a request that only changes the owner
WHEN:
- API to bulk edit is called
THEN:
- Request succeeds
- set_permissions receives no users or groups for any action
"""
self.setup_mock(m, "set_permissions")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {
"set_permissions": None,
"owner": self.user.id,
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(
m.call_args.kwargs["set_permissions"],
{"view": {}, "change": {}},
)
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_passes_validated_owner_id(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit with the owner id given as a string
WHEN:
- API to bulk edit is called
THEN:
- set_permissions receives the owner as an integer id
"""
self.setup_mock(m, "set_permissions")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {
"set_permissions": {"view": {"users": [self.user.id]}},
"owner": str(self.user.id),
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["owner"], self.user.id)
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_merge(self, m) -> None:
self.setup_mock(m, "set_permissions")
@@ -1282,12 +1533,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc1.owner = User.objects.get(username="temp_admin")
self.doc1.save()
user1 = User.objects.create(username="user1")
user1.user_permissions.add(*Permission.objects.all())
grant_all_global(user1)
user1.save()
self.client.force_authenticate(user=user1)
permissions = {
"owner": user1.id,
"view": {"users": [user1.id]},
}
response = self.client.post(
@@ -1336,8 +1587,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc1.owner = User.objects.get(username="temp_admin")
self.doc1.save()
user1 = User.objects.create(username="user1")
assign_perm("view_document", user1, self.doc1)
user1.user_permissions.add(*Permission.objects.all())
grant_object(user1, self.doc1, "view_document")
grant_all_global(user1)
user1.save()
self.client.force_authenticate(user=user1)
@@ -1358,7 +1609,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
m.assert_not_called()
self.assertEqual(response.content, b"Insufficient permissions")
assign_perm("change_document", user1, self.doc1)
grant_object(user1, self.doc1, "change_document")
response = self.client.post(
"/api/documents/bulk_edit/",
@@ -1423,14 +1674,182 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
response = self.client.post(
"/api/documents/rotate/",
json.dumps(
{
"documents": [self.doc2.id, self.doc3.id],
"degrees": 45,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"degrees must be a multiple of 90", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.rotate")
def test_bulk_edit_rotate_rejects_invalid_degrees(self, m) -> None:
"""
GIVEN:
- A legacy rotate bulk edit with degrees that are null, not an
integer, or not a multiple of 90
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- rotate is not called
"""
self.setup_mock(m, "rotate")
for degrees, expected_message in (
(None, b"invalid rotation degrees"),
(True, b"invalid rotation degrees"),
("foo", b"invalid rotation degrees"),
(90.5, b"invalid rotation degrees"),
(45, b"degrees must be a multiple of 90"),
):
with self.subTest(degrees=degrees):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "rotate",
"parameters": {"degrees": degrees},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.rotate")
def test_bulk_edit_rotate_passes_integer_degrees(self, m) -> None:
"""
GIVEN:
- A legacy rotate bulk edit with degrees given as a string
WHEN:
- API to bulk edit is called
THEN:
- rotate receives the degrees as an integer
"""
self.setup_mock(m, "rotate")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "rotate",
"parameters": {"degrees": "-90"},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["degrees"], -90)
@mock.patch("documents.serialisers.bulk_edit.split")
def test_bulk_edit_split_rejects_invalid_pages(self, m) -> None:
"""
GIVEN:
- A legacy split bulk edit of a 5 page document with pages that
are not a string, not numeric, zero, a reversed range, or past
the last page
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- split is not called
"""
self.setup_mock(m, "split")
for pages in (None, "", "a", "1-2-3", "0", "3-1", "1-6", "1-5000000"):
with self.subTest(pages=pages):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {"pages": pages},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"invalid pages specified", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.split")
def test_bulk_edit_split_rejects_unknown_page_count(self, m) -> None:
"""
GIVEN:
- A legacy split bulk edit of a document without a page count
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- split is not called
"""
self.setup_mock(m, "split")
for pages in ("1", "1-5000000"):
with self.subTest(pages=pages):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id],
"method": "split",
"parameters": {"pages": pages},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"document page count is unknown", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.split")
def test_bulk_edit_split_parses_pages(self, m) -> None:
"""
GIVEN:
- A legacy split bulk edit with single pages and ranges
WHEN:
- API to bulk edit is called
THEN:
- split receives one page list per comma separated group
"""
self.setup_mock(m, "split")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {"pages": "1,2-4,5"},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]])
@mock.patch("documents.views.bulk_edit.rotate")
def test_rotate_insufficient_permissions(self, m) -> None:
self.doc1.owner = User.objects.get(username="temp_admin")
self.doc1.save()
user1 = User.objects.create(username="user1")
user1.user_permissions.add(*Permission.objects.all())
grant_all_global(user1)
user1.save()
self.client.force_authenticate(user=user1)
@@ -1491,7 +1910,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc1.owner = User.objects.get(username="temp_admin")
self.doc1.save()
user1 = User.objects.create(username="user1")
user1.user_permissions.add(*Permission.objects.all())
grant_all_global(user1)
user1.save()
self.client.force_authenticate(user=user1)
@@ -1530,11 +1949,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
@mock.patch("documents.views.bulk_edit.merge")
def test_merge_and_delete_requires_change_permission(self, m) -> None:
self.setup_mock(m, "merge")
user = User.objects.create_user(username="no-change")
user.user_permissions.add(
Permission.objects.get(codename="add_document"),
Permission.objects.get(codename="delete_document"),
)
user = UserFactory(username="no-change")
grant_global(user, "add_document", "delete_document")
self.client.force_authenticate(user=user)
response = self.client.post(
@@ -1636,6 +2052,43 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_legacy_bulk_edit_rejects_out_of_bounds_pdf_doc_index(self) -> None:
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "edit_pdf",
"parameters": {
"operations": [{"page": 1, "doc": 2**32}],
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"doc index is out of bounds", response.content)
def test_legacy_bulk_edit_rejects_empty_pdf_operations(self) -> None:
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "edit_pdf",
"parameters": {"operations": []},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(
response.json(),
{"operations": ["This list may not be empty."]},
)
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf(self, m) -> None:
self.setup_mock(m, "edit_pdf")
@@ -1686,6 +2139,13 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"Expected a list of items", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
{"documents": [self.doc2.id], "operations": []},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
response = self.client.post(
"/api/documents/edit_pdf/",
json.dumps(
@@ -1697,7 +2157,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"invalid operation entry", response.content)
self.assertIn(b"Expected a dictionary", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1710,7 +2170,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"page must be an integer", response.content)
self.assertIn(b"valid integer is required", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1723,7 +2183,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"rotate must be an integer", response.content)
self.assertIn(b"valid integer is required", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1736,7 +2196,27 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"doc must be an integer", response.content)
self.assertIn(b"valid integer is required", response.content)
# A negative index fails the field's min_value before the
# object-level bound against len(operations) is checked
for doc_index, expected_message in (
(-1, b"greater than or equal to 0"),
(2**32, b"doc index is out of bounds"),
):
with self.subTest(doc_index=doc_index):
response = self.client.post(
"/api/documents/edit_pdf/",
json.dumps(
{
"documents": [self.doc2.id],
"operations": [{"page": 1, "doc": doc_index}],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1744,7 +2224,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
{
"documents": [self.doc2.id],
"update_document": True,
"operations": [{"page": 1, "doc": 1}, {"page": 2, "doc": 2}],
"operations": [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}],
},
),
content_type="application/json",
@@ -1769,6 +2249,72 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"Invalid source_mode", response.content)
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf_rejects_invalid_operation_values(self, m) -> None:
"""
GIVEN:
- An edit_pdf operation with a non-positive page, a boolean in
place of an integer, or a rotation that is not a multiple of 90
WHEN:
- API to edit the PDF is called
THEN:
- API returns HTTP 400 with the field error
- edit_pdf is not called
"""
self.setup_mock(m, "edit_pdf")
for operation, expected_message in (
({"page": 0}, b"greater than or equal to 1"),
({"page": True}, b"valid integer is required"),
({"page": 1, "rotate": True}, b"valid integer is required"),
({"page": 1, "doc": True}, b"valid integer is required"),
({"page": 1, "rotate": 45}, b"rotate must be a multiple of 90"),
):
with self.subTest(operation=operation):
response = self.client.post(
"/api/documents/edit_pdf/",
json.dumps(
{
"documents": [self.doc2.id],
"operations": [operation],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
m.assert_not_called()
def test_legacy_bulk_edit_keys_pdf_operation_errors_under_operations(
self,
) -> None:
"""
GIVEN:
- A legacy bulk_edit edit_pdf request with an invalid operation
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- The error is keyed under operations and the operation index,
matching the edit_pdf endpoint
"""
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "edit_pdf",
"parameters": {"operations": [{"page": 1, "rotate": 45}]},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(
response.json(),
{"operations": {"0": {"rotate": ["rotate must be a multiple of 90"]}}},
)
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf_page_out_of_bounds(self, m) -> None:
self.setup_mock(m, "edit_pdf")
@@ -1791,7 +2337,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc1.owner = User.objects.get(username="temp_admin")
self.doc1.save()
user1 = User.objects.create(username="user1")
user1.user_permissions.add(*Permission.objects.all())
grant_all_global(user1)
user1.save()
self.client.force_authenticate(user=user1)
@@ -1826,7 +2372,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf_update_requires_change_permission(self, m) -> None:
self.setup_mock(m, "edit_pdf")
user = User.objects.create_user(username="no-change")
user = UserFactory(username="no-change")
self.client.force_authenticate(user=user)
response = self.client.post(
@@ -1853,11 +2399,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
) -> None:
self.setup_mock(edit_pdf_mock, "edit_pdf")
self.setup_mock(remove_password_mock, "remove_password")
user = User.objects.create_user(username="no-delete")
user.user_permissions.add(
Permission.objects.get(codename="add_document"),
Permission.objects.get(codename="change_document"),
)
user = UserFactory(username="no-delete")
grant_global(user, "add_document", "change_document")
self.client.force_authenticate(user=user)
cases = [
@@ -1944,7 +2487,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc1.owner = User.objects.get(username="temp_admin")
self.doc1.save()
user1 = User.objects.create(username="user1")
user1.user_permissions.add(*Permission.objects.all())
grant_all_global(user1)
user1.save()
self.client.force_authenticate(user=user1)
@@ -1984,7 +2527,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
WHEN:
- API to bulk edit documents is called
THEN:
- Audit log is created
- Audit log is created with the old and new correspondent
"""
LogEntry.objects.all().delete()
response = self.client.post(
@@ -2000,7 +2543,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 1)
entry = LogEntry.objects.get_for_object(self.doc1).get()
self.assertEqual(entry.changes, {"correspondent": [None, self.c2.id]})
@override_settings(AUDIT_LOG_ENABLED=True)
def test_bulk_edit_audit_log_enabled_tags(self) -> None:
@@ -2008,16 +2552,18 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
GIVEN:
- Audit log is enabled
WHEN:
- API to bulk edit tags is called
- API to bulk edit tags is called on an untagged document and a
document with several tags
THEN:
- Audit log is created
- Audit log is created for each document with its full tag list
before and after the edit
"""
LogEntry.objects.all().delete()
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc1.id],
"documents": [self.doc1.id, self.doc4.id],
"method": "modify_tags",
"parameters": {
"add_tags": [self.t1.id],
@@ -2029,18 +2575,32 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 1)
entry = LogEntry.objects.get_for_object(self.doc1).get()
self.assertEqual(entry.changes, {"tags": [[], [self.t1.id]]})
entry = LogEntry.objects.get_for_object(self.doc4).get()
self.assertEqual(
entry.changes,
{"tags": [[self.t1.id, self.t2.id], [self.t1.id]]},
)
@override_settings(AUDIT_LOG_ENABLED=True)
def test_bulk_edit_audit_log_enabled_custom_fields(self) -> None:
"""
GIVEN:
- Audit log is enabled
- A document with two custom fields
WHEN:
- API to bulk edit custom fields is called
- API to bulk edit custom fields is called to add a third
THEN:
- Audit log is created
- Audit log is created with every custom field instance before and
after the edit
- Audit log is created for the new custom field instance
"""
cf3 = CustomField.objects.create(name="cf3", data_type="string")
existing = [
CustomFieldInstance.objects.create(document=self.doc1, field=field)
for field in (self.cf2, cf3)
]
LogEntry.objects.all().delete()
response = self.client.post(
"/api/documents/bulk_edit/",
@@ -2058,4 +2618,38 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2)
added = CustomFieldInstance.objects.get(document=self.doc1, field=self.cf1)
existing_ids = [instance.id for instance in existing]
entry = LogEntry.objects.get_for_object(self.doc1).get()
self.assertEqual(
entry.changes,
{"custom_fields": [existing_ids, [*existing_ids, added.id]]},
)
self.assertEqual(LogEntry.objects.get_for_object(added).count(), 1)
def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None:
"""
GIVEN:
- Bulk edit request selects documents via a saved-search query
filter
WHEN:
- The query contains a malformed field value (an invalid date)
THEN:
- The response is a 400 naming the bad value, exactly like the
search list endpoint, never a 500
"""
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"all": True,
"filters": {"query": "added:notadate"},
"method": "set_storage_path",
"parameters": {"storage_path": self.sp1.id},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"notadate", response.content)
+42 -6
View File
@@ -4,20 +4,22 @@ from typing import TYPE_CHECKING
from unittest import mock
import pytest
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from rest_framework import status
from rest_framework.test import APIClient
from rest_framework.test import APITestCase
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
if TYPE_CHECKING:
from django.contrib.auth.models import User
from pytest_mock import MockerFixture
class TestChatStreamingViewInputValidation(APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
def _mock_ai_enabled(self) -> mock.MagicMock:
@@ -38,6 +40,42 @@ class TestChatStreamingViewInputValidation(APITestCase):
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
def test_answer_is_not_compressed(self) -> None:
"""
GIVEN:
- A client that accepts compressed responses
WHEN:
- It asks the chat endpoint a question
THEN:
- The answer is streamed unencoded, chunk for chunk
The stream compressors buffer, so a compressed answer arrives in one
piece. The view cannot opt out by flagging the request: DRF's request
wrapper proxies reads but keeps writes to itself, so the flag never
reaches the Django request the middleware sees.
"""
chunks = [f"token{i} " for i in range(40)]
with (
mock.patch(
"documents.views.AIConfig",
return_value=self._mock_ai_enabled(),
),
mock.patch(
"documents.views.stream_chat_with_documents",
return_value=iter(chunks),
),
):
resp = self.client.post(
"/api/documents/chat/",
{"q": "What is in my archive?"},
format="json",
HTTP_ACCEPT_ENCODING="gzip, deflate, br, zstd",
)
assert resp.status_code == status.HTTP_200_OK
assert not resp.has_header("Content-Encoding")
assert list(resp.streaming_content) == [c.encode() for c in chunks]
def test_missing_question_is_rejected(self) -> None:
with mock.patch(
"documents.views.AIConfig",
@@ -77,9 +115,7 @@ class TestChatStreamingViewUnrestrictedFlag:
needs to reach the view at all. Model-level only: says nothing
about which documents (if any) this user can actually see.
"""
regular_user.user_permissions.add(
*Permission.objects.filter(codename="view_document"),
)
grant_global(regular_user, "view_document")
return user_client
@pytest.mark.parametrize(
+13 -16
View File
@@ -4,23 +4,24 @@ from unittest import mock
from unittest.mock import ANY
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from guardian.shortcuts import assign_perm
from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/custom_fields/"
def setUp(self) -> None:
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
return super().setUp()
@@ -1174,11 +1175,9 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
def test_documentlink_patch_requires_change_permission_on_target_documents(
self,
) -> None:
source_owner = User.objects.create_user(username="source-owner")
source_owner.user_permissions.add(
Permission.objects.get(codename="change_document"),
)
other_user = User.objects.create_user(username="other-user")
source_owner = UserFactory(username="source-owner")
grant_global(source_owner, "change_document")
other_user = UserFactory(username="other-user")
source_doc = Document.objects.create(
title="Source",
@@ -1221,11 +1220,9 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
def test_documentlink_patch_allowed_with_change_permission_on_target_documents(
self,
) -> None:
source_owner = User.objects.create_user(username="source-owner")
source_owner.user_permissions.add(
Permission.objects.get(codename="change_document"),
)
other_user = User.objects.create_user(username="other-user")
source_owner = UserFactory(username="source-owner")
grant_global(source_owner, "change_document")
other_user = UserFactory(username="other-user")
source_doc = Document.objects.create(
title="Source",
@@ -1244,7 +1241,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
data_type=CustomField.FieldDataType.DOCUMENTLINK,
)
assign_perm("change_document", source_owner, target_doc)
grant_object(source_owner, target_doc, "change_document")
self.client.force_authenticate(user=source_owner)
resp = self.client.patch(
@@ -1337,7 +1334,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
self.assertEqual(results[0]["document_count"], 1)
# Test as user without access to the document
non_superuser = User.objects.create_user(username="non_superuser")
non_superuser = UserFactory(username="non_superuser")
non_superuser.user_permissions.add(
*Permission.objects.exclude(codename="view_global_statistics"),
)
+198 -31
View File
@@ -5,11 +5,10 @@ from typing import TYPE_CHECKING
from unittest import mock
from auditlog.models import LogEntry # type: ignore[import-untyped]
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase as DjangoTestCase
from django.test import override_settings
from django.utils import timezone
from rest_framework import status
from rest_framework.test import APITestCase
@@ -18,10 +17,17 @@ from documents.data_models import DocumentSource
from documents.filters import EffectiveContentFilter
from documents.filters import TitleContentFilter
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response
from documents.models import Note
from documents.models import ShareLink
from documents.versioning import annotate_effective_content
from documents.views import DocumentSelectionMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
if TYPE_CHECKING:
from pathlib import Path
@@ -31,7 +37,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
def _make_pdf_upload(self, name: str = "version.pdf") -> SimpleUploadedFile:
@@ -89,11 +95,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
def test_root_endpoint_returns_403_when_user_lacks_permission(self) -> None:
owner = User.objects.create_user(username="owner")
viewer = User.objects.create_user(username="viewer")
viewer.user_permissions.add(
Permission.objects.get(codename="view_document"),
)
owner = UserFactory(username="owner")
viewer = UserFactory(username="viewer")
grant_global(viewer, "view_document")
root = Document.objects.create(
title="root",
checksum="root",
@@ -283,11 +287,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(mock_backend.add_or_update.call_args[0][0].id, root.id)
def test_delete_version_returns_403_without_permission(self) -> None:
owner = User.objects.create_user(username="owner")
other = User.objects.create_user(username="other")
other.user_permissions.add(
Permission.objects.get(codename="delete_document"),
)
owner = UserFactory(username="owner")
other = UserFactory(username="other")
grant_global(other, "delete_document")
root = Document.objects.create(
title="root",
checksum="root",
@@ -371,11 +373,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertTrue(resp.data["is_root"])
def test_update_version_label_returns_403_without_permission(self) -> None:
owner = User.objects.create_user(username="owner")
other = User.objects.create_user(username="other")
other.user_permissions.add(
Permission.objects.get(codename="change_document"),
)
owner = UserFactory(username="owner")
other = UserFactory(username="other")
grant_global(other, "change_document")
root = Document.objects.create(
title="root",
checksum="root",
@@ -553,11 +553,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
def test_metadata_returns_403_when_user_lacks_permission(self) -> None:
owner = User.objects.create_user(username="owner")
other = User.objects.create_user(username="other")
other.user_permissions.add(
Permission.objects.get(codename="view_document"),
)
owner = UserFactory(username="owner")
other = UserFactory(username="other")
grant_global(other, "view_document")
doc = Document.objects.create(
title="root",
checksum="root",
@@ -653,8 +651,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
def test_update_version_returns_403_without_permission(self) -> None:
owner = User.objects.create_user(username="owner")
other = User.objects.create_user(username="other")
owner = UserFactory(username="owner")
other = UserFactory(username="other")
root = Document.objects.create(
title="root",
checksum="root",
@@ -672,8 +670,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
def test_update_version_requires_global_change_permission(self) -> None:
user = User.objects.create_user(username="add-only")
user.user_permissions.add(Permission.objects.get(codename="add_document"))
user = UserFactory(username="add-only")
grant_global(user, "add_document")
root = Document.objects.create(
title="root",
checksum="root",
@@ -829,6 +827,26 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(resp.data["content"], "v1-content")
def test_page_count_resolves_to_latest_version(self) -> None:
root = DocumentFactory(page_count=2)
DocumentFactory(root_document=root, version_index=1, page_count=1)
unversioned = DocumentFactory(page_count=5)
resp = self.client.get("/api/documents/?fields=id,page_count")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(
{doc["id"]: doc["page_count"] for doc in resp.data["results"]},
{root.id: 1, unversioned.id: 5},
)
resp = self.client.get(f"/api/documents/{root.id}/")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(resp.data["page_count"], 1)
resp = self.client.get(f"/api/documents/{root.id}/?version={root.id}")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(resp.data["page_count"], 2)
def _make_root_with_out_of_order_versions(self) -> tuple[Document, ...]:
"""
A root whose newest version has a *lower* id than an older one, which is
@@ -978,7 +996,7 @@ class TestVersionAwareFilters(DjangoTestCase):
superseded content -- selecting documents the list view, filtered by
the same term, does not show.
"""
user = User.objects.create_superuser(username="bulk_selection")
user = UserFactory(username="bulk_selection", superuser=True)
selected = DocumentSelectionMixin()._resolve_document_ids(
user=user,
@@ -1005,7 +1023,7 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
"Select all matching" reconstructs the document list, which never
contains version documents as rows of their own.
"""
user = User.objects.create_superuser(username="bulk_versions")
user = UserFactory(username="bulk_versions", superuser=True)
root = Document.objects.create(
title="shared-title root",
checksum="bulk-root",
@@ -1030,3 +1048,152 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
)
self.assertEqual(selected, [root.id])
class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
def setUp(self):
super().setUp()
self.user = UserFactory()
grant_all_global(self.user)
self.client.force_authenticate(self.user)
self.root = DocumentFactory(owner=UserFactory())
self.version = DocumentFactory(root_document=self.root, owner=None)
@override_settings(AUDIT_LOG_ENABLED=True)
def test_actions_reject_stale_version_ownership(self):
note = Note.objects.create(document=self.version, note="Version note")
for owner in (None, self.user):
self.version.owner = owner
self.version.save(update_fields=["owner"])
for action in (
"notes",
"suggestions",
"ai_suggestions",
"history",
"share_links",
):
with self.subTest(owner=owner, action=action):
response = self.client.get(
f"/api/documents/{self.version.pk}/{action}/",
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
f"/api/documents/{self.version.pk}/notes/",
{"note": "New note"},
)
self.assertEqual(response.status_code, 403)
response = self.client.delete(
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
"/api/share_links/",
{"document": self.version.pk, "file_version": "original"},
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
"/api/share_link_bundles/",
{"document_ids": [self.version.pk], "file_version": "original"},
format="json",
)
self.assertEqual(response.status_code, 400)
response = self.client.post(
"/api/documents/email/",
{
"documents": [self.version.pk],
"addresses": "recipient@example.com",
"subject": "Version",
"message": "Version",
},
format="json",
)
self.assertEqual(response.status_code, 403)
with (
mock.patch("documents.views.AIConfig") as ai_config,
mock.patch("documents.views.stream_chat_with_documents") as chat,
):
ai_config.return_value.ai_enabled = True
response = self.client.post(
"/api/documents/chat/",
{"q": "Version?", "document_id": self.version.pk},
format="json",
)
self.assertEqual(response.status_code, 403)
chat.assert_not_called()
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
self.assertFalse(ShareLink.objects.exists())
@mock.patch("documents.views.build_share_link_bundle.apply_async")
def test_root_permissions_allow_sharing_a_private_version(self, build_mock):
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
grant_object(self.user, self.root, "view_document", "change_document")
note = Note.objects.create(document=self.version, note="Version note")
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data[0]["id"], note.pk)
response = self.client.post(
"/api/share_links/",
{"document": self.version.pk, "file_version": "original"},
)
self.assertEqual(response.status_code, 201)
self.assertEqual(ShareLink.objects.get().document_id, self.version.pk)
response = self.client.get(f"/api/documents/{self.version.pk}/share_links/")
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
response = self.client.post(
"/api/share_link_bundles/",
{"document_ids": [self.version.pk], "file_version": "original"},
format="json",
)
self.assertEqual(response.status_code, 201)
build_mock.assert_called_once()
def test_root_view_permission_does_not_allow_note_changes(self):
grant_object(self.user, self.root, "view_document")
note = Note.objects.create(document=self.version, note="Version note")
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
self.assertEqual(response.status_code, 200)
response = self.client.post(
f"/api/documents/{self.version.pk}/notes/",
{"note": "New note"},
)
self.assertEqual(response.status_code, 403)
response = self.client.delete(
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, 403)
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
@override_settings(AUDIT_LOG_ENABLED=True)
def test_history_uses_root_ownership(self):
self.root.owner = self.user
self.root.save(update_fields=["owner"])
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
response = self.client.get(f"/api/documents/{self.version.pk}/history/")
self.assertEqual(response.status_code, 200)
def test_selection_data_rejects_stale_version_ownership(self):
for owner in (None, self.user):
self.version.owner = owner
self.version.save(update_fields=["owner"])
with self.subTest(owner=owner):
response = self.client.post(
"/api/documents/selection_data/",
{"documents": [self.version.pk]},
format="json",
)
self.assertEqual(response.status_code, 403)
def test_selection_data_allows_private_version_of_permitted_root(self):
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
grant_object(self.user, self.root, "view_document")
other = DocumentFactory(owner=self.user)
response = self.client.post(
"/api/documents/selection_data/",
{"documents": [self.version.pk, other.pk]},
format="json",
)
self.assertEqual(response.status_code, 200)
+168 -85
View File
@@ -23,7 +23,6 @@ from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import DataError
from django.test import override_settings
from django.utils import timezone
from guardian.shortcuts import assign_perm
from rest_framework import status
from rest_framework.test import APITestCase
@@ -48,18 +47,22 @@ from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals.handlers import run_workflows
from documents.tests.factories import DocumentFactory
from documents.tests.factories import TagFactory
from documents.tests.utils import ConsumeTaskMixin
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
cache.clear()
@@ -357,10 +360,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
with Path(filename).open("wb") as f:
f.write(content)
user1 = User.objects.create_user(username="test1")
user2 = User.objects.create_user(username="test2")
user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user1 = UserFactory(username="test1")
user2 = UserFactory(username="test2")
grant_global(user1, "view_document")
grant_global(user2, "view_document")
self.client.force_authenticate(user2)
@@ -383,7 +386,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
response = self.client.get(f"/api/documents/{doc.pk}/thumb/")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
assign_perm("view_document", user2, doc)
grant_object(user2, doc, "view_document")
response = self.client.get(f"/api/documents/{doc.pk}/download/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -760,8 +763,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
- History is returned
"""
# No auditlog permissions
user = User.objects.create_user(username="test")
user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user = UserFactory(username="test")
grant_global(user, "view_document")
self.client.force_authenticate(user=user)
doc = Document.objects.create(
title="First title",
@@ -776,7 +779,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
# superuser
user.is_superuser = True
user.save()
user2 = User.objects.create_user(username="test2")
user2 = UserFactory(username="test2")
doc2 = Document.objects.create(
title="Second title",
checksum="456",
@@ -1073,11 +1076,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.data["duplicate_documents"], [])
def test_has_duplicates_filter_respects_document_permissions(self) -> None:
owner = User.objects.create_user(username="duplicate-owner")
requester = User.objects.create_user(username="duplicate-requester")
requester.user_permissions.add(
Permission.objects.get(codename="view_document"),
)
owner = UserFactory(username="duplicate-owner")
requester = UserFactory(username="duplicate-requester")
grant_global(requester, "view_document")
visible_document = Document.objects.create(
title="visible document",
checksum="permission-match",
@@ -1096,7 +1097,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
[document["id"] for document in response.data["results"]],
)
assign_perm("view_document", requester, hidden_duplicate)
grant_object(requester, hidden_duplicate, "view_document")
response = self.client.get("/api/documents/?has_duplicates=true")
self.assertIn(
visible_document.id,
@@ -1317,10 +1318,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Owner filters work correctly but still respect permissions
"""
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
grant_global(u1, "view_document")
grant_global(u2, "view_document")
u1_doc1 = Document.objects.create(
title="none1",
@@ -1353,7 +1354,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
)
self.client.force_authenticate(user=u1)
assign_perm("view_document", u1, u2_doc2)
grant_object(u1, u2_doc2, "view_document")
# Will not show any u1 docs or u2_doc1 which isn't shared
response = self.client.get(f"/api/documents/?owner__id__none={u1.id}")
@@ -1400,7 +1401,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
[u1_doc1.id, u1_doc2.id, u2_doc2.id],
)
assign_perm("view_document", u2, u1_doc1)
grant_object(u2, u1_doc1, "view_document")
# Will show only documents shared by user
response = self.client.get(f"/api/documents/?shared_by__id={u1.id}")
@@ -1424,8 +1425,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
- The document is returned exactly once, not once per permission path
(regression test for https://github.com/paperless-ngx/paperless-ngx/issues/13331)
"""
user = User.objects.create_user("user1")
user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user = UserFactory(username="user1")
grant_global(user, "view_document")
group = Group.objects.create(name="group1")
user.groups.add(group)
@@ -1433,7 +1434,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
tag2 = TagFactory()
doc = DocumentFactory(title="shared", owner=user)
doc.tags.add(tag1, tag2)
assign_perm("view_document", group, doc)
grant_object(group, doc, "view_document")
self.client.force_authenticate(user=user)
response = self.client.get(
@@ -1452,11 +1453,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- The document does not appear in their results
"""
owner = User.objects.create_user("owner1")
stranger = User.objects.create_user("stranger1")
stranger.user_permissions.add(
*Permission.objects.filter(codename="view_document"),
)
owner = UserFactory(username="owner1")
stranger = UserFactory(username="stranger1")
grant_global(stranger, "view_document")
DocumentFactory(title="private", owner=owner)
@@ -1474,17 +1473,17 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Only the group member sees the document
"""
owner = User.objects.create_user("owner2")
member = User.objects.create_user("member1")
non_member = User.objects.create_user("nonmember1")
owner = UserFactory(username="owner2")
member = UserFactory(username="member1")
non_member = UserFactory(username="nonmember1")
for u in (member, non_member):
u.user_permissions.add(*Permission.objects.filter(codename="view_document"))
grant_global(u, "view_document")
group = Group.objects.create(name="group2")
member.groups.add(group)
doc = DocumentFactory(title="shared2", owner=owner)
assign_perm("view_document", group, doc)
grant_object(group, doc, "view_document")
self.client.force_authenticate(user=member)
response = self.client.get("/api/documents/")
@@ -1785,8 +1784,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Statistics only include inbox counts for tags accessible by the user
"""
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
inbox_tag_u1 = Tag.objects.create(name="inbox_u1", is_inbox_tag=True, owner=u1)
Tag.objects.create(name="inbox_u2", is_inbox_tag=True, owner=u2)
doc_u1 = Document.objects.create(
@@ -1816,11 +1815,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.data["documents_inbox"], 0)
def test_statistics_with_statistics_permission(self) -> None:
owner = User.objects.create_user("owner")
stats_user = User.objects.create_user("stats-user")
stats_user.user_permissions.add(
Permission.objects.get(codename="view_global_statistics"),
)
owner = UserFactory(username="owner")
stats_user = UserFactory(username="stats-user")
grant_global(stats_user, "view_global_statistics")
inbox_tag = Tag.objects.create(
name="stats_inbox",
@@ -1986,7 +1983,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
def test_upload_insufficient_permissions(self) -> None:
self.client.force_authenticate(user=User.objects.create_user("testuser2"))
self.client.force_authenticate(user=UserFactory(username="testuser2"))
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
@@ -2782,9 +2779,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
mock_get_date_parser.assert_not_called()
def test_saved_views(self) -> None:
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u3 = User.objects.create_user("user3")
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
u3 = UserFactory(username="user3")
view_perm = Permission.objects.get(codename="view_savedview")
change_perm = Permission.objects.get(codename="change_savedview")
@@ -2807,9 +2804,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
sort_field="",
)
assign_perm("view_savedview", u1, v2)
assign_perm("change_savedview", u1, v2)
assign_perm("view_savedview", u1, v3)
grant_object(u1, v2, "view_savedview")
grant_object(u1, v2, "change_savedview")
grant_object(u1, v3, "view_savedview")
self.client.force_authenticate(user=u1)
@@ -3064,7 +3061,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertListEqual(saved_view_settings["sidebar_views_visible_ids"], [v2.id])
def test_saved_view_create_update_patch(self) -> None:
User.objects.create_user("user1")
UserFactory(username="user1")
view = {
"name": "test",
@@ -3127,7 +3124,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
- Display options are updated
- Display fields are validated
"""
User.objects.create_user("user1")
UserFactory(username="user1")
view = {
"name": "test",
@@ -3568,11 +3565,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Notes are neither created nor deleted
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.all())
user1 = UserFactory(username="test1")
grant_all_global(user1)
user1.save()
user2 = User.objects.create_user(username="test2")
user2 = UserFactory(username="test2")
user2.save()
doc = Document.objects.create(
@@ -3592,7 +3589,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(resp.content, b"Insufficient permissions to view notes")
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
assign_perm("view_document", user1, doc)
grant_object(user1, doc, "view_document")
resp = self.client.post(
f"/api/documents/{doc.pk}/notes/",
@@ -3615,6 +3612,47 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.content, b"Insufficient permissions to delete notes")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_notes_require_global_document_permissions(self) -> None:
user = UserFactory(username="note_editor")
grant_global(user, "view_note", "add_note", "delete_note")
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="notes",
owner=user,
)
note = Note.objects.create(note="Existing", document=doc, user=user)
self.client.force_authenticate(user)
response = self.client.get(f"/api/documents/{doc.pk}/notes/")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
grant_global(user, "view_document")
user = User.objects.get(pk=user.pk)
self.client.force_authenticate(user)
response = self.client.get(f"/api/documents/{doc.pk}/notes/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.client.post(
f"/api/documents/{doc.pk}/notes/",
data={"note": "New"},
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
grant_global(user, "change_document")
user = User.objects.get(pk=user.pk)
self.client.force_authenticate(user)
response = self.client.post(
f"/api/documents/{doc.pk}/notes/",
data={"note": "New"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.client.delete(
f"/api/documents/{doc.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_delete_note(self) -> None:
"""
GIVEN:
@@ -3748,12 +3786,12 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
- Unique items are created
- Non-unique items are not allowed
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.filter(codename="add_tag"))
user1 = UserFactory(username="test1")
grant_global(user1, "add_tag")
user1.save()
user2 = User.objects.create_user(username="test2")
user2.user_permissions.add(*Permission.objects.filter(codename="add_tag"))
user2 = UserFactory(username="test2")
grant_global(user2, "add_tag")
user2.save()
# User 1 creates tag 1 owned by user 1 by default
@@ -3808,12 +3846,12 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
- Unique items are created
- Non-unique items are not allowed on update
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.filter(codename="change_tag"))
user1 = UserFactory(username="test1")
grant_global(user1, "change_tag")
user1.save()
user2 = User.objects.create_user(username="test2")
user2.user_permissions.add(*Permission.objects.filter(codename="change_tag"))
user2 = UserFactory(username="test2")
grant_global(user2, "change_tag")
user2.save()
# Create name tag 1 owned by user 1
@@ -3857,6 +3895,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
},
)
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
self.assertEqual(resp.data["document_title"], doc.title)
resp = self.client.post(
"/api/share_links/",
@@ -3867,6 +3906,17 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
},
)
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
self.assertEqual(resp.data["document_title"], doc.title)
response = self.client.get("/api/share_links/", format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 2)
self.assertTrue(
all(
link["document_title"] == doc.title for link in response.data["results"]
),
)
response = self.client.get(
f"/api/documents/{doc.pk}/share_links/",
@@ -3878,6 +3928,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
resp_data = response.json()
self.assertEqual(len(resp_data), 2)
self.assertTrue(
all(link["document_title"] == doc.title for link in resp_data),
)
self.assertGreater(len(resp_data[1]["slug"]), 0)
self.assertIsNone(resp_data[1]["expiration"])
@@ -3903,6 +3956,23 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_order_share_links_by_document_title(self) -> None:
document_zulu = Document.objects.create(title="Zulu")
document_alpha = Document.objects.create(title="Alpha")
ShareLink.objects.create(document=document_zulu, slug="zulu-link")
ShareLink.objects.create(document=document_alpha, slug="alpha-link")
response = self.client.get(
"/api/share_links/?ordering=document__title",
format="json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
[link["document_title"] for link in response.data["results"]],
["Alpha", "Zulu"],
)
def test_share_links_permissions_aware(self) -> None:
"""
GIVEN:
@@ -3912,11 +3982,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Links only shown if user has permissions
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.all())
user1 = UserFactory(username="test1")
grant_all_global(user1)
user1.save()
user2 = User.objects.create_user(username="test2")
user2 = UserFactory(username="test2")
user2.save()
doc = Document.objects.create(
@@ -3936,7 +4006,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(resp.content, b"Insufficient permissions to add share link")
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
assign_perm("change_document", user1, doc)
grant_object(user1, doc, "change_document")
resp = self.client.get(
f"/api/documents/{doc.pk}/share_links/",
@@ -3953,11 +4023,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Share link creation is denied until view permission is granted
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.filter(codename="add_sharelink"))
user1 = UserFactory(username="test1")
grant_global(user1, "add_sharelink")
user1.save()
user2 = User.objects.create_user(username="test2")
user2 = UserFactory(username="test2")
user2.save()
doc = Document.objects.create(
@@ -3979,8 +4049,21 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
)
self.assertEqual(create_resp.status_code, status.HTTP_403_FORBIDDEN)
assign_perm("view_document", user1, doc)
grant_object(user1, doc, "view_document")
create_resp = self.client.post(
"/api/share_links/",
data={
"document": doc.pk,
"file_version": "original",
},
format="json",
)
self.assertEqual(create_resp.status_code, status.HTTP_403_FORBIDDEN)
grant_global(user1, "view_document")
user1 = User.objects.get(pk=user1.pk)
self.client.force_authenticate(user1)
create_resp = self.client.post(
"/api/share_links/",
data={
@@ -4001,11 +4084,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- ASN +1 from user2's doc is returned for user1
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.all())
user1 = UserFactory(username="test1")
grant_all_global(user1)
user1.save()
user2 = User.objects.create_user(username="test2")
user2 = UserFactory(username="test2")
user2.save()
doc1 = Document.objects.create(
@@ -4045,8 +4128,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- ASN 1 is returned
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.all())
user1 = UserFactory(username="test1")
grant_all_global(user1)
user1.save()
doc1 = Document.objects.create(
@@ -4074,7 +4157,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Explicit error is returned
"""
user1 = User.objects.create_superuser(username="test1")
user1 = UserFactory(username="test1", superuser=True)
self.client.force_authenticate(user1)
@@ -4252,8 +4335,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
THEN:
- Error response is returned
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.all())
user1 = UserFactory(username="test1")
grant_all_global(user1)
user1.save()
doc = Document.objects.create(
@@ -4358,7 +4441,7 @@ class TestDocumentApiTagColors(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
@@ -4438,7 +4521,7 @@ class TestDocumentApiCustomFieldsSorting(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
self.doc1 = Document.objects.create(
+8 -8
View File
@@ -2,16 +2,16 @@ import json
import shutil
from unittest import mock
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.core import mail
from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import SampleDirMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -20,7 +20,7 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
self.doc1 = Document.objects.create(
@@ -303,8 +303,8 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
THEN:
- Forbidden response is returned
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user1 = UserFactory(username="test1")
grant_global(user1, "view_document")
doc_owned = Document.objects.create(
title="owned_doc",
@@ -338,8 +338,8 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
THEN:
- Request succeeds
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user1 = UserFactory(username="test1")
grant_global(user1, "view_document")
self.client.force_authenticate(user1)
@@ -4,7 +4,6 @@ from collections.abc import Callable
from datetime import date
from urllib.parse import quote
from django.contrib.auth.models import User
from rest_framework.test import APITestCase
from documents.models import CustomField
@@ -13,7 +12,8 @@ from documents.models import Document
from documents.models import SavedView
from documents.models import SavedViewFilterRule
from documents.serialisers import DocumentSerializer
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
class DocumentWrapper:
@@ -35,7 +35,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
# Create one custom field per type. The fields are called f"{type}_field".
+114 -36
View File
@@ -2,10 +2,13 @@ import datetime
import json
from unittest import mock
from django.contrib.auth.models import Permission
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.db import connection
from django.test import override_settings
from guardian.shortcuts import assign_perm
from django.test.utils import CaptureQueriesContext
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from rest_framework import status
from rest_framework.test import APITestCase
@@ -16,14 +19,17 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
class TestApiObjects(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=user)
self.tag1 = Tag.objects.create(name="t1", is_inbox_tag=True)
@@ -148,7 +154,7 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
# A newer document owned by another user must not leak through the
# aggregate for a non-superuser who cannot view it
other = User.objects.create_user(username="other")
other = UserFactory(username="other")
Document.objects.create(
mime_type="application/pdf",
correspondent=self.c1,
@@ -157,10 +163,8 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
owner=other,
)
user = User.objects.create_user(username="regular")
user.user_permissions.add(
Permission.objects.get(codename="view_correspondent"),
)
user = UserFactory(username="regular")
grant_global(user, "view_correspondent")
self.client.force_authenticate(user=user)
response = self.client.get("/api/correspondents/?last_correspondence=true")
@@ -195,7 +199,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=user)
self.sp1 = StoragePath.objects.create(name="sp1", path="Something/{checksum}")
@@ -450,8 +454,9 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertEqual(response.data, "folder/Something.pdf")
def test_test_storage_path_requires_document_view_permission(self) -> None:
owner = User.objects.create_user(username="owner")
unprivileged = User.objects.create_user(username="unprivileged")
owner = UserFactory(username="owner")
unprivileged = UserFactory(username="unprivileged")
grant_global(unprivileged, "view_document")
document = Document.objects.create(
mime_type="application/pdf",
owner=owner,
@@ -473,16 +478,31 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertIn("document", response.data)
def test_test_storage_path_allows_shared_document_view_permission(self) -> None:
owner = User.objects.create_user(username="owner")
viewer = User.objects.create_user(username="viewer")
owner = UserFactory(username="owner")
viewer = UserFactory(username="viewer")
document = Document.objects.create(
mime_type="application/pdf",
owner=owner,
title="Shared",
checksum="123",
)
assign_perm("view_document", viewer, document)
grant_object(viewer, document, "view_document")
self.client.force_authenticate(user=viewer)
response = self.client.post(
f"{self.ENDPOINT}test/",
json.dumps(
{
"document": document.id,
"path": "path/{{ title }}",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
grant_global(viewer, "view_document")
viewer = User.objects.get(pk=viewer.pk)
self.client.force_authenticate(user=viewer)
response = self.client.post(
f"{self.ENDPOINT}test/",
@@ -520,11 +540,12 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
def test_test_storage_path_exposes_basic_document_context_but_not_sensitive_owner_data(
self,
) -> None:
owner = User.objects.create_user(
owner = UserFactory(
username="owner",
password="password",
email="owner@example.com",
)
grant_global(owner, "view_document")
document = Document.objects.create(
mime_type="application/pdf",
owner=owner,
@@ -586,8 +607,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
def test_test_storage_path_includes_related_objects_for_visible_document(
self,
) -> None:
owner = User.objects.create_user(username="owner")
viewer = User.objects.create_user(username="viewer")
owner = UserFactory(username="owner")
viewer = UserFactory(username="viewer")
private_correspondent = Correspondent.objects.create(
name="Private Correspondent",
owner=owner,
@@ -599,7 +620,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
title="Document",
checksum="123",
)
assign_perm("view_document", viewer, document)
grant_object(viewer, document, "view_document")
grant_global(viewer, "view_document")
self.client.force_authenticate(user=viewer)
response = self.client.post(
@@ -631,7 +653,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertEqual(response.data, "Private Correspondent.pdf")
def test_test_storage_path_superuser_can_view_private_related_objects(self) -> None:
owner = User.objects.create_user(username="owner")
owner = UserFactory(username="owner")
private_correspondent = Correspondent.objects.create(
name="Private Correspondent",
owner=owner,
@@ -662,8 +684,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
def test_test_storage_path_includes_doc_type_storage_path_and_tags(
self,
) -> None:
owner = User.objects.create_user(username="owner")
viewer = User.objects.create_user(username="viewer")
owner = UserFactory(username="owner")
viewer = UserFactory(username="viewer")
private_document_type = DocumentType.objects.create(
name="Private Type",
owner=owner,
@@ -686,7 +708,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
checksum="123",
)
document.tags.add(private_tag)
assign_perm("view_document", viewer, document)
grant_object(viewer, document, "view_document")
grant_global(viewer, "view_document")
self.client.force_authenticate(user=viewer)
response = self.client.post(
@@ -722,8 +745,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
def test_test_storage_path_includes_custom_fields_for_visible_document(
self,
) -> None:
owner = User.objects.create_user(username="owner")
viewer = User.objects.create_user(username="viewer")
owner = UserFactory(username="owner")
viewer = UserFactory(username="viewer")
document = Document.objects.create(
mime_type="application/pdf",
owner=owner,
@@ -739,7 +762,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
field=custom_field,
value_int=42,
)
assign_perm("view_document", viewer, document)
grant_object(viewer, document, "view_document")
grant_global(viewer, "view_document")
self.client.force_authenticate(user=viewer)
response = self.client.post(
@@ -761,7 +785,7 @@ class TestBulkEditObjects(APITestCase):
def setUp(self) -> None:
super().setUp()
self.temp_admin = User.objects.create_superuser(username="temp_admin")
self.temp_admin = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.temp_admin)
self.t1 = Tag.objects.create(name="t1")
@@ -842,6 +866,66 @@ class TestBulkEditObjects(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(StoragePath.objects.count(), 0)
def test_bulk_objects_set_permissions_batched_across_object_count(
self,
) -> None:
"""
GIVEN:
- Many tags are being bulk-edited to set permissions at once
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation over a small batch vs. a much larger one
THEN:
- Permissions are applied correctly at both scales
- Query count does not grow with the number of tags, i.e. each
user/group is applied across all tags with one batched call
rather than one call per (tag, identity) pair
"""
group1 = Group.objects.create(name="perm-group")
permissions = {
"view": {"users": [self.user1.id, self.user2.id], "groups": [group1.id]},
"change": {"users": [self.user1.id], "groups": [group1.id]},
}
def run_with_n_tags(n: int) -> int:
tags = [Tag.objects.create(name=f"perm-tag-{n}-{i}") for i in range(n)]
with CaptureQueriesContext(connection) as ctx:
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [t.id for t in tags],
"object_type": "tags",
"operation": "set_permissions",
"permissions": permissions,
"merge": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
for tag in tags:
self.assertEqual(get_users_with_perms(tag).count(), 2)
self.assertEqual(get_groups_with_perms(tag).count(), 1)
return len(ctx.captured_queries)
small_batch_queries = run_with_n_tags(5)
large_batch_queries = run_with_n_tags(50)
# A tolerance rather than equality, matching the N+1 check in
# test_views.py: bulk_create's batch_size caps rows per INSERT, so a
# large enough selection does legitimately add statements, and the
# per-process ContentType cache makes the first run carry an extra
# query. Neither can hide a regression to per-object assignment,
# which would be ~10x the small-batch count here.
self.assertLessEqual(
large_batch_queries,
small_batch_queries + 5,
"Permission assignment appears to scale with object count: "
f"{small_batch_queries} queries for 5 tags vs. "
f"{large_batch_queries} for 50",
)
def test_bulk_objects_delete_all_filtered(self) -> None:
"""
GIVEN:
@@ -933,9 +1017,7 @@ class TestBulkEditObjects(APITestCase):
THEN:
- User is able to delete objects
"""
self.user1.user_permissions.add(
*Permission.objects.filter(codename="delete_tag"),
)
grant_global(self.user1, "delete_tag")
self.user1.save()
self.client.force_authenticate(user=self.user1)
@@ -965,9 +1047,7 @@ class TestBulkEditObjects(APITestCase):
self.t2.owner = User.objects.get(username="temp_admin")
self.t2.save()
self.user1.user_permissions.add(
*Permission.objects.filter(codename="delete_tag"),
)
grant_global(self.user1, "delete_tag")
self.user1.save()
self.client.force_authenticate(user=self.user1)
@@ -1000,9 +1080,7 @@ class TestBulkEditObjects(APITestCase):
self.t2.owner = User.objects.get(username="temp_admin")
self.t2.save()
self.user1.user_permissions.add(
*Permission.objects.filter(codename="delete_tag"),
)
grant_global(self.user1, "delete_tag")
self.user1.save()
self.client.force_authenticate(user=self.user1)
@@ -10,7 +10,7 @@ from documents.models import CustomField
from documents.models import DocumentType
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
+165 -75
View File
@@ -5,9 +5,7 @@ from unittest import mock
from allauth.mfa.models import Authenticator
from allauth.mfa.totp.internal import auth as totp_auth
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from guardian.shortcuts import assign_perm
from guardian.shortcuts import get_perms
from guardian.shortcuts import get_users_with_perms
from rest_framework import status
@@ -19,7 +17,11 @@ from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
class TestApiAuth(DirectoriesMixin, APITestCase):
@@ -93,14 +95,14 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
self.assertNotIn("X-Version", response)
def test_api_version_with_auth(self) -> None:
user = User.objects.create_superuser(username="test")
user = UserFactory(username="test", superuser=True)
self.client.force_authenticate(user)
response = self.client.get("/api/documents/")
self.assertIn("X-Api-Version", response)
self.assertIn("X-Version", response)
def test_api_insufficient_permissions(self) -> None:
user = User.objects.create_user(username="test")
user = UserFactory(username="test")
self.client.force_authenticate(user)
Document.objects.create(title="Test")
@@ -137,8 +139,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
)
def test_api_sufficient_permissions(self) -> None:
user = User.objects.create_user(username="test")
user.user_permissions.add(*Permission.objects.all())
user = UserFactory(username="test")
grant_all_global(user)
user.is_staff = True
self.client.force_authenticate(user)
@@ -166,9 +168,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
)
def test_api_get_object_permissions(self) -> None:
user1 = User.objects.create_user(username="test1")
user2 = User.objects.create_user(username="test2")
user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user1 = UserFactory(username="test1")
user2 = UserFactory(username="test2")
grant_global(user1, "view_document")
self.client.force_authenticate(user1)
self.assertEqual(
@@ -205,7 +207,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
THEN:
- Object created with current user as owner
"""
user1 = User.objects.create_superuser(username="user1")
user1 = UserFactory(username="user1", superuser=True)
self.client.force_authenticate(user1)
@@ -234,7 +236,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
THEN:
- Object created with no owner
"""
user1 = User.objects.create_superuser(username="user1")
user1 = UserFactory(username="user1", superuser=True)
self.client.force_authenticate(user1)
@@ -265,7 +267,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
THEN:
- Object permissions are set appropriately
"""
user1 = User.objects.create_superuser(username="user1")
user1 = UserFactory(username="user1", superuser=True)
user2 = User.objects.create(username="user2")
group1 = Group.objects.create(name="group1")
@@ -313,7 +315,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
THEN:
- Object permissions are set appropriately
"""
user1 = User.objects.create_superuser(username="user1")
user1 = UserFactory(username="user1", superuser=True)
user2 = User.objects.create(username="user2")
group1 = Group.objects.create(name="group1")
@@ -363,7 +365,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
mime_type="application/pdf",
content="this is a document",
)
user1 = User.objects.create_superuser(username="user1")
user1 = UserFactory(username="user1", superuser=True)
user2 = User.objects.create(username="user2")
group1 = Group.objects.create(name="group1")
@@ -413,16 +415,16 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
mime_type="application/pdf",
content="this is a document",
)
user1 = User.objects.create_superuser(username="user1")
user1 = UserFactory(username="user1", superuser=True)
user2 = User.objects.create(username="user2")
group1 = Group.objects.create(name="group1")
doc.owner = user1
doc.save()
assign_perm("view_document", user2, doc)
assign_perm("change_document", user2, doc)
assign_perm("view_document", group1, doc)
assign_perm("change_document", group1, doc)
grant_object(user2, doc, "view_document")
grant_object(user2, doc, "change_document")
grant_object(group1, doc, "view_document")
grant_object(group1, doc, "change_document")
self.client.force_authenticate(user1)
@@ -446,11 +448,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
self.assertIn("change_document", get_perms(group1, doc))
def test_document_permissions_change_requires_owner(self) -> None:
owner = User.objects.create_user(username="owner")
editor = User.objects.create_user(username="editor")
editor.user_permissions.add(
*Permission.objects.all(),
)
owner = UserFactory(username="owner")
editor = UserFactory(username="editor")
grant_all_global(editor)
doc = Document.objects.create(
title="Ownered doc",
@@ -460,8 +460,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
owner=owner,
)
assign_perm("view_document", editor, doc)
assign_perm("change_document", editor, doc)
grant_object(editor, doc, "view_document")
grant_object(editor, doc, "change_document")
self.client.force_authenticate(editor)
response = self.client.patch(
@@ -499,9 +499,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_dynamic_permissions_fields(self) -> None:
user1 = User.objects.create_user(username="user1")
user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
user2 = User.objects.create_user(username="user2")
user1 = UserFactory(username="user1")
grant_global(user1, "view_document")
user2 = UserFactory(username="user2")
Document.objects.create(title="Test", content="content 1", checksum="1")
doc2 = Document.objects.create(
@@ -523,10 +523,10 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
owner=user1,
)
assign_perm("view_document", user1, doc2)
assign_perm("view_document", user1, doc3)
assign_perm("change_document", user1, doc3)
assign_perm("view_document", user2, doc4)
grant_object(user1, doc2, "view_document")
grant_object(user1, doc3, "view_document")
grant_object(user1, doc3, "change_document")
grant_object(user2, doc4, "view_document")
self.client.force_authenticate(user1)
@@ -574,8 +574,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
owned by someone else with no explicit guardian grant -- mirrors
guardian's own ObjectPermissionChecker.has_perm() superuser shortcut.
"""
superuser = User.objects.create_superuser(username="admin")
other_user = User.objects.create_user(username="user2")
superuser = UserFactory(username="admin", superuser=True)
other_user = UserFactory(username="user2")
Document.objects.create(
title="Test",
content="content",
@@ -602,7 +602,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
THEN:
- MFA required error is returned
"""
user1 = User.objects.create_user(username="user1")
user1 = UserFactory(username="user1")
user1.set_password("password")
user1.save()
@@ -626,7 +626,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
THEN:
- MFA code is required
"""
user1 = User.objects.create_user(username="user1")
user1 = UserFactory(username="user1")
user1.set_password("password")
user1.save()
@@ -688,7 +688,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
def test_get_users(self) -> None:
@@ -858,10 +858,8 @@ class TestApiUser(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
regular_user = User.objects.create_user(username="regular_user")
regular_user.user_permissions.add(
*Permission.objects.all(),
)
regular_user = UserFactory(username="regular_user")
grant_all_global(regular_user)
self.client.force_authenticate(regular_user)
Authenticator.objects.create(
user=user1,
@@ -885,9 +883,9 @@ class TestApiUser(DirectoriesMixin, APITestCase):
- Only superusers can change superuser status
"""
user1 = User.objects.create_user(username="user1")
user1.user_permissions.add(*Permission.objects.all())
user2 = User.objects.create_superuser(username="user2")
user1 = UserFactory(username="user1")
grant_all_global(user1)
user2 = UserFactory(username="user2", superuser=True)
self.client.force_authenticate(user1)
@@ -972,9 +970,9 @@ class TestApiUser(DirectoriesMixin, APITestCase):
- Only superusers can change staff status
"""
user1 = User.objects.create_user(username="user1")
user1.user_permissions.add(*Permission.objects.all())
user2 = User.objects.create_superuser(username="user2")
user1 = UserFactory(username="user1")
grant_all_global(user1)
user2 = UserFactory(username="user2", superuser=True)
self.client.force_authenticate(user1)
@@ -1027,7 +1025,7 @@ class TestApiGroup(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
def test_get_groups(self) -> None:
@@ -1128,7 +1126,7 @@ class TestBulkEditObjectPermissions(APITestCase):
def setUp(self) -> None:
super().setUp()
self.temp_admin = User.objects.create_superuser(username="temp_admin")
self.temp_admin = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.temp_admin)
self.t1 = Tag.objects.create(name="t1")
@@ -1276,7 +1274,7 @@ class TestBulkEditObjectPermissions(APITestCase):
},
}
assign_perm("view_tag", self.user3, self.t1)
grant_object(self.user3, self.t1, "view_tag")
self.t1.owner = self.user3
self.t1.save()
@@ -1373,13 +1371,9 @@ class TestBulkEditObjectPermissions(APITestCase):
"""
self.t1.owner = self.user2
self.t1.save()
assign_perm("view_tag", self.user1, self.t1)
assign_perm("change_tag", self.user1, self.t1)
self.user1.user_permissions.add(
*Permission.objects.filter(
codename__in=["view_tag", "change_tag"],
),
)
grant_object(self.user1, self.t1, "view_tag")
grant_object(self.user1, self.t1, "change_tag")
grant_global(self.user1, "view_tag", "change_tag")
user1 = User.objects.get(pk=self.user1.pk)
self.client.force_authenticate(user=user1)
@@ -1426,13 +1420,9 @@ class TestBulkEditObjectPermissions(APITestCase):
"""
owned = Tag.objects.create(name="owned", owner=self.user1)
shared = Tag.objects.create(name="shared", owner=self.user2)
assign_perm("view_tag", self.user1, shared)
assign_perm("change_tag", self.user1, shared)
self.user1.user_permissions.add(
*Permission.objects.filter(
codename__in=["view_tag", "change_tag"],
),
)
grant_object(self.user1, shared, "view_tag")
grant_object(self.user1, shared, "change_tag")
grant_global(self.user1, "view_tag", "change_tag")
user1 = User.objects.get(pk=self.user1.pk)
self.client.force_authenticate(user=user1)
@@ -1473,14 +1463,10 @@ class TestBulkEditObjectPermissions(APITestCase):
"""
self.t1.owner = self.user2
self.t1.save()
assign_perm("view_tag", self.user1, self.t1)
assign_perm("change_tag", self.user1, self.t1)
assign_perm("delete_tag", self.user1, self.t1)
self.user1.user_permissions.add(
*Permission.objects.filter(
codename__in=["view_tag", "change_tag", "delete_tag"],
),
)
grant_object(self.user1, self.t1, "view_tag")
grant_object(self.user1, self.t1, "change_tag")
grant_object(self.user1, self.t1, "delete_tag")
grant_global(self.user1, "view_tag", "change_tag", "delete_tag")
user1 = User.objects.get(pk=self.user1.pk)
self.client.force_authenticate(user=user1)
@@ -1499,6 +1485,110 @@ class TestBulkEditObjectPermissions(APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(Tag.objects.filter(pk=self.t1.id).exists())
def test_bulk_object_set_permissions_rejects_empty_permissions(self) -> None:
"""
GIVEN:
- Existing objects
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation and an empty permissions dict
THEN:
- Validation fails rather than silently applying a no-op
"""
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": {},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_bulk_object_set_permissions_rejects_non_dict_permissions(self) -> None:
"""
GIVEN:
- Existing objects
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation and a non-dict permissions value
THEN:
- Validation fails rather than crashing
"""
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_bulk_object_set_permissions_rejects_unknown_action(self) -> None:
"""
GIVEN:
- Existing objects
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation and an unrecognized permission action name
THEN:
- Validation fails rather than silently no-oping
"""
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": {"not_a_real_action": {"users": [self.user1.id]}},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_bulk_object_set_permissions_null_users_clears_users(self) -> None:
"""
GIVEN:
- An object a user has view permission on
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation, merge off, and an explicit null for the view users
THEN:
- Request succeeds and null is treated as an empty user list,
so the existing view permission is removed
"""
grant_object(self.user1, self.t1, "view_tag")
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": {"view": {"users": None}},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertNotIn(self.user1, get_users_with_perms(self.t1))
def test_bulk_edit_object_permissions_validation(self) -> None:
"""
GIVEN:
@@ -1576,7 +1666,7 @@ class TestFullPermissionsFlag(APITestCase):
def setUp(self) -> None:
super().setUp()
self.admin = User.objects.create_superuser(username="admin")
self.admin = UserFactory(username="admin", superuser=True)
def test_full_perms_flag(self) -> None:
"""
@@ -1,3 +1,5 @@
from __future__ import annotations
import unicodedata
from typing import TYPE_CHECKING
from unittest import mock
@@ -7,8 +9,11 @@ import pytest
from django.core.files.uploadedfile import SimpleUploadedFile
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from paperless_testing.dirs import PaperlessDirs
@pytest.fixture()
@@ -18,22 +23,14 @@ def consume_file_mock():
yield m
@pytest.fixture()
def directories(tmp_path, settings, _media_settings):
scratch = tmp_path / "scratch"
scratch.mkdir()
settings.SCRATCH_DIR = scratch
return scratch
@pytest.mark.django_db
class TestPostDocumentNFCNormalization:
def test_nfd_filename_normalized_to_nfc(
self,
admin_client,
admin_client: APIClient,
consume_file_mock: mock.MagicMock,
directories,
):
paperless_dirs: PaperlessDirs,
) -> None:
"""Uploaded file with NFD filename must have its name stored as NFC."""
nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
@@ -69,10 +66,10 @@ class TestPostDocumentNFCNormalization:
def test_already_nfc_filename_unchanged(
self,
admin_client,
admin_client: APIClient,
consume_file_mock: mock.MagicMock,
directories,
):
paperless_dirs: PaperlessDirs,
) -> None:
"""Uploaded file with already-NFC filename must pass through unchanged."""
nfc = unicodedata.normalize("NFC", "Invoice_2024.pdf")
+16 -17
View File
@@ -8,7 +8,8 @@ from rest_framework import status
from rest_framework.authtoken.models import Token
from rest_framework.test import APITestCase
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
# see allauth.socialaccount.providers.openid.provider.OpenIDProvider
@@ -55,23 +56,23 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(
self.user = UserFactory(
username="temp_admin",
first_name="firstname",
last_name="surname",
superuser=True,
)
self.client.force_authenticate(user=self.user)
def setupSocialAccount(self) -> None:
def setupSocialAccount(self) -> SocialAccount:
SocialApp.objects.create(
name="Keycloak",
provider="openid_connect",
provider_id="keycloak-test",
)
self.user.socialaccount_set.add(
SocialAccount(uid="123456789", provider="keycloak-test"),
bulk=False,
)
social_account = SocialAccount(uid="123456789", provider="keycloak-test")
self.user.socialaccount_set.add(social_account, bulk=False)
return social_account
def test_get_profile(self) -> None:
"""
@@ -109,19 +110,17 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
THEN:
- Profile is returned with social accounts
"""
self.setupSocialAccount()
social_account = self.setupSocialAccount()
openid_provider = (
MockOpenIDConnectProvider(
app=SocialApp.objects.get(provider_id="keycloak-test"),
),
openid_provider = MockOpenIDConnectProvider(
app=SocialApp.objects.get(provider_id="keycloak-test"),
)
mock_list_providers.return_value = [
openid_provider,
]
mock_get_provider_account.return_value = MockOpenIDConnectProviderAccount(
mock_social_account_dict={
"name": openid_provider[0].name,
"name": openid_provider.name,
},
)
@@ -133,7 +132,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
response.data["social_accounts"],
[
{
"id": 1,
"id": social_account.pk,
"provider": "keycloak-test",
"name": "Keycloak",
},
@@ -150,7 +149,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
THEN:
- Profile is returned with "Unknown App" as name
"""
self.setupSocialAccount()
social_account = self.setupSocialAccount()
# Remove the social app
SocialApp.objects.get(provider_id="keycloak-test").delete()
@@ -163,7 +162,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
response.data["social_accounts"],
[
{
"id": 1,
"id": social_account.pk,
"provider": "keycloak-test",
"name": "Unknown App",
},
@@ -401,7 +400,7 @@ class TestApiTOTPViews(APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
def test_get_totp(self) -> None:
+308 -70
View File
@@ -7,10 +7,8 @@ import time_machine
from dateutil.relativedelta import relativedelta
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from django.utils import timezone
from guardian.shortcuts import assign_perm
from rest_framework import status
from rest_framework.test import APITestCase
@@ -27,10 +25,13 @@ from documents.models import Tag
from documents.models import Workflow
from documents.search import get_backend
from documents.search import reset_backend
from documents.tests.factories import DocumentFactory
from documents.tests.utils import DirectoriesMixin
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
pytestmark = pytest.mark.search
@@ -40,7 +41,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
super().setUp()
reset_backend()
self.user = User.objects.create_superuser(username="temp_admin")
self.user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=self.user)
def tearDown(self) -> None:
@@ -244,8 +245,68 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.data["count"], 1)
self.assertEqual(response.data["results"][0]["id"], title_match.id)
def test_simple_search_combined_with_query_filter(self) -> None:
"""
GIVEN:
- A document with the term in its title, created this year
- A document with the term only in its content, created this year
- A document with the term in its title, created years ago
WHEN:
- A title or text search is combined with a relative created
date query, as the filter editor sends it
THEN:
- The simple search keeps its own field scope and the query only
narrows the results (issue #14436)
"""
this_year = timezone.localdate().replace(month=1, day=1)
title_match = Document.objects.create(
title="foo report",
content="nothing relevant",
checksum="F1",
created=this_year,
pk=21,
)
content_match = Document.objects.create(
title="something else",
content="foo appears here",
checksum="F2",
created=this_year,
pk=22,
)
old_title_match = Document.objects.create(
title="foo archive",
content="nothing relevant",
checksum="F3",
created=datetime.date(2001, 1, 1),
pk=23,
)
backend = get_backend()
for doc in (title_match, content_match, old_title_match):
backend.add_or_update(doc)
query = "created:[-1 year to now]"
response = self.client.get(
"/api/documents/",
{"title_search": "foo", "query": query},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
[r["id"] for r in response.data["results"]],
[title_match.id],
)
response = self.client.get(
"/api/documents/",
{"text": "foo", "query": query},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertCountEqual(
[r["id"] for r in response.data["results"]],
[title_match.id, content_match.id],
)
def test_search_rejects_multiple_search_modes(self) -> None:
response = self.client.get("/api/documents/?text=bank&query=bank")
response = self.client.get("/api/documents/?text=bank&title_search=bank")
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(
response.data["detail"],
@@ -786,6 +847,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
tick=False,
):
response = self.client.get("/api/documents/?query=added:previous month")
assert response.status_code == 200, (
f"expected a successful search response, got {response.status_code}: "
f"{response.data!r}"
)
results = response.data["results"]
self.assertEqual(len(results), 1)
@@ -818,6 +883,26 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("invalid-date", str(response.data["query"]))
def test_search_multiple_bad_fields_returns_all_messages(self) -> None:
"""
GIVEN:
- One document added
WHEN:
- Query with multiple bad fields (e.g. invalid date and invalid number)
THEN:
- 400 Bad Request with error messages for every bad field,
so the user can fix them all in one round-trip
"""
response = self.client.get(
"/api/documents/",
{"query": "created:notadate AND asn:notanumber"},
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
messages = response.data["query"]
self.assertEqual(len(messages), 2)
self.assertTrue(any("created" in m for m in messages))
self.assertTrue(any("asn" in m for m in messages))
@override_settings(
TIME_ZONE="UTC",
)
@@ -861,6 +946,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
results = response.data["results"]
self.assertEqual({r["id"] for r in results}, {1, 2})
@mock.patch("documents.search._backend.parse_user_query")
def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None:
"""
GIVEN:
- The query parser itself fails (a whoosh-compat bug, per
QueryParserError's own contract: not user-fixable input)
WHEN:
- Any search request runs
THEN:
- The error surfaces as a 500 monitoring can see, never a 400
blaming the user for a library defect
"""
from whoosh_compat.errors import QueryParserError
m.side_effect = QueryParserError("synthetic parser bug")
self.client.raise_request_exception = False
response = self.client.get("/api/documents/?query=anything")
self.assertEqual(
response.status_code,
status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@mock.patch("documents.search._backend.TantivyBackend.autocomplete")
def test_search_autocomplete_limits(self, m) -> None:
"""
@@ -902,9 +1010,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
THEN:
- Terms only within docs user has access to are returned
"""
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1.user_permissions.add(Permission.objects.get(codename="view_document"))
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
grant_global(u1, "view_document")
self.client.force_authenticate(user=u1)
@@ -944,7 +1052,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, ["applebaum", "apples"])
assign_perm("view_document", u1, d3)
grant_object(u1, d3, "view_document")
backend.add_or_update(d3)
response = self.client.get("/api/search/autocomplete/?term=app")
@@ -952,10 +1060,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.data, ["applebaum", "apples", "appletini"])
def test_search_autocomplete_group_revocation_is_immediate(self) -> None:
user = User.objects.create_user("group-user")
owner = User.objects.create_user("document-owner")
user = UserFactory(username="group-user")
owner = UserFactory(username="document-owner")
group = Group.objects.create(name="temporary-viewers")
user.user_permissions.add(Permission.objects.get(codename="view_document"))
grant_global(user, "view_document")
user.groups.add(group)
document = Document.objects.create(
@@ -964,7 +1072,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
checksum="group-revocation",
owner=owner,
)
assign_perm("view_document", group, document)
grant_object(group, document, "view_document")
get_backend().add_or_update(document)
self.client.force_authenticate(user=user)
@@ -1044,11 +1152,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertIsNone(response.data["corrected_query"])
def test_search_spelling_suggestion_suppressed_for_private_terms(self) -> None:
owner = User.objects.create_user("owner")
attacker = User.objects.create_user("attacker")
attacker.user_permissions.add(
Permission.objects.get(codename="view_document"),
)
owner = UserFactory(username="owner")
attacker = UserFactory(username="attacker")
grant_global(attacker, "view_document")
backend = get_backend()
for i in range(5):
@@ -1150,6 +1256,59 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertIn(d3.id, result_ids)
self.assertNotIn(d4.id, result_ids)
def test_search_more_like_version_uses_its_root(self) -> None:
"""
GIVEN:
- A document similar in content to a root document, and one that is not
- A version of the root document, which is never indexed
WHEN:
- API request for more like the version
THEN:
- The documents similar to the root are returned, not the version
"""
indexed = {}
for name, title, content, day in (
("root", "bank statement 1", "things i paid for in august", (2019, 3, 4)),
(
"similar",
"bank statement 3",
"things i paid for in september",
(2020, 7, 9),
),
(
"other",
"Quarterly Report",
"quarterly revenue profit margin",
(2021, 11, 30),
),
):
with time_machine.travel(
timezone.make_aware(datetime.datetime(*day)),
tick=False,
):
indexed[name] = DocumentFactory(
title=title,
content=content,
created=datetime.date(*day),
added=timezone.make_aware(datetime.datetime(*day)),
)
version = DocumentFactory(
root_document=indexed["root"],
version_index=1,
content="things i paid for in august",
)
backend = get_backend()
for document in indexed.values():
backend.add_or_update(document)
response = self.client.get(f"/api/documents/?more_like_id={version.id}")
self.assertEqual(response.status_code, status.HTTP_200_OK)
result_ids = [r["id"] for r in response.data["results"]]
self.assertIn(indexed["similar"].id, result_ids)
self.assertNotIn(indexed["other"].id, result_ids)
self.assertNotIn(version.id, result_ids)
def test_more_like_requires_id_of_existing_document(self) -> None:
"""
GIVEN:
@@ -1175,11 +1334,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
THEN:
- The request is rejected
"""
owner = User.objects.create_user("owner")
attacker = User.objects.create_user("attacker")
attacker.user_permissions.add(
Permission.objects.get(codename="view_document"),
)
owner = UserFactory(username="owner")
attacker = UserFactory(username="attacker")
grant_global(attacker, "view_document")
private_seed = Document.objects.create(
title="private bank statement",
@@ -1217,6 +1374,39 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.content, b"Insufficient permissions.")
def test_search_more_like_version_requires_view_permission_on_its_root(
self,
) -> None:
"""
GIVEN:
- Another user's private root document with an unowned version
WHEN:
- A user without access requests more-like-this for the version
THEN:
- The request is rejected, the version's missing owner grants nothing
"""
owner = UserFactory(username="owner")
attacker = UserFactory(username="attacker")
grant_global(attacker, "view_document")
private_root = DocumentFactory(
title="private bank statement",
content="quarterly treasury bank statement wire transfer",
owner=owner,
)
version = DocumentFactory(
root_document=private_root,
version_index=1,
owner=None,
)
get_backend().add_or_update(private_root)
self.client.force_authenticate(user=attacker)
response = self.client.get(f"/api/documents/?more_like_id={version.id}")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.content, b"Insufficient permissions.")
def test_search_filtering(self) -> None:
t = Tag.objects.create(name="tag")
t2 = Tag.objects.create(name="tag2")
@@ -1487,11 +1677,11 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
- Only owned docs are returned for regular users
- All docs are returned for superuser
"""
superuser = User.objects.create_superuser("superuser")
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
superuser = UserFactory(username="superuser", superuser=True)
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
grant_global(u1, "view_document")
grant_global(u2, "view_document")
Document.objects.create(checksum="1", content="test 1", owner=u1)
Document.objects.create(checksum="2", content="test 2", owner=u2)
@@ -1541,10 +1731,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
THEN:
- Only docs with granted view permissions are returned
"""
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
grant_global(u1, "view_document")
grant_global(u2, "view_document")
d1 = Document.objects.create(checksum="1", content="test 1", owner=u1)
d2 = Document.objects.create(checksum="2", content="test 2", owner=u2)
@@ -1569,9 +1759,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
r = self.client.get("/api/documents/?query=test&owner__isnull=true")
self.assertEqual(r.data["count"], 1)
assign_perm("view_document", u1, d2)
assign_perm("view_document", u1, d3)
assign_perm("view_document", u2, d1)
grant_object(u1, d2, "view_document")
grant_object(u1, d3, "view_document")
grant_object(u2, d1, "view_document")
backend.add_or_update(d1)
backend.add_or_update(d2)
@@ -1594,8 +1784,8 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(r.data["count"], 1)
def test_search_sorting(self) -> None:
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1 = UserFactory(username="user1")
u2 = UserFactory(username="user2")
c1 = Correspondent.objects.create(name="corres Ax")
c2 = Correspondent.objects.create(name="corres Cx")
c3 = Correspondent.objects.create(name="corres Bx")
@@ -1845,8 +2035,8 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
pk=5,
)
user1 = User.objects.create_user("bank user1")
user2 = User.objects.create_superuser("user2")
user1 = UserFactory(username="bank user1")
user2 = UserFactory(username="user2", superuser=True)
group1 = Group.objects.create(name="bank group1")
Group.objects.create(name="group2")
@@ -1878,7 +2068,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
sort_field="",
owner=user2,
)
assign_perm("view_savedview", user1, shared_view)
grant_object(user1, shared_view, "view_savedview")
mail_account1 = MailAccount.objects.create(name="bank mail account 1")
mail_account2 = MailAccount.objects.create(name="mail account 2")
mail_rule1 = MailRule.objects.create(
@@ -1947,36 +2137,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(len(response.data["documents"]), 1)
self.assertEqual(response.data["documents"][0]["id"], title_match.id)
def test_global_search_returns_latest_version_content(self) -> None:
root = Document.objects.create(
title="bank statement",
content="superseded content",
checksum="GSV1",
pk=23,
)
Document.objects.create(
title="bank statement v2",
content="latest content",
checksum="GSV2",
pk=24,
root_document=root,
version_index=1,
)
self.client.force_authenticate(self.user)
response = self.client.get("/api/search/?query=bank&db_only=true")
self.assertEqual(response.status_code, status.HTTP_200_OK)
returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
self.assertEqual(returned.get(root.id), "latest content")
def test_global_search_filters_owned_mail_objects(self) -> None:
user1 = User.objects.create_user("mail-search-user")
user2 = User.objects.create_user("other-mail-search-user")
user1.user_permissions.add(
Permission.objects.get(codename="view_mailaccount"),
Permission.objects.get(codename="view_mailrule"),
)
user1 = UserFactory(username="mail-search-user")
user2 = UserFactory(username="other-mail-search-user")
grant_global(user1, "view_mailaccount", "view_mailrule")
own_account = MailAccount.objects.create(
name="bank owned account",
@@ -2058,3 +2222,77 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
response = self.client.get("/api/search/?query=no")
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def _assert_query_finds(self, doc: Document, query: str) -> None:
get_backend().add_or_update(doc)
response = self.client.get("/api/documents/", {"query": query})
self.assertEqual(response.status_code, status.HTTP_200_OK)
ids = [r["id"] for r in response.data["results"]]
self.assertIn(doc.id, ids)
def test_search_by_asn(self) -> None:
"""
GIVEN:
- A document with an archive serial number, indexed
WHEN:
- A query filters by "asn:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Has ASN",
content="content",
checksum="asn-checksum",
archive_serial_number=555,
)
self._assert_query_finds(doc, "asn:555")
def test_search_by_page_count(self) -> None:
"""
GIVEN:
- A document with a page count, indexed
WHEN:
- A query filters by "page_count:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Multi-page",
content="content",
checksum="page-count-checksum",
page_count=42,
)
self._assert_query_finds(doc, "page_count:42")
def test_search_by_original_filename(self) -> None:
"""
GIVEN:
- A document with an original filename, indexed
WHEN:
- A query filters by "original_filename:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Named file",
content="content",
checksum="filename-checksum",
original_filename="quarterly-report.pdf",
)
self._assert_query_finds(doc, "original_filename:quarterly-report.pdf")
def test_search_by_checksum(self) -> None:
"""
GIVEN:
- A document with a checksum, indexed
WHEN:
- A query filters by "checksum:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Checksum doc",
content="content",
checksum="deadbeef1234",
)
self._assert_query_finds(doc, "checksum:deadbeef1234")
@@ -0,0 +1,334 @@
"""The search list endpoint's exception handling: what becomes a 400 and
what a library defect surfaces as instead.
Companion to documents/tests/search/test_error_routing.py, which pins the
Cause -> SearchQueryError/QueryError routing inside documents/search/_query.py.
These tests pin the layer above it: DocumentViewSet.list's own except clauses,
which decide what an already-routed error becomes on the wire.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from rest_framework import status
from whoosh_compat.errors import Cause
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from documents.search import SearchQueryError
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.models import Document
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
class TestSearchQueryErrorStillBecomesA400:
def test_search_query_error_becomes_a_400_naming_the_field(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising a SearchQueryError naming a field
WHEN:
- The document list endpoint is queried
THEN:
- The response is a 400 whose body names the field
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("bad value for field 'added'")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_search_query_error,
)
response = admin_client.get("/api/documents/?query=anything")
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "added" in str(response.data["query"])
class TestLibraryDefectsPropagate:
"""The exact regression this task exists to fix: an unexpected or
INTERNAL-cause library error must not be relabeled a 400."""
def test_unexpected_exception_is_not_converted_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising an unrelated exception
(ZeroDivisionError), not a SearchQueryError
WHEN:
- The document list endpoint is queried
THEN:
- The exception propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._backend as backend_mod
def raise_zero_division(*args: object, **kwargs: object) -> object:
raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_zero_division,
)
with pytest.raises(ZeroDivisionError):
admin_client.get("/api/documents/?query=anything")
def test_internal_cause_query_error_is_not_converted_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- A real query string running through the real parse and
routing pipeline (pre-parse rewrites, wc.parse(), and
_map_emit_error's own Cause routing all run for real), except
the final emit call (tantivy_emit) is forced to report a
library-internal defect (Cause.INTERNAL) - the one
library-internal failure mode reachable from a real query
WHEN:
- The document list endpoint is queried
THEN:
- The QueryError propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._query as query_mod
def raise_internal(*args: object, **kwargs: object) -> object:
raise QueryError(
Diagnostic(
kind=DiagnosticKind.BACKEND_REJECTED,
cause=Cause.INTERNAL,
message="synthetic whoosh-compat emitter defect",
),
)
monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
with pytest.raises(QueryError):
admin_client.get("/api/documents/?query=invoice")
class TestSelectionPathsAgreeWithSearch:
"""DocumentSelectionMixin backs bulk edit, bulk download, and a
more_like_id selection filter. It catches only SearchQueryError -- the
same contract the search list endpoint enforces above -- so all three
must map SearchQueryError to a 400 and let anything else surface."""
def test_bulk_edit_maps_search_query_error_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising a SearchQueryError naming a field
WHEN:
- The bulk_edit endpoint is called with a query filter
THEN:
- The response is a 400 whose body names the field
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("bad value for field 'added'")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_search_query_error,
)
response = admin_client.post(
"/api/documents/bulk_edit/",
{
"documents": [],
"all": True,
"filters": {"query": "anything"},
"method": "set_document_type",
"parameters": {"document_type": None},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "added" in str(response.data["query"])
def test_bulk_edit_lets_an_unexpected_exception_surface(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising an unrelated exception
(ZeroDivisionError), not a SearchQueryError
WHEN:
- The bulk_edit endpoint is called with a query filter
THEN:
- The exception propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._backend as backend_mod
def raise_zero_division(*args: object, **kwargs: object) -> object:
raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_zero_division,
)
with pytest.raises(ZeroDivisionError):
admin_client.post(
"/api/documents/bulk_edit/",
{
"documents": [],
"all": True,
"filters": {"query": "anything"},
"method": "set_document_type",
"parameters": {"document_type": None},
},
format="json",
)
def test_bulk_download_maps_search_query_error_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising a SearchQueryError naming a field
WHEN:
- The bulk_download endpoint is called with a query filter
THEN:
- The response is a 400 whose body names the field
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("bad value for field 'added'")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_search_query_error,
)
response = admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"query": "anything"},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "added" in str(response.data["query"])
def test_more_like_id_selection_filter_maps_search_query_error_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- TantivyBackend.more_like_this_ids() raising a
SearchQueryError
WHEN:
- The bulk_download endpoint is called with a more_like_id
filter
THEN:
- The response is a 400
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("similar-document lookup is unavailable")
monkeypatch.setattr(
backend_mod.TantivyBackend,
"more_like_this_ids",
raise_search_query_error,
)
response = admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"more_like_id": searchable_document.pk},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
def test_more_like_id_selection_filter_lets_an_unexpected_exception_surface(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
searchable_document: Document,
) -> None:
"""
GIVEN:
- TantivyBackend.more_like_this_ids() raising an unrelated
exception (ZeroDivisionError), not a SearchQueryError
WHEN:
- The bulk_download endpoint is called with a more_like_id
filter
THEN:
- The exception propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._backend as backend_mod
def raise_zero_division(*args: object, **kwargs: object) -> object:
raise ZeroDivisionError("synthetic bug, unrelated to similarity lookup")
monkeypatch.setattr(
backend_mod.TantivyBackend,
"more_like_this_ids",
raise_zero_division,
)
with pytest.raises(ZeroDivisionError):
admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"more_like_id": searchable_document.pk},
},
format="json",
)
@@ -0,0 +1,277 @@
"""The query-length cap in ``_get_tantivy_query_and_mode``.
whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an
unbounded ``query`` (SearchMode.QUERY) string is a CPU-exhaustion vector
against a single request handler. The GET search endpoint is incidentally
bounded by the web server's header limit, but the POST selection-filter
path (bulk edit, bulk download) is not -- that is the real vector, so it
must be pinned here too, not just the GET path.
The cap is enforced once, in the shared helper both entry points call, so
these tests exercise the real endpoints rather than the helper directly:
a construct that looks right in isolation has repeatedly behaved
differently end to end on this branch.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from unittest import mock
import pytest
from rest_framework import status
import documents.search._backend
from documents.views import _MAX_QUERY_LENGTH
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.models import Document
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
class TestGetSearchEndpointEnforcesTheCap:
def test_query_one_over_the_cap_is_a_400(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The GET search endpoint
WHEN:
- A query one character over `_MAX_QUERY_LENGTH` is submitted
THEN:
- The response is a 400 naming both the actual length and the
cap, and the query is rejected before it ever reaches the
parser -- the 400 alone doesn't prove that, since the parser
could run first and the view could discard the result
"""
query = "a" * (_MAX_QUERY_LENGTH + 1)
with mock.patch(
"documents.search._backend.parse_user_query",
wraps=documents.search._backend.parse_user_query,
) as parse_spy:
response = admin_client.get("/api/documents/", {"query": query})
assert response.status_code == status.HTTP_400_BAD_REQUEST
message = str(response.data["query"])
assert str(_MAX_QUERY_LENGTH) in message
assert str(_MAX_QUERY_LENGTH + 1) in message
parse_spy.assert_not_called()
def test_query_at_exactly_the_cap_is_accepted(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The GET search endpoint
WHEN:
- A query exactly `_MAX_QUERY_LENGTH` characters long is
submitted
THEN:
- The response is a 200 (the cap is inclusive, not exclusive)
"""
query = "a" * _MAX_QUERY_LENGTH
response = admin_client.get("/api/documents/", {"query": query})
assert response.status_code == status.HTTP_200_OK
def test_an_ordinary_query_is_unaffected(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The GET search endpoint and an indexed document
WHEN:
- An ordinary, well-under-the-cap query is submitted
THEN:
- The cap has no effect on a normal search: the matching
document is returned
"""
response = admin_client.get("/api/documents/", {"query": "invoice"})
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
class TestPostSelectionPathsEnforceTheCap:
"""The bulk-edit and bulk-download selection filters share the same
helper the GET search path uses. This is the path that actually
matters: it is not bounded by a web server's header-length limit the
way the GET path incidentally is."""
def test_bulk_edit_query_one_over_the_cap_is_a_400(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The bulk-edit selection-filter endpoint
WHEN:
- Its `filters.query` is one character over `_MAX_QUERY_LENGTH`
THEN:
- The response is a 400 naming both the actual length and the
cap, and the query is rejected before it ever reaches the
parser -- this is the path with no web-server header-length
limit to fall back on, so this is the invariant that matters
"""
query = "a" * (_MAX_QUERY_LENGTH + 1)
with mock.patch(
"documents.search._backend.parse_user_query",
wraps=documents.search._backend.parse_user_query,
) as parse_spy:
response = admin_client.post(
"/api/documents/bulk_edit/",
{
"documents": [],
"all": True,
"filters": {"query": query},
"method": "set_document_type",
"parameters": {"document_type": None},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
message = str(response.data["query"])
assert str(_MAX_QUERY_LENGTH) in message
assert str(_MAX_QUERY_LENGTH + 1) in message
parse_spy.assert_not_called()
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
def test_bulk_edit_query_at_exactly_the_cap_is_accepted(
self,
bulk_update_task_mock: mock.MagicMock,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The bulk-edit selection-filter endpoint
WHEN:
- Its `filters.query` is exactly `_MAX_QUERY_LENGTH` characters
long
THEN:
- The cap check accepts it and the request reaches the real
bulk-edit method (its Celery dispatch is mocked out here,
same as every other bulk-edit test, since nothing here is
testing that method itself)
"""
query = "a" * _MAX_QUERY_LENGTH
response = admin_client.post(
"/api/documents/bulk_edit/",
{
"documents": [],
"all": True,
"filters": {"query": query},
"method": "set_document_type",
"parameters": {"document_type": None},
},
format="json",
)
assert response.status_code == status.HTTP_200_OK
def test_bulk_download_query_one_over_the_cap_is_a_400(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The bulk-download selection-filter endpoint
WHEN:
- Its `filters.query` is one character over `_MAX_QUERY_LENGTH`
THEN:
- The response is a 400 naming both the actual length and the
cap, and the query is rejected before it ever reaches the
parser
"""
query = "a" * (_MAX_QUERY_LENGTH + 1)
with mock.patch(
"documents.search._backend.parse_user_query",
wraps=documents.search._backend.parse_user_query,
) as parse_spy:
response = admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"query": query},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
message = str(response.data["query"])
assert str(_MAX_QUERY_LENGTH) in message
assert str(_MAX_QUERY_LENGTH + 1) in message
parse_spy.assert_not_called()
class TestGlobalSearchEnforcesTheCapToo:
"""GlobalSearchView calls the backend directly, not through the shared helper.
It hardcodes SearchMode.TEXT, which is linear rather than quadratic, so it
was never the CPU-exhaustion vector. It is capped anyway so that "every
user query string reaching the backend passes a length check" is an
invariant rather than a claim with an exception: the view already bounds
the query from below, and a later change letting it select a mode would
otherwise reopen the hole silently.
"""
def test_query_one_over_the_cap_is_a_400(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- GlobalSearchView, which calls the backend directly in
SearchMode.TEXT rather than through the shared cap-checking
helper
WHEN:
- Its query is one character over `_MAX_QUERY_LENGTH`
THEN:
- The response is still a 400, keeping "every user query
string reaching the backend passes a length check" an
invariant with no exception, even though TEXT mode is
linear and was never itself the CPU-exhaustion vector
"""
response = admin_client.get(
"/api/search/",
{"query": "a" * (_MAX_QUERY_LENGTH + 1)},
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
def test_query_at_exactly_the_cap_is_accepted(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- GlobalSearchView
WHEN:
- Its query is exactly `_MAX_QUERY_LENGTH` characters long
THEN:
- The response is a 200 (the cap is inclusive, not exclusive)
"""
response = admin_client.get(
"/api/search/",
{"query": "a" * _MAX_QUERY_LENGTH},
)
assert response.status_code == status.HTTP_200_OK
@@ -0,0 +1,77 @@
"""An unterminated ``[`` date range bracket at the API level.
``created:[2020`` (with or without a dangling ``to <value>``) now raises
BAD_DATE and the search endpoint returns HTTP 400, where it used to parse
past the missing ``]`` and silently pass the malformed range through.
A 400 is correct: malformed input should fail loudly rather than silently
matching an unintended query. Pinned at the API level -- the layer a user
or client actually sees -- rather than only against the parser directly.
The properly closed decoy proves the bracket is what matters, not
whoosh-compat's date grammar generally: ``created:[2020 to 2021]`` parses
and searches cleanly.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from rest_framework import status
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.models import Document
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
class TestUnterminatedBracketReturnsA400:
@pytest.mark.parametrize(
"query",
[
pytest.param("created:[2020", id="missing_upper_bound_and_bracket"),
pytest.param("created:[2020 to 2021", id="missing_closing_bracket"),
],
)
def test_unterminated_bracket_is_a_400(
self,
admin_client: APIClient,
searchable_document: Document,
query: str,
) -> None:
"""
GIVEN:
- The search endpoint
WHEN:
- A date-range query with a missing closing `]` (with or
without a dangling upper bound) is submitted
THEN:
- The response is a 400 naming the field, rather than parsing
past the missing bracket and silently passing the malformed
range through
"""
response = admin_client.get(f"/api/documents/?query={query}")
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "created" in str(response.data["query"])
def test_properly_closed_bracket_still_searches_cleanly(
self,
admin_client: APIClient,
searchable_document: Document,
) -> None:
"""
GIVEN:
- The search endpoint
WHEN:
- A properly closed date-range query is submitted
THEN:
- The response is a 200 (the decoy proving the missing
bracket, not whoosh-compat's date grammar generally, is
what the 400 above is about)
"""
response = admin_client.get(
"/api/documents/?query=created:[2020 to 2021]",
)
assert response.status_code == status.HTTP_200_OK
+7 -11
View File
@@ -5,8 +5,6 @@ from datetime import timedelta
from pathlib import Path
from unittest import mock
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from django.utils import timezone
from rest_framework import status
@@ -14,8 +12,10 @@ from rest_framework.test import APITestCase
from documents.models import PaperlessTask
from documents.permissions import has_system_status_permission
from documents.tests.factories import PaperlessTaskFactory
from paperless import version
from paperless_testing.factories import PaperlessTaskFactory
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
class TestSystemStatus(APITestCase):
@@ -23,9 +23,7 @@ class TestSystemStatus(APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_superuser(
username="temp_admin",
)
self.user = UserFactory(username="temp_admin", superuser=True)
self.tmp_dir = Path(tempfile.mkdtemp())
self.override = override_settings(MEDIA_ROOT=self.tmp_dir)
self.override.enable()
@@ -96,7 +94,7 @@ class TestSystemStatus(APITestCase):
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
self.assertEqual(response["WWW-Authenticate"], "Token")
normal_user = User.objects.create_user(username="normal_user")
normal_user = UserFactory(username="normal_user")
self.client.force_login(normal_user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
@@ -107,10 +105,8 @@ class TestSystemStatus(APITestCase):
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
user = User.objects.create_user(username="status_user")
user.user_permissions.add(
Permission.objects.get(codename="view_system_monitoring"),
)
user = UserFactory(username="status_user")
grant_global(user, "view_system_monitoring")
self.client.force_login(user)
response = self.client.get(self.ENDPOINT)
+20 -38
View File
@@ -11,22 +11,21 @@ from datetime import timedelta
from unittest import mock
import pytest
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.utils import timezone
from guardian.shortcuts import assign_perm
from rest_framework import status
from rest_framework.test import APIClient
from documents.filters import PaperlessTaskFilterSet
from documents.models import PaperlessTask
from documents.tests.factories import DocumentFactory
from documents.tests.factories import PaperlessTaskFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import PaperlessTaskFactory
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
pytestmark = pytest.mark.api
ENDPOINT = "/api/tasks/"
ACCEPT_V10 = "application/json; version=10"
ACCEPT_V9 = "application/json; version=9"
@@ -346,21 +345,16 @@ class TestGetTasksV10:
self,
admin_user: User,
regular_user: User,
user_client: APIClient,
) -> None:
"""Regular users see their own tasks and unowned (system) tasks; other users' tasks are hidden."""
regular_user.user_permissions.add(
Permission.objects.get(codename="view_paperlesstask"),
)
client = APIClient()
client.force_authenticate(user=regular_user)
client.credentials(HTTP_ACCEPT=ACCEPT_V10)
grant_global(regular_user, "view_paperlesstask")
PaperlessTaskFactory(owner=admin_user) # other user — not visible
unowned_task = PaperlessTaskFactory() # unowned (system task) — visible
own_task = PaperlessTaskFactory(owner=regular_user)
response = client.get(ENDPOINT)
response = user_client.get(ENDPOINT)
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 2
@@ -590,9 +584,7 @@ class TestGetTasksV9:
regular_user: User,
) -> None:
"""Non-staff users see their own tasks plus unowned tasks via v9 API."""
regular_user.user_permissions.add(
Permission.objects.get(codename="view_paperlesstask"),
)
grant_global(regular_user, "view_paperlesstask")
client = APIClient()
client.force_authenticate(user=regular_user)
@@ -732,19 +724,17 @@ class TestAcknowledge:
assert response.status_code == status.HTTP_403_FORBIDDEN
def test_succeeds_with_change_permission(self, regular_user: User) -> None:
def test_succeeds_with_change_permission(
self,
regular_user: User,
user_client: APIClient,
) -> None:
"""Users granted change_paperlesstask permission can acknowledge tasks."""
regular_user.user_permissions.add(
Permission.objects.get(codename="change_paperlesstask"),
)
grant_global(regular_user, "change_paperlesstask")
regular_user.save()
client = APIClient()
client.force_authenticate(user=regular_user)
client.credentials(HTTP_ACCEPT=ACCEPT_V10)
task = PaperlessTaskFactory()
response = client.post(
response = user_client.post(
ENDPOINT + "acknowledge/",
{"tasks": [task.id]},
format="json",
@@ -807,9 +797,7 @@ class TestSummaryPermissions:
regular_user,
) -> None:
"""A user with view_system_monitoring but no document permissions can access summary/."""
regular_user.user_permissions.add(
Permission.objects.get(codename="view_system_monitoring"),
)
grant_global(regular_user, "view_system_monitoring")
response = user_client.get(ENDPOINT + "summary/")
@@ -822,9 +810,7 @@ class TestSummaryPermissions:
admin_user,
) -> None:
"""Monitoring user sees aggregate data for all tasks, not just unowned ones."""
regular_user.user_permissions.add(
Permission.objects.get(codename="view_system_monitoring"),
)
grant_global(regular_user, "view_system_monitoring")
PaperlessTaskFactory(
owner=admin_user,
task_type=PaperlessTask.TaskType.CONSUME_FILE,
@@ -845,9 +831,7 @@ class TestSummaryPermissions:
) -> None:
"""A regular user with view_paperlesstask but not view_system_monitoring sees only
their own tasks and unowned tasks in the summary, not other users' tasks."""
regular_user.user_permissions.add(
Permission.objects.get(codename="view_paperlesstask"),
)
grant_global(regular_user, "view_paperlesstask")
PaperlessTaskFactory(
owner=regular_user,
@@ -1012,9 +996,7 @@ class TestDuplicateDocumentsPermissions:
@pytest.fixture()
def user_v9_client(self, regular_user: User) -> APIClient:
regular_user.user_permissions.add(
Permission.objects.get(codename="view_paperlesstask"),
)
grant_global(regular_user, "view_paperlesstask")
client = APIClient()
client.force_authenticate(user=regular_user)
client.credentials(HTTP_ACCEPT=ACCEPT_V9)
@@ -1085,7 +1067,7 @@ class TestDuplicateDocumentsPermissions:
) -> None:
"""A user with explicit guardian view_document permission sees the duplicate_of document."""
doc = DocumentFactory(owner=admin_user, title="Granted Doc")
assign_perm("view_document", regular_user, doc)
grant_object(regular_user, doc, "view_document")
PaperlessTaskFactory(
owner=regular_user,
status=PaperlessTask.Status.SUCCESS,
+69 -8
View File
@@ -1,21 +1,22 @@
from datetime import date
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.core.cache import cache
from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_all_global
class TestTrashAPI(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.user = User.objects.create_user(username="temp_admin")
self.user.user_permissions.add(*Permission.objects.all())
self.user = UserFactory(username="temp_admin")
grant_all_global(self.user)
self.client.force_authenticate(user=self.user)
cache.clear()
@@ -69,6 +70,16 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(Document.global_objects.count(), 0)
def test_trash_list_requires_global_document_view_permission(self) -> None:
user = UserFactory(username="trash_owner")
document = Document.objects.create(title="Owned", owner=user)
document.delete()
self.client.force_authenticate(user)
response = self.client.get("/api/trash/")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_trash_api_empty_all(self) -> None:
"""
GIVEN:
@@ -130,7 +141,7 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
created=date(2023, 1, 2),
)
document_not_owned.delete()
user2 = User.objects.create_user(username="user2")
user2 = UserFactory(username="user2")
document_u2 = Document.objects.create(
title="Title3",
content="content3",
@@ -148,7 +159,7 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
self.assertEqual(resp.data["results"][1]["id"], document_u1.pk)
# superuser sees all documents
superuser = User.objects.create_superuser(username="superuser")
superuser = UserFactory(username="superuser", superuser=True)
self.client.force_authenticate(user=superuser)
resp = self.client.get("/api/trash/")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
@@ -164,7 +175,7 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
- 403 Forbidden
"""
user2 = User.objects.create_user(username="user2")
user2 = UserFactory(username="user2")
document = Document.objects.create(
title="Title",
content="content",
@@ -269,3 +280,53 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
Document.objects.filter(root_document=root).values_list("id", flat=True),
[version.pk for version in versions],
)
def test_api_trash_version_follows_root_owner(self) -> None:
"""
GIVEN:
- A deleted version of user2's document, owned by nobody
- A deleted version of the user's document, owned by user2
WHEN:
- The user lists the trash and tries to restore or empty the versions
THEN:
- Only the version of the user's own document is listed
- The other version can't be restored or emptied
- The version of the user's own document can be restored
"""
user2 = UserFactory(username="user2")
other_version = DocumentFactory(
root_document=DocumentFactory(owner=user2),
version_index=1,
)
other_version.delete()
own_version = DocumentFactory(
owner=user2,
root_document=DocumentFactory(owner=self.user),
version_index=1,
)
own_version.delete()
resp = self.client.get("/api/trash/")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(
[doc["id"] for doc in resp.data["results"]],
[own_version.pk],
)
for action in ("restore", "empty"):
with self.subTest(action=action):
resp = self.client.post(
"/api/trash/",
{"action": action, "documents": [other_version.pk]},
)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(
Document.deleted_objects.filter(pk=other_version.pk).exists(),
)
resp = self.client.post(
"/api/trash/",
{"action": "restore", "documents": [own_version.pk]},
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertTrue(Document.objects.filter(pk=own_version.pk).exists())
+5 -5
View File
@@ -1,13 +1,13 @@
import json
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
from documents.tests.utils import DirectoriesMixin
from paperless.version import __full_version_str__
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
class TestApiUiSettings(DirectoriesMixin, APITestCase):
@@ -15,7 +15,7 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
self.test_user = User.objects.create_superuser(username="test")
self.test_user = UserFactory(username="test", superuser=True)
self.test_user.first_name = "Test"
self.test_user.last_name = "User"
self.test_user.save()
@@ -91,7 +91,7 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
)
def test_api_set_ui_settings_insufficient_global_permissions(self) -> None:
not_superuser = User.objects.create_user(username="test_not_superuser")
not_superuser = UserFactory(username="test_not_superuser")
self.client.force_authenticate(user=not_superuser)
settings = {
@@ -111,7 +111,7 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_api_set_ui_settings_sufficient_global_permissions(self) -> None:
not_superuser = User.objects.create_user(username="test_not_superuser")
not_superuser = UserFactory(username="test_not_superuser")
not_superuser.user_permissions.add(
*Permission.objects.filter(codename__contains="uisettings"),
)
+45 -2
View File
@@ -14,7 +14,8 @@ from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
class TestApiWorkflows(DirectoriesMixin, APITestCase):
@@ -25,7 +26,7 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
def setUp(self) -> None:
super().setUp()
user = User.objects.create_superuser(username="temp_admin")
user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=user)
self.user2 = User.objects.create(username="user2")
self.user3 = User.objects.create(username="user3")
@@ -194,6 +195,48 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(Workflow.objects.count(), 2)
def test_api_create_workflow_ignores_nested_action_id(self) -> None:
"""
GIVEN:
- An existing workflow action
WHEN:
- API request to create a workflow includes that action's ID
THEN:
- A new action is created without changing the existing action
"""
original_title = self.action.assign_title
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Workflow 2",
"order": 1,
"triggers": [
{
"sources": [DocumentSource.ApiUpload],
"type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
"filter_filename": "*",
},
],
"actions": [
{
"id": self.action.id,
"assign_title": "New Action Title",
},
],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.action.refresh_from_db()
self.assertEqual(self.action.assign_title, original_title)
new_action = Workflow.objects.get(name="Workflow 2").actions.get()
self.assertNotEqual(new_action.id, self.action.id)
self.assertEqual(new_action.assign_title, "New Action Title")
def test_api_create_workflow_nested(self) -> None:
"""
GIVEN:
+476 -76
View File
@@ -1,28 +1,47 @@
from __future__ import annotations
import shutil
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
from unittest import mock
from typing import TYPE_CHECKING
from unittest.mock import patch
import pytest
import zxingcpp
from django.conf import settings
from django.test import TestCase
from django.test import override_settings
from rest_framework import status
from documents import tasks
from documents.barcodes import Barcode
from documents.barcodes import BarcodePlugin
from documents.barcodes import read_barcode_values
from documents.consumer import ConsumerError
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import Document
from documents.models import DocumentBarcode
from documents.models import Tag
from documents.plugins.base import StopConsumeTaskError
from documents.tests.utils import ConsumeTaskMixin
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless.config import BarcodeConfig
from paperless.models import ApplicationConfiguration
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.fakes.progress import FakeProgressManager
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Generator
from pytest_django.fixtures import Settings
from pytest_mock import MockerFixture
from rest_framework.test import APIClient
from paperless_testing.dirs import PaperlessDirs
class GetReaderPluginMixin:
@@ -31,7 +50,7 @@ class GetReaderPluginMixin:
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
FakeProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
@@ -86,6 +105,7 @@ class TestBarcode(
self.assertDictEqual(separator_page_numbers, {1: False})
@override_settings(CONSUMER_ENABLE_ASN_BARCODE=True)
@pytest.mark.usefixtures("fake_progress_manager")
def test_asn_barcode_duplicate_in_trash_fails(self) -> None:
"""
GIVEN:
@@ -110,15 +130,14 @@ class TestBarcode(
dupe_asn = settings.SCRATCH_DIR / "barcode-39-asn-123-second.pdf"
shutil.copy(test_file, dupe_asn)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dupe_asn,
),
None,
)
with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dupe_asn,
),
None,
)
@override_settings(
CONSUMER_BARCODE_TIFF_SUPPORT=True,
@@ -606,6 +625,7 @@ class TestBarcodeNewConsume(
TestCase,
):
@override_settings(CONSUMER_ENABLE_BARCODES=True)
@pytest.mark.usefixtures("fake_progress_manager")
def test_consume_barcode_file(self) -> None:
"""
GIVEN:
@@ -624,34 +644,33 @@ class TestBarcodeNewConsume(
overrides = DocumentMetadataOverrides(tag_ids=[1, 2, 9])
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
self.assertEqual(
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=temp_copy,
),
overrides,
self.assertEqual(
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=temp_copy,
),
{"reason": "Barcode splitting complete!"},
)
# 2 new document consume tasks created
self.assertEqual(self.consume_file_mock.call_count, 2)
overrides,
),
{"reason": "Barcode splitting complete!"},
)
# 2 new document consume tasks created
self.assertEqual(self.consume_file_mock.call_count, 2)
self.assertIsNotFile(temp_copy)
self.assertIsNotFile(temp_copy)
# Check the split files exist
# Check the original_path is set
# Check the source is unchanged
# Check the overrides are unchanged
for (
new_input_doc,
new_doc_overrides,
) in self.get_all_consume_task_call_args():
self.assertIsFile(new_input_doc.original_file)
self.assertEqual(new_input_doc.original_path, temp_copy)
self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder)
self.assertEqual(overrides, new_doc_overrides)
# Check the split files exist
# Check the original_path is set
# Check the source is unchanged
# Check the overrides are unchanged
for (
new_input_doc,
new_doc_overrides,
) in self.get_all_consume_task_call_args():
self.assertIsFile(new_input_doc.original_file)
self.assertEqual(new_input_doc.original_path, temp_copy)
self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder)
self.assertEqual(overrides, new_doc_overrides)
class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, TestCase):
@@ -660,7 +679,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
FakeProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
@@ -745,6 +764,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
self.assertEqual(asn, None)
@override_settings(CONSUMER_ENABLE_ASN_BARCODE=True)
@pytest.mark.usefixtures("fake_progress_manager")
def test_consume_barcode_file_asn_assignment(self) -> None:
"""
GIVEN:
@@ -762,19 +782,18 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
dst = settings.SCRATCH_DIR / "barcode-39-asn-123.pdf"
shutil.copy(test_file, dst)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
document = Document.objects.first()
assert document is not None
document = Document.objects.first()
assert document is not None
self.assertEqual(document.archive_serial_number, 123)
self.assertEqual(document.archive_serial_number, 123)
def test_scan_file_for_qrcode_without_upscale(self) -> None:
"""
@@ -819,7 +838,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
FakeProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
@@ -827,6 +846,89 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
yield reader
reader.cleanup()
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_ASN_BARCODE=True,
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_SPLIT=True,
)
def test_first_page_tags_without_split(self) -> None:
test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-123.pdf"
with self.get_reader(test_file) as reader:
reader.barcodes = [
Barcode(0, "ASN123,TAG:invoice", reader.settings),
]
existing_tag = Tag.objects.create(name="existing")
reader.metadata.tag_ids = [existing_tag.pk]
reader.run()
self.assertEqual(reader.metadata.asn, 123)
self.assertCountEqual(
reader.metadata.tag_ids,
[existing_tag.pk, Tag.objects.get(name="invoice").pk],
)
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_ASN_BARCODE=True,
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_SPLIT=False,
)
def test_asn_split_preserves_tags_when_tag_split_disabled(self) -> None:
source = self.BARCODE_SAMPLE_DIR / "split-by-tag-basic.pdf"
test_file = self.dirs.scratch_dir / "asn-split.pdf"
shutil.copy(source, test_file)
with self.get_reader(test_file) as reader:
reader.barcodes = [
Barcode(0, "ASN123", reader.settings),
Barcode(0, "TAG:invoice", reader.settings),
Barcode(1, "ASN456", reader.settings),
Barcode(1, "TAG:receipt", reader.settings),
]
existing_tag = Tag.objects.create(name="existing")
reader.metadata.tag_ids = [existing_tag.pk]
with patch("documents.tasks.consume_file.apply_async") as consume:
with self.assertRaises(StopConsumeTaskError):
reader.run()
self.assertEqual(consume.call_count, 2)
for call in consume.call_args_list:
self.assertCountEqual(
call.kwargs["kwargs"]["overrides"].tag_ids,
[
existing_tag.pk,
Tag.objects.get(name="invoice").pk,
Tag.objects.get(name="receipt").pk,
],
)
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_SPLIT=False,
CONSUMER_BARCODE_RETAIN_SPLIT_PAGES=False,
)
def test_discarded_separator_page_preserves_tags(self) -> None:
source = self.BARCODE_SAMPLE_DIR / "split-by-tag-basic.pdf"
test_file = self.dirs.scratch_dir / "separator-split.pdf"
shutil.copy(source, test_file)
with self.get_reader(test_file) as reader:
reader.barcodes = [
Barcode(0, reader.settings.barcode_string, reader.settings),
Barcode(0, "TAG:invoice", reader.settings),
]
with patch("documents.tasks.consume_file.apply_async") as consume:
with self.assertRaises(StopConsumeTaskError):
reader.run()
consume.assert_called_once()
self.assertEqual(
consume.call_args.kwargs["kwargs"]["overrides"].tag_ids,
[Tag.objects.get(name="invoice").pk],
)
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_MAPPING={"TAG:(.*)": "\\g<1>"},
@@ -1024,6 +1126,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
CELERY_TASK_ALWAYS_EAGER=True,
OCR_MODE="auto",
)
@pytest.mark.usefixtures("fake_progress_manager")
def test_consume_barcode_file_tag_split_and_assignment(self) -> None:
"""
GIVEN:
@@ -1042,34 +1145,33 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
dst = settings.SCRATCH_DIR / "split-by-tag-basic.pdf"
shutil.copy(test_file, dst)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
result = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
result = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
self.assertEqual(result, {"reason": "Barcode splitting complete!"})
self.assertEqual(result, {"reason": "Barcode splitting complete!"})
documents = Document.objects.all().order_by("id")
self.assertEqual(documents.count(), 3)
documents = Document.objects.all().order_by("id")
self.assertEqual(documents.count(), 3)
doc1 = documents[0]
self.assertEqual(doc1.tags.count(), 0)
doc1 = documents[0]
self.assertEqual(doc1.tags.count(), 0)
doc2 = documents[1]
self.assertEqual(doc2.tags.count(), 1)
_tag_1 = doc2.tags.first()
assert _tag_1 is not None
self.assertEqual(_tag_1.name, "invoice")
doc2 = documents[1]
self.assertEqual(doc2.tags.count(), 1)
_tag_1 = doc2.tags.first()
assert _tag_1 is not None
self.assertEqual(_tag_1.name, "invoice")
doc3 = documents[2]
self.assertEqual(doc3.tags.count(), 1)
_tag_2 = doc3.tags.first()
assert _tag_2 is not None
self.assertEqual(_tag_2.name, "receipt")
doc3 = documents[2]
self.assertEqual(doc3.tags.count(), 1)
_tag_2 = doc3.tags.first()
assert _tag_2 is not None
self.assertEqual(_tag_2.name, "receipt")
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=True,
@@ -1127,3 +1229,301 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
document_list = reader.separate_pages(separator_pages)
self.assertEqual(len(document_list), 3)
SAMPLE_VALUES = [
{"page": 1, "value": "javascript:alert(1)", "format": "QRCode"},
{"page": 2, "value": "https://example.com/invoice/4711", "format": "QRCode"},
]
@pytest.fixture
def samples_dir() -> Path:
return Path(__file__).parent / "samples"
@pytest.fixture
def barcode_samples_dir(samples_dir: Path) -> Path:
return samples_dir / "barcodes"
@pytest.fixture
def store_barcodes(settings: Settings) -> None:
settings.CONSUMER_STORE_BARCODE_VALUES = True
@pytest.fixture
def barcode_reader(
paperless_dirs: PaperlessDirs,
) -> Generator[Callable[[Path], BarcodePlugin], None, None]:
readers: list[BarcodePlugin] = []
def make(path: Path) -> BarcodePlugin:
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=path),
DocumentMetadataOverrides(),
FakeProgressManager(path.name, None),
paperless_dirs.scratch_dir,
"task-id",
)
reader.setup()
readers.append(reader)
return reader
yield make
for reader in readers:
reader.cleanup()
@pytest.fixture
def consume_sample(
paperless_dirs: PaperlessDirs,
barcode_samples_dir: Path,
fake_progress_manager: type[FakeProgressManager],
settings: Settings,
) -> Callable[..., Document]:
settings.CELERY_TASK_ALWAYS_EAGER = True
settings.OCR_MODE = "auto"
def consume(name: str, *, root_document_id: int | None = None) -> Document:
dst = paperless_dirs.scratch_dir / name
shutil.copy(barcode_samples_dir / name, dst)
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ApiUpload
if root_document_id
else DocumentSource.ConsumeFolder,
original_file=dst,
root_document_id=root_document_id,
),
None,
)
return Document.objects.latest("id")
return consume
def _stored(document: Document) -> list[dict]:
return list(document.barcodes.values("page", "value", "format"))
def test_formats_cover_zxing() -> None:
"""
DocumentBarcode.Format matches the concrete formats of zxing-cpp, so a
zxing-cpp update that adds or removes one fails here
"""
members = zxingcpp.BarcodeFormat.__members__
# skip NONE and the groups like AllLinear, including their aliases
seen = {int(m) for n, m in members.items() if n == "NONE" or n.startswith("All")}
concrete = set()
for name, member in members.items():
# aliases such as DataBarExpanded come after the name zxing reports
if int(member) not in seen:
seen.add(int(member))
concrete.add(name)
assert set(DocumentBarcode.Format.values) == concrete
@pytest.mark.django_db
class TestBarcodeValues:
@pytest.mark.parametrize(
("filename", "mime_type", "tiff_support", "expected"),
[
pytest.param("simple.jpg", "image/jpeg", True, None, id="jpeg"),
pytest.param("simple.tiff", "image/tiff", False, None, id="tiff-off"),
pytest.param("simple.tiff", "image/tiff", True, [], id="tiff-on"),
],
)
def test_read_values_file_types(
self,
paperless_dirs: PaperlessDirs,
samples_dir: Path,
settings: Settings,
filename: str,
mime_type: str,
tiff_support: bool, # noqa: FBT001
expected: list | None,
) -> None:
"""
Files the scan doesn't support return None, so stored barcodes are
kept instead of being replaced with an empty list
"""
settings.CONSUMER_BARCODE_TIFF_SUPPORT = tiff_support
values = read_barcode_values(
samples_dir / filename,
mime_type,
BarcodeConfig(),
paperless_dirs.scratch_dir,
)
assert values == expected
def test_values_detected(
self,
barcode_reader: Callable[[Path], BarcodePlugin],
barcode_samples_dir: Path,
store_barcodes: None,
) -> None:
reader = barcode_reader(barcode_samples_dir / "barcode-qr-url.pdf")
assert reader.able_to_run
reader.run()
assert reader.metadata.barcodes == SAMPLE_VALUES
def test_values_disabled(
self,
barcode_reader: Callable[[Path], BarcodePlugin],
barcode_samples_dir: Path,
settings: Settings,
) -> None:
settings.CONSUMER_ENABLE_ASN_BARCODE = True
reader = barcode_reader(barcode_samples_dir / "barcode-qr-url.pdf")
reader.run()
assert reader.metadata.barcodes is None
def test_consume_and_reprocess(
self,
consume_sample: Callable[..., Document],
admin_client: APIClient,
store_barcodes: None,
) -> None:
"""
GIVEN:
- PDF with a QR code on each of its two pages
WHEN:
- File is consumed, the values are lost, and the document is reprocessed
THEN:
- The barcodes are stored, shown in the API and searchable
- Reprocessing reads them again
"""
document = consume_sample("barcode-qr-url.pdf")
assert _stored(document) == SAMPLE_VALUES
response = admin_client.get(f"/api/documents/{document.pk}/metadata/")
assert response.status_code == status.HTTP_200_OK
assert response.data["barcodes"] == SAMPLE_VALUES
response = admin_client.get(f"/api/documents/{document.pk}/")
assert "barcodes" not in response.data
response = admin_client.get("/api/documents/?query=barcodes:invoice")
assert [x["id"] for x in response.data["results"]] == [document.pk]
response = admin_client.get("/api/documents/?query=invoice")
assert response.data["results"] == []
document.barcodes.all().delete()
modified = Document.objects.get(pk=document.pk).modified
tasks.update_document_content_maybe_archive_file(document.pk)
assert _stored(document) == SAMPLE_VALUES
assert Document.objects.get(pk=document.pk).modified > modified
@pytest.mark.parametrize(
"read_values",
[
pytest.param({"side_effect": RuntimeError("broken")}, id="scan-fails"),
pytest.param({"return_value": None}, id="not-scannable"),
],
)
def test_reprocess_keeps_values(
self,
consume_sample: Callable[..., Document],
mocker: MockerFixture,
store_barcodes: None,
read_values: dict,
) -> None:
"""
GIVEN:
- A document with stored barcodes
WHEN:
- It is reprocessed, but the barcodes can't be read
THEN:
- The stored barcodes are kept
"""
document = consume_sample("barcode-qr-url.pdf")
mocker.patch("documents.tasks.read_barcode_values", **read_values)
tasks.update_document_content_maybe_archive_file(document.pk)
assert _stored(document) == SAMPLE_VALUES
def test_reprocess_tiff_support_off_keeps_values(
self,
consume_sample: Callable[..., Document],
settings: Settings,
store_barcodes: None,
) -> None:
"""
GIVEN:
- A TIFF document with barcodes stored while TIFF support was on
WHEN:
- TIFF support is turned off and the document is reprocessed
THEN:
- The stored barcodes are kept
"""
settings.CONSUMER_BARCODE_TIFF_SUPPORT = True
document = consume_sample("patch-code-t-middle.tiff")
stored = _stored(document)
assert stored
settings.CONSUMER_BARCODE_TIFF_SUPPORT = False
tasks.update_document_content_maybe_archive_file(document.pk)
assert _stored(document) == stored
def test_reprocess_values_disabled(
self,
consume_sample: Callable[..., Document],
settings: Settings,
store_barcodes: None,
) -> None:
document = consume_sample("barcode-qr-url.pdf")
settings.CONSUMER_STORE_BARCODE_VALUES = False
assert tasks._read_barcodes_for_reprocess(document) is None
def test_consume_version_stores_own_values(
self,
consume_sample: Callable[..., Document],
admin_client: APIClient,
store_barcodes: None,
) -> None:
"""
GIVEN:
- A document with stored barcodes
WHEN:
- A new version with a different barcode is consumed, like after
rotating or removing pages
THEN:
- The version keeps its own barcodes, the original ones are kept
- The metadata and the search use those of the newest version
"""
root = consume_sample("barcode-qr-url.pdf")
version = consume_sample("barcode-128-custom.pdf", root_document_id=root.pk)
assert version.root_document == root
assert _stored(version) == [
{"page": 1, "value": "CUSTOM BARCODE", "format": "Code128"},
]
assert root.barcodes.count() == 2
assert [x.value for x in root.get_effective_barcodes()] == ["CUSTOM BARCODE"]
response = admin_client.get(f"/api/documents/{root.pk}/metadata/")
assert [x["value"] for x in response.data["barcodes"]] == ["CUSTOM BARCODE"]
response = admin_client.get('/api/documents/?query=barcodes:"custom barcode"')
assert [x["id"] for x in response.data["results"]] == [root.pk]
response = admin_client.get("/api/documents/?query=barcodes:invoice")
assert response.data["results"] == []
def test_consume_version_scan_fails(
self,
consume_sample: Callable[..., Document],
mocker: MockerFixture,
store_barcodes: None,
) -> None:
"""
A failed scan of a new version is logged and doesn't stop consumption
"""
root = consume_sample("barcode-qr-url.pdf")
mocker.patch(
"documents.consumer.read_barcode_values",
side_effect=RuntimeError("broken"),
)
version = consume_sample("barcode-128-custom.pdf", root_document_id=root.pk)
assert version.root_document == root
assert not version.barcodes.exists()
+277 -32
View File
@@ -4,12 +4,16 @@ from pathlib import Path
from unittest import mock
import pikepdf
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.db import connection
from django.test import TestCase
from guardian.shortcuts import assign_perm
from django.test.utils import CaptureQueriesContext
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from pytest_mock import MockerFixture
from documents import bulk_edit
from documents.models import Correspondent
@@ -19,7 +23,10 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.permissions import set_permissions_for_objects
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object
class TestBulkEdit(DirectoriesMixin, TestCase):
@@ -436,7 +443,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
def test_set_permissions(self, m) -> None:
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
assign_perm("view_document", self.group1, self.doc1)
grant_object(self.group1, self.doc1, "view_document")
permissions = {
"view": {
@@ -478,8 +485,8 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
self.doc1.owner = self.user1
self.doc1.save()
assign_perm("view_document", self.user1, self.doc1)
assign_perm("view_document", self.group1, self.doc1)
grant_object(self.user1, self.doc1, "view_document")
grant_object(self.group1, self.doc1, "view_document")
permissions = {
"view": {
@@ -515,6 +522,173 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.assertEqual(groups_with_perms.count(), 2)
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
def test_set_permissions_batched_across_document_count(
self,
m,
) -> None:
"""
GIVEN:
- Many documents are being bulk-edited to set permissions at once
WHEN:
- set_permissions runs over a small batch vs. a much larger one
THEN:
- Permissions are applied correctly at both scales
- Query count does not grow with the number of documents, i.e.
each user/group is applied across all documents with one
batched call rather than one call per (document, identity)
pair
"""
permissions = {
"view": {
"users": [self.user1.id, self.user2.id],
"groups": [self.group2.id],
},
"change": {
"users": [self.user1.id],
"groups": [self.group2.id],
},
}
def run_with_n_documents(n: int) -> int:
docs = [
Document.objects.create(checksum=f"perm-{n}-{i}", title=f"perm-{n}-{i}")
for i in range(n)
]
with CaptureQueriesContext(connection) as ctx:
bulk_edit.set_permissions(
[doc.id for doc in docs],
set_permissions=permissions,
owner=self.owner,
merge=False,
)
for doc in docs:
self.assertEqual(get_users_with_perms(doc).count(), 2)
self.assertEqual(get_groups_with_perms(doc).count(), 1)
return len(ctx.captured_queries)
small_batch_queries = run_with_n_documents(5)
large_batch_queries = run_with_n_documents(50)
# A tolerance rather than equality, matching the N+1 check in
# test_views.py: bulk_create's batch_size caps rows per INSERT, so a
# large enough selection does legitimately add statements, and the
# per-process ContentType cache makes the first run carry an extra
# query. Neither can hide a regression to per-document assignment,
# which would be ~10x the small-batch count here.
self.assertLessEqual(
large_batch_queries,
small_batch_queries + 5,
"Permission assignment appears to scale with document count: "
f"{small_batch_queries} queries for 5 documents vs. "
f"{large_batch_queries} for 50",
)
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
def test_set_permissions_grants_direct_perm_even_if_already_granted_via_group(
self,
m,
) -> None:
"""
GIVEN:
- A user already has view access to a document via group
membership, with no direct grant of their own
WHEN:
- set_permissions explicitly grants that same user direct view
access via bulk_edit
THEN:
- A direct permission grant is created for the user, not skipped
because they already have equivalent access via the group
Regression test: guardian's queryset-aware assign_perm() (routed to
when the target is a list/queryset) skips creating a direct row for
anyone whose ObjectPermissionChecker.has_perm() already returns True
-- which includes group-derived access. The single-object assign_perm
this bulk path replaces has no such check; it always ensures a
direct row via get_or_create. Losing that guarantee would mean
revoking the group's grant later silently strips access that was
supposed to be explicit.
"""
self.doc1.owner = self.user1
self.doc1.save()
self.user1.groups.add(self.group1)
grant_object(self.group1, self.doc1, "view_document")
bulk_edit.set_permissions(
[self.doc1.id],
set_permissions={
"view": {"users": [self.user1.id], "groups": []},
},
merge=True,
)
direct_users = get_users_with_perms(
self.doc1,
only_with_perms_in=["view_document"],
with_group_users=False,
)
self.assertIn(self.user1, direct_users)
def test_set_permissions_for_objects_raises_for_unknown_action(self) -> None:
"""
GIVEN:
- An unrecognized permission action name with users to grant it
to
WHEN:
- set_permissions_for_objects is called
THEN:
- Permission.DoesNotExist is raised, not a silent no-op
The API rejects unknown action names before they get here, but a
direct caller could still pass one. Resolving the Permission via a
bare `.filter()` (which returns empty instead of raising) would
silently drop the grant and report success.
"""
with self.assertRaises(Permission.DoesNotExist):
set_permissions_for_objects(
{"not_a_real_action": {"users": [self.user1.id], "groups": []}},
Document,
[self.doc1.pk],
)
def test_set_permissions_for_objects_unknown_action_applies_nothing(
self,
) -> None:
"""
GIVEN:
- A permissions dict with a valid action ordered ahead of an
unrecognized one
WHEN:
- set_permissions_for_objects is called
THEN:
- Permission.DoesNotExist is raised
- The valid action ahead of it is not applied either
Every action is resolved before any row is written, so a bad action
name cannot leave a half-applied change behind. That matters because
BulkEditObjectsView turns this exception into a 400: without the
up-front resolution the client would be told the request failed
while the leading action had already been committed.
"""
with self.assertRaises(Permission.DoesNotExist):
set_permissions_for_objects(
{
"view": {"users": [self.user1.id], "groups": []},
"not_a_real_action": {"users": [self.user1.id], "groups": []},
},
Document,
[self.doc1.pk],
)
self.assertNotIn(
self.user1,
get_users_with_perms(
self.doc1,
only_with_perms_in=["view_document"],
with_group_users=False,
),
)
@mock.patch("documents.models.Document.delete")
def test_delete_documents_old_uuid_field(self, m) -> None:
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
@@ -1466,6 +1640,16 @@ class TestPDFActions(DirectoriesMixin, TestCase):
mock_group.assert_not_called()
mock_consume_file.assert_not_called()
@mock.patch("pikepdf.open")
def test_edit_pdf_rejects_invalid_operations(self, mock_open) -> None:
for operations in ([], [{"page": 1, "doc": 2**32}]):
with self.subTest(operations=operations):
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaisesRegex(ValueError, "index is out of bounds"):
bulk_edit.edit_pdf([self.doc2.id], operations)
mock_open.assert_not_called()
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@@ -1789,18 +1973,22 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertIn("Error removing password from document", cm.output[0])
class TestBulkEditReprocess(DirectoriesMixin, TestCase):
def setUp(self) -> None:
super().setUp()
self.doc = Document.objects.create(
title="test",
checksum="A",
mime_type="application/pdf",
@pytest.mark.django_db
class TestBulkEditReprocess:
@pytest.fixture
def mock_task(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch(
"documents.bulk_edit.update_document_content_maybe_archive_file",
)
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
def test_reprocess_defaults_to_local(self, mock_task: mock.Mock) -> None:
@staticmethod
def _queued_ids(mock_task: mock.MagicMock) -> list[int]:
return [
call.kwargs["kwargs"]["document_id"]
for call in mock_task.apply_async.call_args_list
]
def test_reprocess_defaults_to_local(self, mock_task: mock.MagicMock) -> None:
"""
GIVEN:
- A reprocess request that says nothing about remote OCR
@@ -1809,18 +1997,17 @@ class TestBulkEditReprocess(DirectoriesMixin, TestCase):
THEN:
- The task is queued without asking for the remote engine
"""
result = bulk_edit.reprocess([self.doc.id])
doc = DocumentFactory()
assert bulk_edit.reprocess([doc.id]) == "OK"
self.assertEqual(result, "OK")
mock_task.apply_async.assert_called_once()
_, kwargs = mock_task.apply_async.call_args
self.assertEqual(
kwargs["kwargs"],
{"document_id": self.doc.id, "remote_ocr": False},
)
assert mock_task.apply_async.call_args.kwargs["kwargs"] == {
"document_id": doc.id,
"remote_ocr": False,
}
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
def test_reprocess_passes_remote_ocr(self, mock_task: mock.Mock) -> None:
def test_reprocess_passes_remote_ocr(self, mock_task: mock.MagicMock) -> None:
"""
GIVEN:
- A reprocess request that explicitly asks for remote OCR
@@ -1829,14 +2016,72 @@ class TestBulkEditReprocess(DirectoriesMixin, TestCase):
THEN:
- The request is forwarded to the task for every document
"""
other = Document.objects.create(
title="test2",
checksum="B",
mime_type="application/pdf",
docs = DocumentFactory.create_batch(2)
bulk_edit.reprocess([doc.id for doc in docs], remote_ocr=True)
assert mock_task.apply_async.call_count == 2
for call in mock_task.apply_async.call_args_list:
assert call.kwargs["kwargs"]["remote_ocr"]
def test_reprocess_root_uses_latest_version(
self,
mock_task: mock.MagicMock,
) -> None:
"""
GIVEN:
- A root document with two versions
WHEN:
- reprocess is called with the root document
THEN:
- The latest version is reprocessed, not the root's original file
"""
root = DocumentFactory()
DocumentFactory(root_document=root, version_index=1)
latest = DocumentFactory(root_document=root, version_index=2)
bulk_edit.reprocess([root.id])
assert self._queued_ids(mock_task) == [latest.id]
def test_reprocess_explicit_version(self, mock_task: mock.MagicMock) -> None:
"""
GIVEN:
- A root document with two versions
WHEN:
- reprocess is called with the older version
THEN:
- That version is reprocessed
"""
root = DocumentFactory()
older = DocumentFactory(root_document=root, version_index=1)
DocumentFactory(root_document=root, version_index=2)
bulk_edit.reprocess([older.id])
assert self._queued_ids(mock_task) == [older.id]
def test_reprocess_root_and_latest_version_dispatches_once(
self,
mock_task: mock.MagicMock,
) -> None:
"""
GIVEN:
- A root document with two versions, the latest created on a
different date than the root
WHEN:
- reprocess is called with both the root and its latest version
THEN:
- The latest version is reprocessed only once
"""
root = DocumentFactory(created=date(2024, 1, 1))
DocumentFactory(root_document=root, version_index=1)
latest = DocumentFactory(
root_document=root,
version_index=2,
created=date(2025, 1, 1),
)
bulk_edit.reprocess([self.doc.id, other.id], remote_ocr=True)
bulk_edit.reprocess([root.id, latest.id])
self.assertEqual(mock_task.apply_async.call_count, 2)
for call in mock_task.apply_async.call_args_list:
self.assertTrue(call.kwargs["kwargs"]["remote_ocr"])
assert self._queued_ids(mock_task) == [latest.id]
-58
View File
@@ -1,58 +0,0 @@
from documents.caching import StoredLRUCache
from paperless.signed_pickle import HMAC_SIZE
from paperless.signed_pickle import signed_pickle_dumps
from paperless.signed_pickle import signed_pickle_loads
def test_lru_cache_entries() -> None:
CACHE_TTL = 1
# LRU cache with a capacity of 2 elements
cache = StoredLRUCache("test_lru_cache_key", 2, backend_ttl=CACHE_TTL)
cache.set(1, 1)
cache.set(2, 2)
assert cache.get(2) == 2
assert cache.get(1) == 1
# The oldest entry (2) should be removed
cache.set(3, 3)
assert cache.get(3) == 3
assert not cache.get(2)
assert cache.get(1) == 1
# Save the cache, restore it and check it overwrites the current cache in memory
cache.save()
cache.set(4, 4)
assert not cache.get(3)
cache.load()
assert not cache.get(4)
assert cache.get(3) == 3
assert cache.get(1) == 1
def test_stored_lru_cache_key_ttl(mocker) -> None:
mock_backend = mocker.Mock()
cache = StoredLRUCache("test_key", backend=mock_backend, backend_ttl=321)
# Simulate storing values
cache.set("x", "X")
cache.set("y", "Y")
cache.save()
# Assert backend.set was called with pickled data, key and TTL
mock_backend.set.assert_called_once()
key, data, timeout = mock_backend.set.call_args[0]
assert key == "test_key"
assert timeout == 321
assert signed_pickle_loads(data) == {"x": "X", "y": "Y"}
def test_stored_lru_cache_rejects_tampered_data(mocker) -> None:
serialized_data = bytearray(signed_pickle_dumps({"x": "X"}))
serialized_data[HMAC_SIZE] ^= 0xFF
mock_backend = mocker.Mock()
mock_backend.get.return_value = bytes(serialized_data)
cache = StoredLRUCache("test_key", backend=mock_backend)
cache.load()
assert cache.get("x") is None
+518 -65
View File
@@ -1,16 +1,26 @@
import re
import pickle
import warnings
from datetime import UTC
from datetime import datetime
from pathlib import Path
from unittest import mock
import numpy as np
import pytest
from django.conf import settings
from django.db import connection
from django.test import TestCase
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from pytest_django.fixtures import Settings
from pytest_mock import MockerFixture
from sklearn.neural_network import MLPClassifier
from documents.classifier import ClassifierModelCorruptError
from documents.classifier import DocumentClassifier
from documents.classifier import IncompatibleClassifierVersionError
from documents.classifier import _predict_with_threshold
from documents.classifier import _text_analyzer
from documents.classifier import load_classifier
from documents.models import Correspondent
from documents.models import Document
@@ -18,18 +28,13 @@ from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.tests.helpers import dummy_preprocess
from paperless.settings import CLASSIFIER_LANGUAGES
from paperless.signed_pickle import HMAC_SIZE
from paperless.signed_pickle import signed_pickle_dumps
def dummy_preprocess(content: str, **kwargs):
"""
Simpler, faster pre-processing for testing purposes
"""
content = content.lower().strip()
content = re.sub(r"\s+", " ", content)
return content
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import TagFactory
class TestClassifier(DirectoriesMixin, TestCase):
@@ -625,6 +630,103 @@ class TestClassifier(DirectoriesMixin, TestCase):
self.assertEqual(self.classifier.predict_storage_path(doc1.content), sp.pk)
self.assertIsNone(self.classifier.predict_storage_path(doc2.content))
def test_predict_rejects_prediction_below_match_threshold(self) -> None:
"""
GIVEN:
- Classifiers trained against test data with confident predictions
WHEN:
- CLASSIFIER_MATCH_THRESHOLD exceeds the model's confidence
THEN:
- Every predict_* method discards the match in favor of no match
"""
c1 = Correspondent.objects.create(
name="c1",
matching_algorithm=Correspondent.MATCH_AUTO,
)
dt1 = DocumentType.objects.create(
name="dt1",
matching_algorithm=DocumentType.MATCH_AUTO,
)
sp1 = StoragePath.objects.create(
name="sp1",
matching_algorithm=StoragePath.MATCH_AUTO,
)
doc1 = Document.objects.create(
title="doc1",
content="this is a document from c1",
correspondent=c1,
document_type=dt1,
storage_path=sp1,
checksum="A",
)
Document.objects.create(
title="doc2",
content="this is a document from no one",
checksum="B",
)
self.classifier.train()
predictors = {
"correspondent": self.classifier.predict_correspondent,
"document_type": self.classifier.predict_document_type,
"storage_path": self.classifier.predict_storage_path,
}
# No real prediction can reach a confidence this high, so this
# isolates the threshold check from the model's actual output.
with override_settings(CLASSIFIER_MATCH_THRESHOLD=0.999999):
for name, predict in predictors.items():
with self.subTest(field=name):
self.assertIsNone(predict(doc1.content))
def test_train_uses_balanced_sample_weight(self) -> None:
"""
GIVEN:
- A training set with correspondents, document types and storage paths
WHEN:
- The classifier is trained
THEN:
- Each MLP classifier is fit with balanced sample weights, so that
over-represented classes don't dominate predictions
"""
c1 = Correspondent.objects.create(
name="c1",
matching_algorithm=Correspondent.MATCH_AUTO,
)
dt1 = DocumentType.objects.create(
name="dt1",
matching_algorithm=DocumentType.MATCH_AUTO,
)
sp1 = StoragePath.objects.create(
name="sp1",
matching_algorithm=StoragePath.MATCH_AUTO,
)
Document.objects.create(
title="doc1",
content="this is a document from c1",
correspondent=c1,
document_type=dt1,
storage_path=sp1,
checksum="A",
)
Document.objects.create(
title="doc2",
content="this is a document from no one",
checksum="B",
)
with mock.patch(
"sklearn.utils.class_weight.compute_sample_weight",
return_value=None,
) as mocked_compute_sample_weight:
self.classifier.train()
self.assertEqual(mocked_compute_sample_weight.call_count, 3)
for call in mocked_compute_sample_weight.call_args_list:
self.assertEqual(call.args[0], "balanced")
def test_one_tag_predict(self) -> None:
t1 = Tag.objects.create(name="t1", matching_algorithm=Tag.MATCH_AUTO, pk=12)
@@ -810,65 +912,416 @@ class TestClassifier(DirectoriesMixin, TestCase):
load_classifier(raise_exception=True)
def test_preprocess_content() -> None:
class TestClassifierSave:
@pytest.fixture
def model_file(self, tmp_path: Path, settings: Settings) -> Path:
settings.MODEL_FILE = tmp_path / "classifier.pickle"
return settings.MODEL_FILE
@pytest.fixture
def classifier(self) -> DocumentClassifier:
classifier = DocumentClassifier()
classifier.last_doc_change_time = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC)
classifier.last_auto_type_hash = b"\x01" * 32
return classifier
def test_save_writes_signed_pickle(
self,
model_file: Path,
classifier: DocumentClassifier,
) -> None:
"""
GIVEN:
- A classifier with training state
WHEN:
- The classifier is saved
THEN:
- The file is the HMAC of the pickled state followed by that pickle
- The pickle uses the highest protocol
- The saved state loads back into a new classifier
- No temporary file is left behind
"""
classifier.save()
raw = model_file.read_bytes()
signature = raw[: DocumentClassifier.HMAC_SIZE]
data = raw[DocumentClassifier.HMAC_SIZE :]
assert signature == DocumentClassifier._compute_hmac(data)
# A pickle opens with the PROTO opcode followed by the protocol number
assert data[:2] == bytes([pickle.PROTO[0], pickle.HIGHEST_PROTOCOL])
assert pickle.loads(data)[:3] == (
DocumentClassifier.FORMAT_VERSION,
classifier.last_doc_change_time,
classifier.last_auto_type_hash,
)
loaded = DocumentClassifier()
loaded.load()
assert loaded.last_doc_change_time == classifier.last_doc_change_time
assert loaded.last_auto_type_hash == classifier.last_auto_type_hash
assert not model_file.with_name(f"{model_file.name}.part").exists()
def test_save_failure_removes_partial_file(
self,
model_file: Path,
classifier: DocumentClassifier,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- An existing classifier model file
WHEN:
- Saving a new classifier fails part way through writing
THEN:
- The error is raised
- The partially written temporary file is removed
- The existing model file is left untouched
"""
model_file.write_bytes(b"existing model")
mocker.patch(
"documents.classifier.pickle.dump",
side_effect=RuntimeError("disk full"),
)
with pytest.raises(RuntimeError, match="disk full"):
classifier.save()
assert not model_file.with_name(f"{model_file.name}.part").exists()
assert model_file.read_bytes() == b"existing model"
class _StubProbaClassifier:
"""
A fake scikit-learn classifier exposing just enough of the API for
`_predict_with_threshold`: `classes_` and `predict_proba`.
"""
def __init__(self, classes: list[int], probabilities: list[float]) -> None:
self.classes_ = np.array(classes)
self._probabilities = np.array([probabilities])
def predict_proba(self, X) -> np.ndarray:
return self._probabilities
@pytest.mark.parametrize(
("classes", "probabilities", "threshold", "expected"),
[
# confident prediction above the threshold is returned
([-1, 3], [0.1, 0.9], 0.6, 3),
# prediction below the threshold is discarded
([-1, 3], [0.45, 0.55], 0.6, None),
# boundary: exactly at the threshold is accepted, not discarded
([-1, 3], [0.4, 0.6], 0.6, 3),
# the winning class is the "no match" pseudo-class, regardless of its
# own confidence
([-1, 3], [0.99, 0.01], 0.0, None),
# threshold of 0.0 disables the confidence check entirely
([-1, 3], [0.45, 0.55], 0.0, 3),
],
)
def test_predict_with_threshold(classes, probabilities, threshold, expected) -> None:
classifier = _StubProbaClassifier(classes, probabilities)
result = _predict_with_threshold(classifier, X=None, threshold=threshold)
assert result == expected
def test_classifier_match_threshold_default() -> None:
"""
GIVEN:
- Advanced text processing is enabled (default)
WHEN:
- Classifier preprocesses a document's content
- No PAPERLESS_CLASSIFIER_MATCH_THRESHOLD environment variable is set
THEN:
- Processed content matches the expected output (stemmed words)
- The classifier match threshold defaults to 0.3
"""
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
content = f.read()
with (Path(__file__).parent / "samples" / "preprocessed_content_advanced.txt").open(
"r",
) as f:
expected_preprocess_content = f.read().rstrip()
classifier = DocumentClassifier()
result = classifier.preprocess_content(content)
assert result == expected_preprocess_content
assert settings.CLASSIFIER_MATCH_THRESHOLD == 0.3
def test_preprocess_content_nltk_disabled() -> None:
"""
GIVEN:
- Advanced text processing is disabled
WHEN:
- Classifier preprocesses a document's content
THEN:
- Processed content matches the expected output (unstemmed words)
"""
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
content = f.read()
with (Path(__file__).parent / "samples" / "preprocessed_content.txt").open(
"r",
) as f:
expected_preprocess_content = f.read().rstrip()
classifier = DocumentClassifier()
with mock.patch("documents.classifier.ADVANCED_TEXT_PROCESSING_ENABLED", new=False):
result = classifier.preprocess_content(content)
assert result == expected_preprocess_content
class TestPredictWithThresholdSingleClass:
@pytest.fixture
def single_class_model(self) -> tuple[MLPClassifier, np.ndarray]:
X = np.random.default_rng(0).random((2, 5))
return MLPClassifier(tol=0.01, random_state=0).fit(X, [3, 3]), X[:1]
def test_returns_only_class(
self,
single_class_model: tuple[MLPClassifier, np.ndarray],
) -> None:
"""
GIVEN:
- A classifier fitted on a single class (every document shares one label)
WHEN:
- A prediction is requested with the threshold disabled
THEN:
- No error is raised and the only class is returned
"""
classifier, X = single_class_model
assert _predict_with_threshold(classifier, X, threshold=0.0) == 3
def test_applies_threshold(
self,
single_class_model: tuple[MLPClassifier, np.ndarray],
) -> None:
"""
GIVEN:
- A classifier fitted on a single class
WHEN:
- A prediction is requested
THEN:
- The class's confidence is still compared against the threshold
"""
classifier, X = single_class_model
confidence = classifier.predict_proba(X)[0][0]
assert _predict_with_threshold(classifier, X, threshold=confidence - 0.01) == 3
assert (
_predict_with_threshold(classifier, X, threshold=confidence + 0.01) is None
)
def test_preprocess_content_nltk_load_fail(mocker) -> None:
"""
GIVEN:
- NLTK stop words fail to load
WHEN:
- Classifier preprocesses a document's content
THEN:
- Processed content matches the expected output (unstemmed words)
"""
_module = mocker.MagicMock(name="nltk_corpus_mock")
_module.stopwords.words.side_effect = AttributeError()
mocker.patch.dict("sys.modules", {"nltk.corpus": _module})
classifier = DocumentClassifier()
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
content = f.read()
with (Path(__file__).parent / "samples" / "preprocessed_content.txt").open(
"r",
) as f:
expected_preprocess_content = f.read().rstrip()
result = classifier.preprocess_content(content)
assert result == expected_preprocess_content
class TestPreprocessContent:
@pytest.fixture
def samples(self) -> Path:
return Path(__file__).parent / "samples"
@pytest.fixture
def content(self, samples: Path) -> str:
return (samples / "content.txt").read_text()
def test_supported_language(
self,
settings: Settings,
samples: Path,
content: str,
) -> None:
"""
GIVEN:
- The classifier language is English, the default
WHEN:
- Document content is preprocessed
THEN:
- Stop words are removed and the remaining words are stemmed
"""
settings.CLASSIFIER_LANGUAGE = "english"
expected = (samples / "preprocessed_content_advanced.txt").read_text()
assert DocumentClassifier().preprocess_content(content) == expected.rstrip()
def test_unsupported_language(
self,
settings: Settings,
samples: Path,
content: str,
) -> None:
"""
GIVEN:
- No classifier language (the OCR language has no stemming support)
WHEN:
- Document content is preprocessed
THEN:
- The content is only lowercased and split into words
"""
settings.CLASSIFIER_LANGUAGE = None
expected = (samples / "preprocessed_content.txt").read_text()
assert DocumentClassifier().preprocess_content(content) == expected.rstrip()
@pytest.mark.parametrize(
"language",
[pytest.param("english", id="supported"), pytest.param(None, id="unsupported")],
)
def test_empty_content(self, settings: Settings, language: str | None) -> None:
"""
GIVEN:
- Empty document content
WHEN:
- The content is preprocessed
THEN:
- The result is empty
"""
settings.CLASSIFIER_LANGUAGE = language
assert DocumentClassifier().preprocess_content("") == ""
@pytest.mark.django_db
class TestClassifierTrainTagLabels:
@pytest.fixture(autouse=True)
def _simple_preprocess(self, mocker: MockerFixture) -> None:
mocker.patch.object(
DocumentClassifier,
"preprocess_content",
side_effect=dummy_preprocess,
)
@pytest.fixture
def auto_tags(self) -> list[Tag]:
return TagFactory.create_batch(2, matching_algorithm=MatchingModel.MATCH_AUTO)
def test_train_query_count_does_not_scale_with_documents(
self,
auto_tags: list[Tag],
) -> None:
"""
GIVEN:
- Documents with auto matching tags
WHEN:
- The classifier is trained, then more documents are added and it is
trained again
THEN:
- Both trainings run the same number of queries
"""
for doc in DocumentFactory.create_batch(2):
doc.tags.set(auto_tags)
with CaptureQueriesContext(connection) as few_documents:
DocumentClassifier().train()
for doc in DocumentFactory.create_batch(6):
doc.tags.set(auto_tags)
with CaptureQueriesContext(connection) as more_documents:
DocumentClassifier().train()
assert len(more_documents) == len(few_documents)
def test_train_uses_only_auto_tags_as_labels(
self,
auto_tags: list[Tag],
) -> None:
"""
GIVEN:
- Documents with both auto matching and non auto matching tags
WHEN:
- The classifier is trained
THEN:
- Only the auto matching tags are used as tag labels
"""
manual_tag = TagFactory(matching_algorithm=MatchingModel.MATCH_ANY)
first, second, third = DocumentFactory.create_batch(3)
first.tags.set([auto_tags[0], manual_tag])
second.tags.set([auto_tags[1], manual_tag])
third.tags.set([*auto_tags, manual_tag])
classifier = DocumentClassifier()
classifier.train()
assert list(classifier.tags_binarizer.classes_) == sorted(
tag.pk for tag in auto_tags
)
@pytest.mark.django_db
class TestClassifierTrainContent:
def test_train_content_follows_label_order_across_chunks(
self,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- More documents than fit in one content chunk
WHEN:
- The classifier is trained
THEN:
- Every document's content is preprocessed once, in document order
"""
mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 2)
docs = DocumentFactory.create_batch(5)
preprocess = mocker.patch.object(
DocumentClassifier,
"preprocess_content",
side_effect=dummy_preprocess,
)
DocumentClassifier().train()
assert [call.args[0] for call in preprocess.call_args_list] == [
doc.content for doc in sorted(docs, key=lambda doc: doc.pk)
]
def test_train_document_deleted_while_training(
self,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- Two documents
WHEN:
- The second document is deleted after its labels were gathered, but
before its content is fetched
THEN:
- Training completes
- The deleted document is trained with empty content, keeping labels
and content aligned
"""
mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 1)
first, second = DocumentFactory.create_batch(2)
def delete_second_then_preprocess(content: str, **kwargs) -> str:
if content == first.content:
second.delete()
return dummy_preprocess(content)
preprocess = mocker.patch.object(
DocumentClassifier,
"preprocess_content",
side_effect=delete_second_then_preprocess,
)
assert DocumentClassifier().train()
assert [call.args[0] for call in preprocess.call_args_list] == [
first.content,
"",
]
class TestTextAnalyzer:
@pytest.mark.parametrize(
"language",
[
pytest.param(language, id=language)
for language in sorted(set(CLASSIFIER_LANGUAGES.values()))
],
)
def test_builds_for_every_classifier_language(self, language: str) -> None:
"""
GIVEN:
- A language the classifier supports
WHEN:
- Text is analyzed with its classifier language
THEN:
- Tokens are produced
"""
assert _text_analyzer(language).analyze("Paperless invoice 2026")
def test_english_removes_snowball_stop_words(self) -> None:
"""
GIVEN:
- English text with a contraction and stop words missing from
Tantivy's own English list
WHEN:
- The text is analyzed
THEN:
- All stop words are removed, including the contraction
- The remaining words are stemmed
"""
tokens = _text_analyzer("english").analyze(
"They were about to pay the invoices, don't worry",
)
assert tokens == ["pay", "invoic", "worri"]
def test_keeps_underscores_within_tokens(self) -> None:
"""
GIVEN:
- Text with a word joined by an underscore
WHEN:
- The text is analyzed
THEN:
- The word stays one token
"""
tokens = _text_analyzer("english").analyze("tax_id")
assert tokens == ["tax_id"]
+36 -7
View File
@@ -30,11 +30,14 @@ from documents.models import Tag
from documents.parsers import ParseError
from documents.plugins.helpers import ProgressStatusOptions
from documents.tasks import sanity_check
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import GetConsumerMixin
from paperless_mail.models import MailRule
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
from paperless_testing.fakes.progress import FakeProgressManager
from paperless_testing.permissions import grant_object
class _BaseNewStyleParser:
@@ -769,14 +772,14 @@ class TestConsumer(
original_modified = timezone.now() - datetime.timedelta(days=1)
Document.objects.filter(pk=root_doc.pk).update(modified=original_modified)
actor = User.objects.create_user(
actor = UserFactory(
username="actor",
email="actor@example.com",
password="password",
)
version_file = self.get_test_file2()
status = DummyProgressManager(version_file.name, None)
status = FakeProgressManager(version_file.name, None)
overrides = DocumentMetadataOverrides(
version_label="v2",
actor_id=actor.pk,
@@ -839,7 +842,7 @@ class TestConsumer(
assert root_doc is not None
version_file = self.get_test_file2()
status = DummyProgressManager(version_file.name, None)
status = FakeProgressManager(version_file.name, None)
overrides = DocumentMetadataOverrides(
filename="valid_pdf_version-upload",
actor_id=999999,
@@ -896,7 +899,7 @@ class TestConsumer(
assert root_doc is not None
def consume_version(version_file: Path) -> Document:
status = DummyProgressManager(version_file.name, None)
status = FakeProgressManager(version_file.name, None)
overrides = DocumentMetadataOverrides()
doc = ConsumableDocument(
DocumentSource.ApiUpload,
@@ -1682,6 +1685,32 @@ class TestMetadataOverrides(TestCase):
self.assertEqual(base.actor_id, 2)
self.assertEqual(base.version_label, "v2")
def test_from_document_copies_user_and_group_grants_separately(self) -> None:
"""
GIVEN:
- A document with view and change granted directly to one user and
to a group with another user as a member
WHEN:
- Overrides are built from the document
THEN:
- The user lists hold only the directly granted user
- The group lists hold the group
"""
direct_user = UserFactory()
group_member = UserFactory()
group = Group.objects.create(name="group")
group_member.groups.add(group)
document = DocumentFactory()
grant_object(direct_user, document, "view_document", "change_document")
grant_object(group, document, "view_document", "change_document")
overrides = DocumentMetadataOverrides.from_document(document)
self.assertEqual(overrides.view_users, [direct_user.pk])
self.assertEqual(overrides.change_users, [direct_user.pk])
self.assertEqual(overrides.view_groups, [group.pk])
self.assertEqual(overrides.change_groups, [group.pk])
class TestBarcodeApplyDetectedASN(TestCase):
"""
@@ -0,0 +1,425 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import TYPE_CHECKING
import pytest
from django.db import connection
from django.test.utils import CaptureQueriesContext
from rest_framework import status
from documents.models import Document
from documents.versioning import LATEST_VERSION_CONTENT_PREFETCH_ATTR
from documents.versioning import has_prefetched_effective_content
from documents.versioning import latest_version_content_prefetch
from documents.views import DocumentViewSet
from paperless_testing.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
class TestNeedsEffectiveContentAnnotation:
"""
DocumentViewSet._needs_effective_content_annotation() decides whether
the effective_content correlated subquery is worth attaching to the
queryset at all -- see TestDocumentListEffectiveContentAnnotation below
for why. This only checks that decision's own logic (a plain query-param
membership test), not that Django/DRF's filtering machinery works.
"""
@pytest.mark.parametrize(
("params", "expected"),
[
({}, False),
({"ordering": "-added"}, False),
({"tags__id__in": "1,2"}, False),
({"search": ""}, False),
({"search": " "}, False),
({"content__icontains": ""}, False),
({"search": "foo"}, True),
({"title_content": "foo"}, True),
({"content__istartswith": "foo"}, True),
({"content__iendswith": "foo"}, True),
({"content__icontains": "foo"}, True),
({"content__iexact": "foo"}, True),
],
)
def test_detects_content_filter_params(
self,
params: dict[str, str],
expected: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A view bound to a request carrying the given query params
WHEN:
- Checking whether the effective_content annotation is needed
THEN:
- It is needed only for requests that actually filter on it
"""
view = DocumentViewSet()
view.request = SimpleNamespace(query_params=params)
assert view._needs_effective_content_annotation() is expected
class TestNeedsEffectiveContentPrefetch:
"""
DocumentViewSet._needs_effective_content_prefetch() decides whether the
single-version content prefetch is worth attaching. It has to read the
`fields` param exactly the way get_serializer() does, or a request whose
response includes content ends up without the prefetch and pays
get_effective_content()'s per-instance fallback instead.
"""
@pytest.mark.parametrize(
("params", "expected"),
[
pytest.param({}, True, id="no-fields-param-keeps-every-field"),
pytest.param({"fields": ""}, True, id="blank-fields-keeps-every-field"),
pytest.param(
{"fields": "id,content"},
True,
id="content-among-requested-fields",
),
pytest.param({"fields": "content"}, True, id="content-only"),
pytest.param({"fields": "id"}, False, id="content-not-requested"),
pytest.param(
{"fields": "id,title"},
False,
id="several-fields-without-content",
),
],
)
def test_detects_whether_content_can_reach_the_response(
self,
params: dict[str, str],
expected: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A view bound to a request carrying the given query params
WHEN:
- Checking whether the content prefetch is needed
THEN:
- It is needed exactly when get_serializer() would emit content,
which treats a blank `fields` the same as an absent one
"""
view = DocumentViewSet()
view.request = SimpleNamespace(query_params=params)
assert view._needs_effective_content_prefetch() is expected
@pytest.mark.django_db
class TestDocumentListEffectiveContentAnnotation:
"""
DocumentViewSet.get_queryset() only attaches the effective_content
correlated subquery when a request actually filters on it. Attaching it
unconditionally re-executes it once per candidate row before the page's
LIMIT is applied -- fine on SQLite/Postgres, but pathological on
MariaDB's default cardinality estimation for the root_document_id
self-join once candidate counts get large (see the root_document_id /
effective_content perf investigation).
"""
def test_list_without_content_filter_skips_annotation_but_returns_latest_content(
self,
admin_client: APIClient,
) -> None:
"""
GIVEN:
- A root document whose latest version has different content
WHEN:
- Listing documents with no search/content-filter param
THEN:
- The response still reflects the latest version's content
- The database never evaluates effective_content per row
"""
root = DocumentFactory(content="old-root-content")
DocumentFactory(
root_document=root,
version_index=1,
content="new-version-content",
)
with CaptureQueriesContext(connection) as ctx:
response = admin_client.get("/api/documents/?fields=id,content")
assert response.status_code == status.HTTP_200_OK
assert response.data["results"] == [
{"id": root.id, "content": "new-version-content"},
]
assert not any(
"effective_content" in query["sql"] for query in ctx.captured_queries
)
@pytest.mark.parametrize(
"fields_param",
[
pytest.param("", id="blank-fields"),
pytest.param("id,content", id="content-requested"),
],
)
def test_content_resolves_without_a_query_per_document(
self,
admin_client: APIClient,
fields_param: str,
) -> None:
"""
GIVEN:
- One versioned root document, then two more
WHEN:
- Listing documents with a `fields` param that keeps content
THEN:
- Every root's content resolves to its latest version's
- The query count does not grow with the number of documents,
i.e. a blank `fields` does not skip the prefetch and fall back
to loading each root's deferred version content
"""
first = DocumentFactory(content="first-root-content")
DocumentFactory(
root_document=first,
version_index=1,
content="first-version-content",
)
with CaptureQueriesContext(connection) as one_document:
response = admin_client.get(f"/api/documents/?fields={fields_param}")
assert response.status_code == status.HTTP_200_OK
assert [r["content"] for r in response.data["results"]] == [
"first-version-content",
]
for index in range(2):
root = DocumentFactory(content=f"root-content-{index}")
DocumentFactory(
root_document=root,
version_index=1,
content=f"version-content-{index}",
)
with CaptureQueriesContext(connection) as three_documents:
response = admin_client.get(f"/api/documents/?fields={fields_param}")
assert response.status_code == status.HTTP_200_OK
assert sorted(r["content"] for r in response.data["results"]) == [
"first-version-content",
"version-content-0",
"version-content-1",
]
assert len(_get_document_queries(three_documents)) == len(
_get_document_queries(one_document),
)
def test_list_without_content_field_skips_prefetch_and_omits_content(
self,
admin_client: APIClient,
) -> None:
"""
GIVEN:
- A versioned root document
WHEN:
- Listing documents without asking for content
THEN:
- Content is neither serialized nor resolved
- Nothing pays for the prefetch or the per-instance fallback
"""
root = DocumentFactory(content="root-content")
DocumentFactory(
root_document=root,
version_index=1,
content="version-content",
)
with CaptureQueriesContext(connection) as ctx:
response = admin_client.get("/api/documents/?fields=id")
assert response.status_code == status.HTTP_200_OK
assert response.data["results"] == [{"id": root.id}]
assert _get_effective_content_fallback_queries(ctx) == []
# Only the list query itself reads a content column: no extra query
# for the skipped prefetch, none for a per-instance fallback
content_queries = [
query
for query in ctx.captured_queries
if '"documents_document"."content"' in query["sql"]
]
assert len(content_queries) == 1
def test_latest_version_content_prefetch_carries_only_the_newest_version(
self,
) -> None:
"""
GIVEN:
- A root document with two versions
WHEN:
- Fetching the root through latest_version_content_prefetch()
THEN:
- The prefetch carries only the single newest version, not every
historical version's content (the whole point of not reusing
the metadata-only "versions" prefetch for this)
"""
root = DocumentFactory(content="root-content")
DocumentFactory(
root_document=root,
version_index=1,
content="older-version-content",
)
DocumentFactory(
root_document=root,
version_index=2,
content="newest-version-content",
)
fetched_root = (
Document.objects.filter(pk=root.pk)
.prefetch_related(
latest_version_content_prefetch(),
)
.get()
)
latest = getattr(fetched_root, LATEST_VERSION_CONTENT_PREFETCH_ATTR)
assert [v.content for v in latest] == ["newest-version-content"]
class TestHasPrefetchedEffectiveContent:
"""
DocumentSerializer.to_representation() only calls get_effective_content()
when has_prefetched_effective_content() says it's cheap -- otherwise a
caller that never set up an annotation or prefetch (TrashView,
GlobalSearchView, which build their own querysets and don't display
content at all) would pay for a per-instance query nobody asked for.
"""
def test_false_with_no_annotation_or_prefetch(self) -> None:
"""
GIVEN:
- A document the ORM never annotated or prefetched for
WHEN:
- Asking whether its effective content is already resolved
THEN:
- It is not, so the serializer must leave it alone
"""
document = DocumentFactory.build()
assert has_prefetched_effective_content(document) is False
def test_true_with_effective_content_annotation(self) -> None:
"""
GIVEN:
- A document carrying the queryset's effective_content annotation
WHEN:
- Asking whether its effective content is already resolved
THEN:
- It is, straight off the annotation
"""
document = DocumentFactory.build()
document.effective_content = "resolved"
assert has_prefetched_effective_content(document) is True
def test_true_with_lean_prefetch_attr_even_when_empty(self) -> None:
"""
GIVEN:
- A document the lean content prefetch ran for, finding no versions
WHEN:
- Asking whether its effective content is already resolved
THEN:
- It is: an empty prefetch is an answer, not a missing one
"""
document = DocumentFactory.build()
setattr(document, LATEST_VERSION_CONTENT_PREFETCH_ATTR, [])
assert has_prefetched_effective_content(document) is True
def test_true_with_metadata_versions_prefetch_cache(self) -> None:
"""
GIVEN:
- A document carrying only the metadata "versions" prefetch
WHEN:
- Asking whether its effective content is already resolved
THEN:
- It is, via get_effective_content()'s prefetch-cache branch
"""
document = DocumentFactory.build()
document._prefetched_objects_cache = {"versions": []}
assert has_prefetched_effective_content(document) is True
def _get_document_queries(
ctx: CaptureQueriesContext,
) -> list[dict[str, str]]:
"""
The queries a list request spends on the documents themselves, i.e.
everything but the one-time django_content_type lookup guardian's
permission filtering makes. That lookup is process-cached, and the
autouse fixture in conftest clears the cache before every test, so it
lands in whichever request happens to run first and never repeats --
counting it makes a request look like it costs one query more than the
identical request after it.
"""
return [q for q in ctx.captured_queries if '"django_content_type"' not in q["sql"]]
def _get_effective_content_fallback_queries(
ctx: CaptureQueriesContext,
) -> list[dict[str, str]]:
"""
Document.get_effective_content()'s per-instance fallback (no annotation,
no prefetch) is a `.values_list("content", flat=True).first()` query --
a SELECT of just the content column. Distinct from get_versions()'s own,
unrelated per-instance metadata query (id/checksum/added/etc, no
content) run to build the "versions" response field, which isn't part
of what this test file covers.
"""
return [
q
for q in ctx.captured_queries
if q["sql"].startswith('SELECT "documents_document"."content" FROM')
]
@pytest.mark.django_db
class TestTrashAndGlobalSearchEffectiveContentIsNeverPerInstance:
"""
TrashView and GlobalSearchView serialize Document instances with
DocumentSerializer too, but build their querysets independently of
DocumentViewSet.get_queryset(). TrashView doesn't display content at all,
so it keeps the document's own unresolved content; GlobalSearchView
annotates effective_content itself, so it shows the latest version's.
Neither should ever fall back to a per-instance query.
"""
def test_trash_list_shows_unresolved_content_with_no_extra_query(
self,
admin_client: APIClient,
) -> None:
"""
GIVEN:
- A trashed root document whose own content differs from what a
version would have had (also trashed, deletion cascades)
WHEN:
- Listing trash
THEN:
- The response shows the document's own content
- Nothing ever queries for versions to resolve it
"""
root = DocumentFactory(content="own-content")
DocumentFactory(
root_document=root,
version_index=1,
content="version-content",
)
root.delete()
with CaptureQueriesContext(connection) as ctx:
response = admin_client.get("/api/trash/")
assert response.status_code == status.HTTP_200_OK
[result] = [r for r in response.data["results"] if r["id"] == root.id]
assert result["content"] == "own-content"
assert _get_effective_content_fallback_queries(ctx) == []
@@ -4,13 +4,16 @@ from datetime import date
from pathlib import Path
from unittest import mock
import pytest
from django.test import TestCase
from django.test import override_settings
from faker import Faker
from pytest_mock import MockerFixture
from documents.models import Correspondent
from documents.models import Document
from documents.tasks import empty_trash
from paperless_testing.factories import DocumentFactory
class TestDocument(TestCase):
@@ -257,3 +260,72 @@ def test_suggestion_content() -> None:
)
doc.content = content_over_limit
assert doc.suggestion_content == expected_cropped_content
class TestPublicFilenameLength:
@pytest.fixture(autouse=True)
def _no_parser_registry(self, mocker: MockerFixture) -> None:
# The parser registry reads ApplicationConfiguration from the database
extensions = {"application/pdf": ".pdf", "image/jpeg": ".jpg"}
mocker.patch(
"documents.models.get_default_file_extension",
side_effect=extensions.__getitem__,
)
LIMIT = Document.MAX_PUBLIC_FILENAME_BYTES
@pytest.mark.parametrize(
("title", "archive", "counter", "max_bytes", "tail"),
[
pytest.param("a" * 300, False, 0, LIMIT, ".jpg", id="ascii"),
pytest.param("a" * 300, True, 0, LIMIT, ".pdf", id="archive"),
pytest.param("ä" * 200, False, 0, LIMIT, ".jpg", id="umlaut"),
pytest.param("ä" * 200, False, 1, LIMIT, "_01.jpg", id="count"),
pytest.param("a" * 300, False, 1, 100, "_01.jpg", id="max"),
],
)
def test_long_title_keeps_tail_within_limit(
self,
title: str,
*,
archive: bool,
counter: int,
max_bytes: int,
tail: str,
) -> None:
"""
GIVEN:
- A document whose title alone is longer than the byte limit
WHEN:
- The public file name is generated
THEN:
- The name ends with the counter and extension
- The name fits in the limit
"""
name = DocumentFactory.build(
title=title,
mime_type="image/jpeg",
).get_public_filename(
archive=archive,
counter=counter,
max_bytes=max_bytes,
)
assert name.endswith(tail)
assert len(name.encode()) <= max_bytes
def test_long_title_counters_give_distinct_names(self) -> None:
"""
GIVEN:
- A document whose title alone is longer than the byte limit
WHEN:
- Public file names are generated with increasing counters
THEN:
- Each counter produces a distinct name
"""
doc = DocumentFactory.build(title="ä" * 200, mime_type="application/pdf")
names = [doc.get_public_filename(counter=counter) for counter in range(3)]
assert len(set(names)) == 3
assert all(len(n.encode()) <= self.LIMIT for n in names)
+49 -39
View File
@@ -2,8 +2,8 @@ import datetime as dt
import os
import shutil
from pathlib import Path
from unittest import mock
import pytest
from django.test import TestCase
from django.test import override_settings
from pdfminer.high_level import extract_text
@@ -15,22 +15,26 @@ from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.double_sided import STAGING_FILE_NAME
from documents.double_sided import TIMEOUT_MINUTES
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
@pytest.mark.usefixtures("fake_progress_manager")
@override_settings(
CONSUMER_RECURSIVE=True,
CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=True,
)
class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
SAMPLE_DIR = Path(__file__).parent / "samples"
class TestDoubleSided(
DirectoriesMixin,
FileSystemAssertsMixin,
SampleDirMixin,
TestCase,
):
def setUp(self) -> None:
super().setUp()
self.dirs.double_sided_dir = self.dirs.consumption_dir / "double-sided"
self.dirs.double_sided_dir.mkdir()
self.double_sided_dir = self.dirs.consumption_dir / "double-sided"
self.double_sided_dir.mkdir()
self.staging_file = self.dirs.scratch_dir / STAGING_FILE_NAME
def consume_file(self, srcname, dstname: str | Path = "foo.pdf"):
@@ -39,20 +43,16 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
destination file does not exist afterwards
"""
src = self.SAMPLE_DIR / srcname
dst = self.dirs.double_sided_dir / dstname
dst = self.double_sided_dir / dstname
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(src, dst)
with mock.patch(
"documents.tasks.ProgressManager",
DummyProgressManager,
):
msg = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
msg = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
self.assertIsNotFile(dst)
return msg
@@ -214,31 +214,41 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertIsNotFile(self.staging_file)
self.assertIsInstance(msg.get("document_id"), int)
def test_subdirectory_upload(self) -> None:
def test_consume_double_sided_in_nested_dir(self) -> None:
"""
GIVEN:
- A staging file exists
WHEN:
- A file gets uploaded into foo/bar/double-sided
or double-sided/foo/bar
- A file is uploaded into foo/bar/double-sided
THEN:
- The collated file gets put into foo/bar
- The collated file is placed into foo/bar
"""
# TODO: parameterize this instead
for path in [
Path("foo") / "bar" / "double-sided",
Path("double-sided") / "foo" / "bar",
]:
with self.subTest(path=str(path)):
# Ensure we get fresh directories for each run
self.tearDown()
self.setUp()
self.create_staging_file()
self.consume_file(
"double-sided-odd.pdf",
Path("foo") / "bar" / "double-sided" / "foo.pdf",
)
self.assertIsFile(
self.dirs.consumption_dir / "foo" / "bar" / "foo-collated.pdf",
)
self.create_staging_file()
self.consume_file("double-sided-odd.pdf", Path(path) / "foo.pdf")
self.assertIsFile(
self.dirs.consumption_dir / "foo" / "bar" / "foo-collated.pdf",
)
def test_consume_double_sided_with_nested_subdir(self) -> None:
"""
GIVEN:
- A staging file exists
WHEN:
- A file is uploaded into double-sided/foo/bar
THEN:
- The collated file is placed into foo/bar
"""
self.create_staging_file()
self.consume_file(
"double-sided-odd.pdf",
Path("double-sided") / "foo" / "bar" / "foo.pdf",
)
self.assertIsFile(
self.dirs.consumption_dir / "foo" / "bar" / "foo-collated.pdf",
)
@override_settings(CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=False)
def test_disabled_double_sided_dir_upload(self) -> None:
+5 -5
View File
@@ -8,7 +8,6 @@ from unittest import mock
import pytest
from auditlog.context import disable_auditlog
from django.conf import settings
from django.contrib.auth.models import User
from django.db import DatabaseError
from django.db import connection
from django.test import TestCase
@@ -30,9 +29,10 @@ from documents.models import DocumentType
from documents.models import StoragePath
from documents.serialisers import DocumentSerializer
from documents.tasks import empty_trash
from documents.tests.factories import DocumentFactory
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
@@ -1323,7 +1323,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
- Document without owner returns "none"
"""
u1 = User.objects.create_user("user1")
u1 = UserFactory(username="user1")
owned_doc = Document.objects.create(
title="The Title",
+4 -4
View File
@@ -13,10 +13,10 @@ import pytest
from documents.file_handling import generate_filename
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.tests.factories import CorrespondentFactory
from documents.tests.factories import DocumentFactory
from documents.tests.factories import StoragePathFactory
from documents.tests.factories import TagFactory
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import StoragePathFactory
from paperless_testing.factories import TagFactory
@pytest.mark.django_db
+24 -2
View File
@@ -20,8 +20,9 @@ if TYPE_CHECKING:
from documents.file_handling import generate_filename
from documents.models import Document
from documents.tasks import update_document_content_maybe_archive_file
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
@@ -116,6 +117,27 @@ class TestMakeIndex:
call_command("document_index", "reindex", skip_checks=True)
mock_get_backend.return_value.rebuild.assert_called_once()
def test_reindex_skips_versions(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- A root document with a version
WHEN:
- The reindex command runs
THEN:
- Only the root document is handed to the rebuild, since a version
is indexed as its root
"""
root = DocumentFactory()
DocumentFactory(root_document=root, version_index=1)
mock_get_backend = mocker.patch(
"documents.management.commands.document_index.get_backend",
)
call_command("document_index", "reindex", skip_checks=True)
documents = mock_get_backend.return_value.rebuild.call_args.args[0]
assert list(documents.values_list("pk", flat=True)) == [root.pk]
def test_optimize(self) -> None:
"""Optimize command must execute without error (Tantivy handles optimization automatically)."""
call_command("document_index", "optimize", skip_checks=True)
+226 -10
View File
@@ -2,6 +2,7 @@ import hashlib
import json
import shutil
import tempfile
from datetime import date
from datetime import timedelta
from io import StringIO
from pathlib import Path
@@ -26,13 +27,13 @@ from django.test import override_settings
from django.utils import timezone
from guardian.models import GroupObjectPermission
from guardian.models import UserObjectPermission
from guardian.shortcuts import assign_perm
from documents.management.commands import document_exporter
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentBarcode
from documents.models import DocumentType
from documents.models import Note
from documents.models import ShareLink
@@ -44,13 +45,20 @@ from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.sanity_checker import check_sanity
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from documents.settings import EXPORTER_THUMBNAIL_NAME
from documents.tests.utils import SampleDirMixin
from documents.tests.utils import paperless_environment
from paperless.models import ApplicationConfiguration
from paperless_mail.models import MailAccount
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.dirs import PaperlessDirs
from paperless_testing.dirs import paperless_environment
from paperless_testing.factories import DocumentBarcodeFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object
@pytest.mark.management
@@ -105,8 +113,8 @@ class TestExportImport(
user=self.user,
)
assign_perm("view_document", self.user2, self.d2)
assign_perm("view_document", self.group1, self.d3)
grant_object(self.user2, self.d2, "view_document")
grant_object(self.group1, self.d3, "view_document")
self.t1 = Tag.objects.create(name="t")
self.dt1 = DocumentType.objects.create(name="dt")
@@ -677,12 +685,13 @@ class TestExportImport(
THEN:
- Error is raised
"""
args = ["document_exporter", "/tmp/foo/bar"]
with tempfile.TemporaryDirectory() as tmp_dir:
args = ["document_exporter", str(Path(tmp_dir) / "does-not-exist")]
with self.assertRaises(CommandError) as e:
call_command(*args, skip_checks=True)
with self.assertRaises(CommandError) as e:
call_command(*args, skip_checks=True)
self.assertEqual("That path doesn't exist", str(e.exception))
self.assertEqual("That path doesn't exist", str(e.exception))
def test_export_target_exists_but_is_file(self) -> None:
"""
@@ -855,6 +864,37 @@ class TestExportImport(
self.assertEqual(Document.objects.count(), 4)
self.assertEqual(CustomFieldInstance.objects.count(), 1)
def _export_import_barcodes(self, *, split_manifest: bool) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
DocumentBarcodeFactory(document=self.d1, value="https://example.com")
DocumentBarcodeFactory(document=self.d2, page=2, value="DE8937")
self._do_export(split_manifest=split_manifest)
with paperless_environment():
Document.objects.all().delete()
self.assertEqual(DocumentBarcode.objects.count(), 0)
call_command(
"document_importer",
"--no-progress-bar",
self.target,
skip_checks=True,
)
self.assertEqual(
set(DocumentBarcode.objects.values_list("document", "page", "value")),
{(self.d1.pk, 1, "https://example.com"), (self.d2.pk, 2, "DE8937")},
)
def test_export_import_barcodes(self) -> None:
self._export_import_barcodes(split_manifest=False)
def test_export_import_barcodes_split_manifest(self) -> None:
self._export_import_barcodes(split_manifest=True)
def test_folder_prefix(self) -> None:
"""
GIVEN:
@@ -1363,6 +1403,45 @@ class TestCryptExportImport(
social_token = SocialToken.objects.first()
self.assertIsNotNone(social_token)
def test_import_passphrase_with_float_field(self) -> None:
"""
GIVEN:
- An application configuration with a float field set
WHEN:
- Export and import are run with a passphrase
THEN:
- The import succeeds and the float value is preserved
"""
ApplicationConfiguration.objects.update_or_create(
pk=1,
defaults={"rotate_pages_threshold": 12.5},
)
call_command(
"document_exporter",
"--no-progress-bar",
"--passphrase",
"securepassword",
self.target,
skip_checks=True,
)
ApplicationConfiguration.objects.all().delete()
call_command(
"document_importer",
"--no-progress-bar",
"--passphrase",
"securepassword",
self.target,
skip_checks=True,
)
self.assertEqual(
ApplicationConfiguration.objects.get().rotate_pages_threshold,
12.5,
)
def test_import_crypt_no_passphrase(self) -> None:
"""
GIVEN:
@@ -1428,3 +1507,140 @@ class TestCryptExportImport(
("No passphrase was given, sensitive fields will be in plaintext"),
stdout.read(),
)
@pytest.mark.management
@pytest.mark.django_db
class TestExportDerivedFileNameCollisions:
def test_same_stem_different_extension_get_distinct_derived_files(
self,
paperless_dirs: PaperlessDirs,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Two documents with the same title and created date
- One original is a PDF, the other a PNG, both with archive and thumbnail
WHEN:
- The exporter runs
THEN:
- Each document is exported under a distinct original, thumbnail and archive name
- Each exported file holds its own document's content
"""
for pk, (filename, mime_type) in enumerate(
[("collide.pdf", "application/pdf"), ("collide.png", "image/png")],
start=1,
):
(paperless_dirs.originals_dir / filename).write_bytes(
f"original {pk}".encode(),
)
(paperless_dirs.archive_dir / f"{pk:07}.pdf").write_bytes(
f"archive {pk}".encode(),
)
(paperless_dirs.thumbnail_dir / f"{pk:07}.webp").write_bytes(
f"thumbnail {pk}".encode(),
)
DocumentFactory(
pk=pk,
title="collide",
created=date(2026, 10, 7),
filename=filename,
archive_filename=f"{pk:07}.pdf",
mime_type=mime_type,
)
target = tmp_path / "export"
target.mkdir()
call_command(
"document_exporter",
str(target),
"--no-progress-bar",
skip_checks=True,
)
manifest = json.loads((target / "manifest.json").read_text())
records = {r["pk"]: r for r in manifest if r["model"] == "documents.document"}
for key in (
EXPORTER_FILE_NAME,
EXPORTER_THUMBNAIL_NAME,
EXPORTER_ARCHIVE_NAME,
):
assert records[1][key] != records[2][key]
for pk in (1, 2):
record = records[pk]
assert (target / record[EXPORTER_FILE_NAME]).read_text() == f"original {pk}"
assert (
target / record[EXPORTER_THUMBNAIL_NAME]
).read_text() == f"thumbnail {pk}"
assert (
target / record[EXPORTER_ARCHIVE_NAME]
).read_text() == f"archive {pk}"
def test_long_titles_export_within_file_name_limit(
self,
paperless_dirs: PaperlessDirs,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Two documents with the same created date and the same title
- The title alone is longer than 255 bytes and contains umlauts
WHEN:
- The exporter runs
THEN:
- The export finishes with distinct original, thumbnail and archive names
- Every exported file name fits in the byte limit
- Each exported file holds its own document's content
"""
for pk in (1, 2):
(paperless_dirs.originals_dir / f"{pk:07}.pdf").write_bytes(
f"original {pk}".encode(),
)
(paperless_dirs.archive_dir / f"{pk:07}.pdf").write_bytes(
f"archive {pk}".encode(),
)
(paperless_dirs.thumbnail_dir / f"{pk:07}.webp").write_bytes(
f"thumbnail {pk}".encode(),
)
DocumentFactory(
pk=pk,
title="Bestellbestätigung " * 15,
created=date(2026, 10, 15),
filename=f"{pk:07}.pdf",
archive_filename=f"{pk:07}.pdf",
mime_type="application/pdf",
)
target = tmp_path / "export"
target.mkdir()
call_command(
"document_exporter",
str(target),
"--no-progress-bar",
skip_checks=True,
)
manifest = json.loads((target / "manifest.json").read_text())
records = {r["pk"]: r for r in manifest if r["model"] == "documents.document"}
for key in (
EXPORTER_FILE_NAME,
EXPORTER_THUMBNAIL_NAME,
EXPORTER_ARCHIVE_NAME,
):
assert records[1][key] != records[2][key]
for pk in (1, 2):
record = records[pk]
for key, content in (
(EXPORTER_FILE_NAME, f"original {pk}"),
(EXPORTER_THUMBNAIL_NAME, f"thumbnail {pk}"),
(EXPORTER_ARCHIVE_NAME, f"archive {pk}"),
):
assert (
len(Path(record[key]).name.encode())
<= Document.MAX_PUBLIC_FILENAME_BYTES
)
assert (target / record[key]).read_text() == content
+9 -9
View File
@@ -8,7 +8,7 @@ from django.core.management import call_command
from django.test import TestCase
from documents.models import Document
from documents.tests.factories import DocumentFactory
from paperless_testing.factories import DocumentFactory
@pytest.mark.management
@@ -123,14 +123,14 @@ class TestFuzzyMatchCommand(TestCase):
- Output contains clickable links to the documents instead of titles
"""
# Content similarity is 86.667
Document.objects.create(
doc1 = Document.objects.create(
checksum="BEEFCAFE",
title="A",
content="first document scanned by bob",
mime_type="application/pdf",
filename="test.pdf",
)
Document.objects.create(
doc2 = Document.objects.create(
checksum="DEADBEAF",
title="A",
content="first document scanned by alice",
@@ -145,8 +145,8 @@ class TestFuzzyMatchCommand(TestCase):
"http://localhost:8000",
)
self.assertIn("Found 1 matching pair(s)", stdout)
self.assertIn("http://localhost:8000/documents/1/details", stdout)
self.assertIn("http://localhost:8000/documents/2/details", stdout)
self.assertIn(f"http://localhost:8000/documents/{doc1.pk}/details", stdout)
self.assertIn(f"http://localhost:8000/documents/{doc2.pk}/details", stdout)
def test_with_3_matches(self) -> None:
"""
@@ -198,14 +198,14 @@ class TestFuzzyMatchCommand(TestCase):
- Documents 1 and 2 remain
"""
# Content similarity is 86.667
Document.objects.create(
doc1 = Document.objects.create(
checksum="BEEFCAFE",
title="A",
content="first document scanned by bob",
mime_type="application/pdf",
filename="test.pdf",
)
Document.objects.create(
doc2 = Document.objects.create(
checksum="DEADBEAF",
title="A",
content="second document scanned by alice",
@@ -235,8 +235,8 @@ class TestFuzzyMatchCommand(TestCase):
self.assertIn("Deleting 1 document(s)", stdout)
self.assertEqual(Document.objects.count(), 2)
self.assertIsNotNone(Document.objects.get(pk=1))
self.assertIsNotNone(Document.objects.get(pk=2))
self.assertIsNotNone(Document.objects.get(pk=doc1.pk))
self.assertIsNotNone(Document.objects.get(pk=doc2.pk))
def test_document_deletion_cancelled(self) -> None:
"""
@@ -15,9 +15,9 @@ from documents.management.commands.document_importer import _deserialize_record
from documents.models import Document
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
@pytest.mark.management
@@ -14,12 +14,12 @@ from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.factories import CorrespondentFactory
from documents.tests.factories import DocumentFactory
from documents.tests.factories import DocumentTypeFactory
from documents.tests.factories import StoragePathFactory
from documents.tests.factories import TagFactory
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import DocumentTypeFactory
from paperless_testing.factories import StoragePathFactory
from paperless_testing.factories import TagFactory
# ---------------------------------------------------------------------------
# Module-level type aliases
@@ -7,14 +7,14 @@ from django.contrib.auth.models import User
from django.core.management import call_command
from django.test import TestCase
from documents.tests.utils import DirectoriesMixin
from paperless_testing.dirs import DirectoriesMixin
@pytest.mark.management
class TestManageSuperUser(DirectoriesMixin, TestCase):
def call_command(self, environ):
out = StringIO()
with mock.patch.dict(os.environ, environ):
with mock.patch.dict(os.environ, environ, clear=True):
call_command(
"manage_superuser",
"--no-color",
@@ -9,8 +9,8 @@ from django.test import TestCase
from documents.management.commands.document_thumbnails import _process_document
from documents.models import Document
from documents.parsers import get_default_thumbnail
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
@pytest.mark.management
+4 -4
View File
@@ -7,10 +7,10 @@ from documents import matching
from documents.models import Document
from documents.models import MatchingModel
from documents.signals import document_consumption_finished
from documents.tests.factories import CorrespondentFactory
from documents.tests.factories import DocumentFactory
from documents.tests.factories import DocumentTypeFactory
from documents.tests.factories import TagFactory
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import DocumentTypeFactory
from paperless_testing.factories import TagFactory
@pytest.fixture(
@@ -2,8 +2,6 @@ import json
from unittest import mock
from auditlog.models import LogEntry
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from rest_framework import status
@@ -12,6 +10,8 @@ from rest_framework.test import APITestCase
from documents.bulk_edit import merge_as_versions
from documents.models import Document
from documents.serialisers import MergeDocumentsAsVersionsSerializer
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
class TestMergeDocumentsAsVersionsSerializer(TestCase):
@@ -294,7 +294,7 @@ class TestMergeDocumentsAsVersions(TestCase):
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.search.get_backend")
def test_writes_audit_log_entry(self, *_mocks) -> None:
user = User.objects.create_user(username="merger")
user = UserFactory(username="merger")
root = Document.objects.create(checksum="A", title="Root")
source = Document.objects.create(checksum="B", title="Source")
LogEntry.objects.all().delete()
@@ -335,12 +335,8 @@ class TestMergeDocumentsAsVersions(TestCase):
class TestMergeDocumentsAsVersionsAPI(APITestCase):
def setUp(self) -> None:
self.user = User.objects.create_user(username="user")
self.user.user_permissions.add(
Permission.objects.get(codename="change_document"),
Permission.objects.get(codename="view_document"),
Permission.objects.get(codename="delete_document"),
)
self.user = UserFactory(username="user")
grant_global(self.user, "change_document", "view_document", "delete_document")
self.doc1 = Document.objects.create(
checksum="A",
title="A",
@@ -382,7 +378,7 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
@mock.patch("documents.views.bulk_edit.merge_as_versions")
def test_requires_change_permission(self, merge_mock) -> None:
merge_mock.__name__ = "merge_as_versions"
user = User.objects.create_user(username="no-change")
user = UserFactory(username="no-change")
self.doc1.owner = user
self.doc1.save()
self.doc2.owner = user
@@ -405,11 +401,8 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
def test_requires_delete_permission(self, merge_mock) -> None:
merge_mock.__name__ = "merge_as_versions"
# Owns them and may change them, but may not make them stop being documents
user = User.objects.create_user(username="no-delete")
user.user_permissions.add(
Permission.objects.get(codename="change_document"),
Permission.objects.get(codename="view_document"),
)
user = UserFactory(username="no-delete")
grant_global(user, "change_document", "view_document")
for doc in (self.doc1, self.doc2):
doc.owner = user
doc.save()
@@ -1,4 +1,4 @@
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
SAVED_VIEWS_KEY = "saved_views"
DASHBOARD_VIEWS_VISIBLE_IDS_KEY = "dashboard_views_visible_ids"
@@ -7,7 +7,7 @@ from django.conf import settings
from django.db import connection
from django.test import override_settings
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
def _sha256(data: bytes) -> str:
@@ -1,4 +1,4 @@
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
class TestMigrateShareLinkBundlePermissions(TestMigrations):
+2 -2
View File
@@ -2,8 +2,8 @@ import pytest
from documents.models import Correspondent
from documents.models import Document
from documents.tests.factories import CorrespondentFactory
from documents.tests.factories import DocumentFactory
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory
@pytest.mark.django_db
File diff suppressed because it is too large. Load diff
@@ -1,11 +1,11 @@
import pytest
from django.contrib.auth.models import User
from guardian.shortcuts import assign_perm
from rest_framework.test import APIRequestFactory
from documents.filters import PermittedObjectsFilter
from documents.models import Tag
from documents.tests.factories import TagFactory
from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_object
class _DummyView:
@@ -15,8 +15,8 @@ class _DummyView:
@pytest.mark.django_db
class TestPermittedObjectsFilter:
def test_superuser_bypasses_filtering_entirely(self):
superuser = User.objects.create_superuser(username="root")
owner = User.objects.create_user(username="owner")
superuser = UserFactory(username="root", superuser=True)
owner = UserFactory(username="owner")
TagFactory(owner=owner)
request = APIRequestFactory().get("/")
request.user = superuser
@@ -29,13 +29,13 @@ class TestPermittedObjectsFilter:
assert result.count() == Tag.objects.count()
def test_non_superuser_sees_only_owned_unowned_and_granted(self):
owner = User.objects.create_user(username="owner")
grantee = User.objects.create_user(username="grantee")
owner = UserFactory(username="owner")
grantee = UserFactory(username="grantee")
owned = TagFactory(owner=grantee)
unowned = TagFactory(owner=None)
granted = TagFactory(owner=owner)
hidden = TagFactory(owner=owner)
assign_perm("view_tag", grantee, granted)
grant_object(grantee, granted, "view_tag")
request = APIRequestFactory().get("/")
request.user = grantee
@@ -49,11 +49,11 @@ class TestPermittedObjectsFilter:
assert hidden.pk not in visible_ids
def test_include_granted_false_excludes_explicitly_shared_objects(self):
owner = User.objects.create_user(username="owner2")
grantee = User.objects.create_user(username="grantee2")
owner = UserFactory(username="owner2")
grantee = UserFactory(username="grantee2")
owned = TagFactory(owner=grantee)
granted = TagFactory(owner=owner)
assign_perm("view_tag", grantee, granted)
grant_object(grantee, granted, "view_tag")
request = APIRequestFactory().get("/")
request.user = grantee
@@ -74,15 +74,15 @@ class TestPermittedObjectsFilter:
[("inactive", False), ("inactive_super", True)],
)
def test_inactive_user_sees_nothing(self, username: str, *, is_superuser: bool):
user = User.objects.create_user(
user = UserFactory(
username=username,
is_active=False,
is_superuser=is_superuser,
)
TagFactory(owner=None)
TagFactory(owner=user)
granted = TagFactory(owner=User.objects.create_user(username=f"o_{username}"))
assign_perm("view_tag", user, granted)
granted = TagFactory(owner=UserFactory(username=f"o_{username}"))
grant_object(user, granted, "view_tag")
request = APIRequestFactory().get("/")
request.user = user
@@ -94,7 +94,7 @@ class TestPermittedObjectsFilter:
assert result.count() == 0
def test_inactive_user_sees_nothing_with_include_granted_false(self):
user = User.objects.create_user(username="inactive_owner", is_active=False)
user = UserFactory(username="inactive_owner", is_active=False)
TagFactory(owner=user)
TagFactory(owner=None)
request = APIRequestFactory().get("/")
+7
View File
@@ -1,5 +1,6 @@
import pytest
import regex
from django.conf import settings
from pytest_mock import MockerFixture
from documents.regex import safe_regex_finditer
@@ -9,6 +10,12 @@ from documents.regex import safe_regex_sub
from documents.regex import validate_regex_pattern
def test_regex_timeout_uses_configured_setting() -> None:
from documents.regex import REGEX_TIMEOUT_SECONDS
assert REGEX_TIMEOUT_SECONDS == settings.MATCH_REGEX_TIMEOUT_SECONDS
class TestValidateRegexPattern:
def test_valid_pattern(self) -> None:
validate_regex_pattern(r"\d+")
Loaded 100 of 516 files, more files were not shown because too many files have changed in this diff. Show more