diff --git a/docs/usage.md b/docs/usage.md index 9a6d7fd5f..8bc8f270c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -886,6 +886,19 @@ Matching documents with logical expressions: ``` shopname AND (product1 OR product2) +invoice NOT draft +``` + +`AND`, `OR` and `NOT` must be written in capitals, and parentheses group sub-expressions. Terms written next to each other with no operator between them are combined with `AND`. + +!!! warning + + A leading `-` does **not** exclude a term. Separators are stripped during indexing, so `invoice -secret` searches for `invoice` and `secret`, which is the opposite of what you probably intended. Use `NOT` to exclude a term: `invoice NOT secret`. + +Matching an exact phrase, in order, by quoting it: + +``` +"quick brown fox" ``` Matching specific tags, correspondents or types: @@ -893,8 +906,12 @@ Matching specific tags, correspondents or types: ``` type:invoice tag:unpaid correspondent:university certificate +tag:bills,unpaid ``` +- `document_type` may be abbreviated to `type`, and `storage_path` to `path`. +- A comma-separated list after `tag:` requires **all** of the listed tags, so `tag:bills,unpaid` matches only documents tagged both `bills` and `unpaid`. + Matching dates: ``` @@ -908,16 +925,18 @@ Matching by archive metadata: ``` asn:100 page_count:12 -checksum:a1b2c3d4 +num_notes:0 +checksum:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 original_filename:invoice.pdf ``` - `asn` matches a document's Archive Serial Number. - `page_count` matches a document's page count. -- `checksum` matches the checksum of the original document file (not the - archived/processed version). -- `original_filename` matches the filename of the document as originally - consumed. +- `num_notes` matches how many notes a document has. +- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Because the field is not stemmed, that prefix is matched literally. +- `original_filename` matches the filename of the document as originally consumed. + +`asn`, `page_count` and `num_notes` are numeric and also accept ranges, for example `asn:[50 to 150]`. Matching inexact words: @@ -954,6 +973,27 @@ Supported date keywords: `today`, `yesterday`, `previous week`, `this month`, `previous month`, `this year`, `previous year`, `previous quarter`. +These other date forms also work after a date field: + +``` +added:tomorrow +created:2005-03-04 +added:january +modified:"next monday" +added:"last monday" +created:[2005-01-01 to 2005-01-31] +``` + +- `tomorrow`, like `today` and `yesterday`, covers that whole day. +- An ISO date such as `2005-03-04` covers that whole day, and `2005-01` covers that whole month. +- A month name such as `january` covers that whole month in the current year. +- `next ` and `last ` each cover that whole day and must be quoted. A bare weekday name such as `monday` is not accepted. +- A range takes two of the above as its bounds, for example `created:[2005 to 2009]` or `added:[2005-01-01 to 2005-01-31]`. + +!!! warning + + `now`, `noon`, `midnight` and relative offsets such as `"-3 days"` or `"-1 week"` are accepted by the parser but resolve to a single instant rather than to a span of time, so they match only a document whose timestamp is exactly that instant, which in practice means no documents at all. Spellings like `now-3days` and `"3 days ago"` are rejected outright. A timestamp carrying a time of day, such as `2005-01-01T00:00:00Z`, is not understood either: the time portion is split off and searched as ordinary text, which usually leaves the query matching nothing. To bound a search by time, use a range with whole-day bounds instead. + #### Searching custom fields Custom field names and values are included in the full-text index, but they @@ -999,9 +1039,7 @@ notes.user:alice notes.note:insurance The bare `notes:` prefix is shorthand for `notes.note:`. -All of these constructs can be combined as you see fit. If you want to -learn more about the query language used by paperless, see the -[Tantivy query language documentation](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html). +All of these constructs can be combined as you see fit. What is described above is the whole of the query language paperless supports. It resembles other search query languages without being identical to any of them, so a construct that is not documented here is most likely treated as ordinary search text rather than as syntax, and an unrecognized field name is searched as text too. !!! note diff --git a/src/documents/tests/search/test_documented_syntax.py b/src/documents/tests/search/test_documented_syntax.py new file mode 100644 index 000000000..0e5340a8e --- /dev/null +++ b/src/documents/tests/search/test_documented_syntax.py @@ -0,0 +1,309 @@ +"""Pins the search syntax that ``docs/usage.md`` promises users. + +Every query here appears verbatim, or as a direct paraphrase, in the +"Document searches" section of ``docs/usage.md``. Each case indexes real +documents and asserts on matched document IDs rather than on the parsed +query, because a query that parses cleanly is not necessarily a query that +means what the documentation says it means: ``added:now - 3 days`` parses +without a single diagnostic and then matches nothing. + +The negative cases matter as much as the positive ones. They pin the +behaviours the docs explicitly warn about, so that if any of them ever +starts working the warning can be removed deliberately rather than being +left standing as a lie. +""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from typing import TYPE_CHECKING + +import pytest +import time_machine + +from documents.models import Document +from documents.models import DocumentType +from documents.models import Note +from documents.models import StoragePath +from documents.models import Tag + +if TYPE_CHECKING: + from collections.abc import Generator + + from django.contrib.auth.models import User + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +# A Monday, so that "next monday"/"last monday" land a clean week either side. +FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) + +# The checksum used in the docs' `checksum:` example. +DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestLogicalExpressions: + @pytest.fixture + def docs(self, backend: TantivyBackend) -> dict[str, int]: + return { + "secret": _index( + backend, + title="Invoice one", + content="invoice secret contents", + checksum="doc-syntax-secret", + ).pk, + "plain": _index( + backend, + title="Invoice two", + content="invoice ordinary contents", + checksum="doc-syntax-plain", + ).pk, + } + + def test_not_excludes_a_term( + self, + backend: TantivyBackend, + docs: dict[str, int], + ) -> None: + assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]} + + def test_leading_hyphen_requires_the_term_instead_of_excluding_it( + self, + backend: TantivyBackend, + docs: dict[str, int], + ) -> None: + # The docs warn about exactly this: separators are stripped at index + # time, so "-secret" is the term "secret" and the query is an AND. + assert _matched_ids(backend, "invoice -secret") == {docs["secret"]} + + def test_or_inside_parentheses_matches_either_branch( + self, + backend: TantivyBackend, + docs: dict[str, int], + ) -> None: + matched = _matched_ids(backend, "invoice AND (secret OR ordinary)") + assert matched == {docs["secret"], docs["plain"]} + + +class TestPhraseSearch: + def test_quoted_phrase_requires_the_words_in_order( + self, + backend: TantivyBackend, + ) -> None: + doc = _index( + backend, + title="Phrase", + content="the quick brown fox jumps", + checksum="doc-syntax-phrase", + ) + assert _matched_ids(backend, '"quick brown fox"') == {doc.pk} + assert _matched_ids(backend, '"brown quick fox"') == set() + + +class TestFieldAliases: + def test_type_is_an_alias_for_document_type( + self, + backend: TantivyBackend, + ) -> None: + doc_type = DocumentType.objects.create(name="invoice") + doc = _index( + backend, + title="Typed", + content="body", + checksum="doc-syntax-typed", + document_type=doc_type, + ) + assert _matched_ids(backend, "type:invoice") == {doc.pk} + assert _matched_ids(backend, "document_type:invoice") == {doc.pk} + + def test_path_is_an_alias_for_storage_path( + self, + backend: TantivyBackend, + ) -> None: + storage_path = StoragePath.objects.create( + name="archive", + path="archive/{{ title }}", + ) + doc = _index( + backend, + title="Pathed", + content="body", + checksum="doc-syntax-pathed", + storage_path=storage_path, + ) + assert _matched_ids(backend, "path:archive") == {doc.pk} + assert _matched_ids(backend, "storage_path:archive") == {doc.pk} + + +class TestTagCommaList: + def test_comma_list_requires_every_listed_tag( + self, + backend: TantivyBackend, + ) -> None: + bills = Tag.objects.create(name="bills") + unpaid = Tag.objects.create(name="unpaid") + archived = Tag.objects.create(name="archived") + + both = Document.objects.create( + title="Both tags", + content="body", + checksum="doc-syntax-tag-both", + ) + both.tags.add(bills, unpaid) + backend.add_or_update(both) + + one = Document.objects.create( + title="One tag", + content="body", + checksum="doc-syntax-tag-one", + ) + one.tags.add(bills, archived) + backend.add_or_update(one) + + assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk} + assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk} + + +class TestArchiveMetadataFields: + @pytest.fixture + def doc(self, backend: TantivyBackend, admin_user: User) -> Document: + doc = Document.objects.create( + title="Metadata", + content="body", + checksum=DOC_CHECKSUM, + archive_serial_number=100, + page_count=12, + original_filename="invoice.pdf", + ) + Note.objects.create(document=doc, user=admin_user, note="a note") + backend.add_or_update(doc) + return doc + + @pytest.mark.parametrize( + "query", + [ + "asn:100", + "asn:[50 to 150]", + "page_count:12", + "page_count:[10 to 20]", + "num_notes:1", + "num_notes:[1 to 5]", + "original_filename:invoice.pdf", + f"checksum:{DOC_CHECKSUM}", + "checksum:9f86d081*", + ], + ) + def test_documented_metadata_query_matches( + self, + backend: TantivyBackend, + doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == {doc.pk} + + @pytest.mark.parametrize( + "query", + [ + # The docs say only a complete, lowercase checksum matches. + "checksum:9f86d081", + f"checksum:{DOC_CHECKSUM.upper()}", + ], + ) + def test_partial_or_uppercase_checksum_matches_nothing( + self, + backend: TantivyBackend, + doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == set() + + +class TestDocumentedDateForms: + @pytest.fixture(autouse=True) + def frozen_now(self) -> Generator[None, None, None]: + with time_machine.travel(FROZEN_NOW, tick=False): + yield + + @pytest.fixture + def dated(self, backend: TantivyBackend) -> dict[str, int]: + stamps = { + "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC), + "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC), + "tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC), + "next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC), + "last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC), + "january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC), + "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC), + } + return { + label: _index( + backend, + title=label, + content="dated body", + checksum=f"doc-syntax-date-{label}", + added=stamp, + ).pk + for label, stamp in stamps.items() + } + + @pytest.mark.parametrize( + ("query", "label"), + [ + ("added:today", "today"), + ("added:yesterday", "yesterday"), + ("added:tomorrow", "tomorrow"), + ('added:"next monday"', "next_monday"), + ('added:"last monday"', "last_monday"), + ("added:january", "january"), + ("added:2005-03-04", "old"), + ("added:2005-03", "old"), + ("added:[2005-01-01 to 2005-12-31]", "old"), + ("added:[2005 to 2009]", "old"), + ], + ) + def test_documented_date_form_matches_its_day_or_month( + self, + backend: TantivyBackend, + dated: dict[str, int], + query: str, + label: str, + ) -> None: + assert _matched_ids(backend, query) == {dated[label]} + + @pytest.mark.parametrize( + "query", + [ + # Zero-width: these resolve to a single instant, not a span, so + # nothing in a realistic corpus lands on them. The docs warn + # about them rather than presenting them as usable. + "added:now", + "added:noon", + "added:midnight", + 'added:"-3 days"', + 'added:"-1 week"', + # The trap: this reports no diagnostics but parses as + # And(added:now, "3", "days") - added:now plus stray text. + "added:now - 3 days", + # A time-of-day component is split off and searched as text. + "added:2005-03-04T15:30:00Z", + ], + ) + def test_forms_the_docs_warn_about_match_nothing( + self, + backend: TantivyBackend, + dated: dict[str, int], + query: str, + ) -> None: + assert _matched_ids(backend, query) == set()