mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-18 16:53:21 +00:00
Minor updates from line drifts
This commit is contained in:
@@ -16,7 +16,7 @@
|
||||
- `page_count:>5`, `asn:<10`, `page_count:>=5`, `asn:[1 TO 10]`, `tag_id:1,2,3` parse OK (comparison operators produce correct `RangeQuery`).
|
||||
- `asn:[1 TO]` / `asn:[TO 10]` are a **Syntax Error** (open numeric ranges unsupported; only open _date_ ranges work via sentinels).
|
||||
- `scan()` only tokenizes fields in `KNOWN_FIELDS`; unknown `foobar:hello` stays a `Passthrough` and only fails at `parse_query` -> detected by the backstop, not proactively.
|
||||
- `difflib.get_close_matches("corespondent", pool)` -> `["correspondent"]`; `has_tags`/`http`/`12` -> `[]` (bare message).
|
||||
- `difflib.get_close_matches("correspondent", pool)` -> `["correspondent"]`; `has_tags`/`http`/`12` -> `[]` (bare message).
|
||||
- `tantivy.Schema` exposes no field-name list, so the drift guard is parse-based.
|
||||
|
||||
## File Structure
|
||||
@@ -74,7 +74,7 @@ class TestErrorClasses:
|
||||
assert str(err) == "Unknown search field 'has_tags'."
|
||||
|
||||
def test_unknown_field_message_with_suggestion(self):
|
||||
err = UnknownFieldError("corespondent", suggestion="correspondent")
|
||||
err = UnknownFieldError("correspondent", suggestion="correspondent")
|
||||
assert err.suggestion == "correspondent"
|
||||
assert str(err) == (
|
||||
"Unknown search field 'corespondent'. Did you mean 'correspondent'?"
|
||||
@@ -350,9 +350,9 @@ from documents.search._translate import map_tantivy_error
|
||||
class TestMapTantivyError:
|
||||
def test_unknown_field_maps_with_suggestion(self):
|
||||
exc = ValueError("Field does not exist: 'corespondent'")
|
||||
mapped = map_tantivy_error(exc, "corespondent:foo")
|
||||
mapped = map_tantivy_error(exc, "correspondent:foo")
|
||||
assert isinstance(mapped, UnknownFieldError)
|
||||
assert mapped.field == "corespondent"
|
||||
assert mapped.field == "correspondent"
|
||||
assert mapped.suggestion == "correspondent"
|
||||
|
||||
def test_unknown_field_maps_without_suggestion(self):
|
||||
@@ -481,7 +481,7 @@ git commit -m "feat(search): map tantivy parse errors to user-safe messages"
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/documents/search/_query.py` (import `map_tantivy_error`; add `_parse_query_friendly`; use it at lines 231-235 and 253-259)
|
||||
- Modify: `src/documents/search/_query.py` (import `map_tantivy_error`; add `_parse_query_friendly`; use it at lines 216-220 and 238-244)
|
||||
- Test: `src/documents/tests/search/test_error_shapes.py`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
@@ -506,7 +506,7 @@ class TestBackstopViaParseUserQuery:
|
||||
|
||||
def test_unknown_field_suggestion(self, index: tantivy.Index):
|
||||
with pytest.raises(UnknownFieldError) as exc_info:
|
||||
parse_user_query(index, "corespondent:bob", UTC)
|
||||
parse_user_query(index, "correspondent:bob", UTC)
|
||||
assert exc_info.value.suggestion == "correspondent"
|
||||
|
||||
def test_legacy_backend_field_is_unknown(self, index: tantivy.Index):
|
||||
@@ -555,13 +555,13 @@ Expected: FAIL — unknown-field/syntax cases currently raise the bare Tantivy `
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
In `src/documents/search/_query.py`, add the import alongside the existing translate imports (after line 20):
|
||||
In `src/documents/search/_query.py`, add the import alongside the existing translate imports (after line 13):
|
||||
|
||||
```python
|
||||
from documents.search._translate import map_tantivy_error
|
||||
```
|
||||
|
||||
Add a module-level helper (place it just above `parse_user_query`, before line 191):
|
||||
Add a module-level helper (place it just above `parse_user_query`, before line 176):
|
||||
|
||||
```python
|
||||
def _parse_query_friendly(
|
||||
@@ -584,7 +584,7 @@ def _parse_query_friendly(
|
||||
raise
|
||||
```
|
||||
|
||||
In `parse_user_query`, replace the exact-query parse (lines 231-235):
|
||||
In `parse_user_query`, replace the exact-query parse (lines 216-220):
|
||||
|
||||
```python
|
||||
exact = _parse_query_friendly(
|
||||
@@ -596,7 +596,7 @@ In `parse_user_query`, replace the exact-query parse (lines 231-235):
|
||||
)
|
||||
```
|
||||
|
||||
and the fuzzy parse (lines 253-259):
|
||||
and the fuzzy parse (lines 238-244):
|
||||
|
||||
```python
|
||||
fuzzy = _parse_query_friendly(
|
||||
@@ -610,7 +610,7 @@ and the fuzzy parse (lines 253-259):
|
||||
)
|
||||
```
|
||||
|
||||
(`SearchQueryError` is already imported in `_query.py` at line 19.)
|
||||
(`SearchQueryError` is already imported in `_query.py` at line 12.)
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
@@ -712,7 +712,7 @@ Expected: PASS. (These assert current truth; they guard against future drift. If
|
||||
|
||||
- [ ] **Step 4: Write the failing view-level test**
|
||||
|
||||
In `src/documents/tests/test_api_search.py`, locate `test_search_added_invalid_date` (around line 723) and add this test directly after it, inside the same `TestDocumentSearchApi` class (mirrors that test's structure):
|
||||
In `src/documents/tests/test_api_search.py`, locate `test_search_added_invalid_date` (around line 765) and add this test directly after it, inside the same `TestDocumentSearchApi` class (mirrors that test's structure):
|
||||
|
||||
```python
|
||||
def test_search_unknown_field_returns_400(self) -> None:
|
||||
|
||||
@@ -95,7 +95,7 @@ class TestBulkEditPermissionMatrix:
|
||||
Notes:
|
||||
|
||||
- Mock the underlying `bulk_edit.<fn>` (patch `documents.views.bulk_edit.<fn>`) so the operations don't actually run — this test is purely about the permission gate returning 200 vs 403.
|
||||
- A superuser short-circuits to allowed (`views.py:2697`); include one superuser row to pin that.
|
||||
- A superuser short-circuits to allowed (`views.py:2833`); include one superuser row to pin that.
|
||||
- This is verbose by design; the matrix is the security contract. Prefer one parametrized test over hand-written methods.
|
||||
- **Cover the six moved single-action endpoints too (REQUIRED — C2).** `/api/documents/rotate/`, `/merge/`, `/delete/`, `/reprocess/`, `/edit_pdf/`, `/remove_password/` run the **same** `_has_document_permissions` gate via `_execute_document_action`, and that path is rewritten in Task 3 (C1). Add a parallel parametrized test that POSTs to each (their request bodies are the dedicated serializers' fields — e.g. `{"documents": [...], "degrees": 90}` for rotate — **not** a `method`+`parameters` envelope). The existing `test_api_bulk_edit.py` already covers these endpoints' permission gates (`test_rotate_insufficient_permissions:1320`, `test_merge_and_delete_insufficient_permissions:1381`, `test_edit_pdf_insufficient_permissions:1635`, `test_remove_password_insufficient_permissions:1719`), so this is hardening rather than the sole net — but make the moved-endpoint matrix explicit here so the `_execute_document_action` rewrite is guarded by a parametrized characterization, not scattered one-offs.
|
||||
- **`edit_pdf` test docs need a `page_count` (M3).** `clean_parameters` for `edit_pdf` bounds-checks `op["page"]` against `Document.page_count` (`serialisers.py:2052-2059`); this test mocks execution but **not** validation, so an `edit_pdf` row with `page: 1` needs its target doc created with `page_count >= 1`, else it fails with a 400 (out-of-bounds) instead of the expected 200/403.
|
||||
@@ -103,7 +103,7 @@ Notes:
|
||||
- [ ] **Step 2: Run it against CURRENT code — it must PASS**
|
||||
|
||||
Run: `bash /c/Users/tholmes/Documents/Coding/paperless/vmtest.sh "src/documents/tests/test_bulk_operations.py -v"`
|
||||
Expected: PASS. If any row is red, the spec matrix is misread — reconcile against `views.py:2713-2760` before writing any production code.
|
||||
Expected: PASS. If any row is red, the spec matrix is misread — reconcile against `views.py:2843-2906` before writing any production code.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
|
||||
@@ -519,7 +519,7 @@ git commit -m "Feature: consume_file owns and cleans the staged work_root"
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/documents/management/commands/document_consumer.py:340-353`
|
||||
- Modify: `src/documents/management/commands/document_consumer.py:344-357`
|
||||
- Test: `src/documents/tests/test_management_consumer.py:99-103`
|
||||
|
||||
- [ ] **Step 1: Repoint the consumer test fixture**
|
||||
@@ -548,7 +548,7 @@ Expected: FAIL — the folder site still calls `consume_file.apply_async`, not t
|
||||
|
||||
- [ ] **Step 3: Migrate the folder enqueue site**
|
||||
|
||||
In `src/documents/management/commands/document_consumer.py`, add `from documents import ingest` at the top, and replace the enqueue block (lines ~340-353):
|
||||
In `src/documents/management/commands/document_consumer.py`, add `from documents import ingest` at the top, and replace the enqueue block (lines ~344-357):
|
||||
|
||||
```python
|
||||
# Queue for consumption
|
||||
@@ -594,7 +594,7 @@ rewrite and the site migration land together.
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/documents/tests/utils.py:242-274`
|
||||
- Modify: `src/documents/views.py:3149-3192` (PostDocumentView) and `:1917-1976` (update_version)
|
||||
- Modify: `src/documents/views.py:3275-3338` (PostDocumentView) and `:2036-2098` (update_version)
|
||||
- Modify: `src/documents/tests/test_api_document_versions.py` (patch target)
|
||||
|
||||
- [ ] **Step 1: Rewrite `ConsumeTaskMixin` to patch the seam**
|
||||
@@ -643,7 +643,7 @@ Expected: FAIL — `PostDocumentView` still calls `consume_file.apply_async`, so
|
||||
- [ ] **Step 3: Migrate `PostDocumentView.post`**
|
||||
|
||||
In `src/documents/views.py`, ensure `from documents import ingest` is imported,
|
||||
then replace the staging + dispatch (lines ~3149-3192) with a `stage_document`
|
||||
then replace the staging + dispatch (lines ~3275-3338) with a `stage_document`
|
||||
block:
|
||||
|
||||
```python
|
||||
@@ -687,7 +687,7 @@ The old `SCRATCH_DIR.mkdir` + `mkdtemp` + `write_bytes` + the explicit
|
||||
|
||||
- [ ] **Step 4: Migrate `update_version`**
|
||||
|
||||
In `src/documents/views.py` `update_version` (lines ~1917-1976), replace its
|
||||
In `src/documents/views.py` `update_version` (lines ~2036-2098), replace its
|
||||
`mkdtemp`/`write`/`consume_file.apply_async` with the same pattern, preserving its
|
||||
specific fields (`root_document_id`, `version_label`, `actor_id`):
|
||||
|
||||
@@ -732,7 +732,7 @@ git commit -m "Refactor: route API/WebUI/version ingest through the staging seam
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/paperless_mail/mail.py` (`_handle_message` ~716-760, `_process_attachments` ~861-908, `_process_eml` ~952-1006)
|
||||
- Modify: `src/paperless_mail/mail.py` (`_handle_message` ~790-835, `_process_attachments` ~893-1024, `_process_eml` ~1026-1097)
|
||||
- Test: `src/paperless_mail/tests/test_mail.py`, `test_mail_nfc.py` (verify, likely no change)
|
||||
|
||||
- [ ] **Step 1: Wrap the message's staging in an `ExitStack`**
|
||||
@@ -774,7 +774,7 @@ work_root for the message.)
|
||||
|
||||
- [ ] **Step 2: Stage each attachment via the stack**
|
||||
|
||||
Replace the attachment staging (`mail.py:861-908`) inside `_process_attachments`:
|
||||
Replace the attachment staging (`mail.py:893-1024`) inside `_process_attachments`:
|
||||
|
||||
```python
|
||||
staged = staging_stack.enter_context(
|
||||
@@ -804,7 +804,7 @@ Replace the attachment staging (`mail.py:861-908`) inside `_process_attachments`
|
||||
|
||||
The old `SCRATCH_DIR.mkdir` + `mkdtemp` + `write_bytes` + `consume_file.s(...).set(...)`
|
||||
are gone; `stage_document` handles the temp dir and `build_consume_signature` the
|
||||
header. Do the analogous replacement in `_process_eml` (`mail.py:952-1006`),
|
||||
header. Do the analogous replacement in `_process_eml` (`mail.py:1026-1097`),
|
||||
staging the `.eml` bytes and building the signature the same way.
|
||||
|
||||
- [ ] **Step 3: Run the mail suites**
|
||||
@@ -885,14 +885,14 @@ folder source needs it (its loose file in `CONSUMPTION_DIR` is removed on succes
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/documents/consumer.py:417-422`
|
||||
- Modify: `src/documents/consumer.py:408-414`
|
||||
- Test: `src/documents/tests/test_consumer.py`
|
||||
|
||||
- [ ] **Step 1: Use the handed-in working dir instead of a second `TemporaryDirectory`**
|
||||
|
||||
`ConsumerPlugin` already receives the task's working dir as `self.base_tmp_dir`
|
||||
(the `tmp_dir` arg from `tasks.py:227-233`). Replace its own
|
||||
`tempfile.TemporaryDirectory(...)` (`consumer.py:417`) with a subfolder of that
|
||||
`tempfile.TemporaryDirectory(...)` (`consumer.py:408`) with a subfolder of that
|
||||
handed-in dir:
|
||||
|
||||
```python
|
||||
|
||||
@@ -389,7 +389,7 @@ git commit -m "Add HybridRetriever: lexical fallback for archive AI chat retriev
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/paperless_ai/chat.py:95-112`
|
||||
- Modify: `src/paperless_ai/chat.py:149-177`
|
||||
- Modify: `src/paperless_ai/tests/test_chat.py`
|
||||
|
||||
**Interfaces:**
|
||||
@@ -436,7 +436,7 @@ Expected: FAIL — `HybridRetriever` is never constructed yet (`chat.py` still b
|
||||
|
||||
- [ ] **Step 3: Wire `HybridRetriever` into `_stream_chat_with_documents`**
|
||||
|
||||
In `src/paperless_ai/chat.py`, inside `_stream_chat_with_documents` (currently lines 95-112), add the `HybridRetriever` import alongside the other lazy llama-index imports and replace the retriever construction:
|
||||
In `src/paperless_ai/chat.py`, inside `_stream_chat_with_documents` (currently lines 149-177), add the `HybridRetriever` import alongside the other lazy llama-index imports and replace the retriever construction:
|
||||
|
||||
```python
|
||||
from llama_index.core.prompts import PromptTemplate
|
||||
|
||||
@@ -133,7 +133,7 @@ Individual failures are logged and counted but do not abort the run. Bidirection
|
||||
| `src/documents/signals/handlers.py` | `shutil.move()` → `storage.move()`; remove `create_source_path_directory` / `delete_empty_directories` callsites |
|
||||
| `src/documents/tasks.py` | Same as signals |
|
||||
| `src/documents/file_handling.py` | `exists()` checks and directory references use storage API |
|
||||
| `src/documents/views/` | File-serving views use `storage.open()` within context; wrap for `FileResponse` lifecycle |
|
||||
| `src/documents/views.py` | File-serving views use `storage.open()` within context; wrap for `FileResponse` lifecycle |
|
||||
| `src/documents/management/commands/document_importer.py` | Replace `Path.glob()` and direct copies with storage API |
|
||||
| `src/documents/management/commands/document_exporter.py` | Replace direct file copies and `FileLock`-guarded writes with storage API |
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ hard-to-fix bugs (see the revert/refix history around password removal: #12803,
|
||||
`DOCUMENT_ADDED` workflow fires from `run_workflows_added`, which runs while
|
||||
the consumer is still inside its transaction — _before_ the consumed file is
|
||||
copied to `document.source_path` (`document_consumption_finished` is sent at
|
||||
`consumer.py:658`; the file copy happens after, at `consumer.py:670+`). The
|
||||
`consumer.py:654`; the file copy happens after, at `consumer.py:666+`). The
|
||||
staged path is therefore threaded through as `original_file` /
|
||||
`caller_supplied_original_file` parameters. Actions that read the file
|
||||
(password removal, email attachments) depend on this plumbing being correct.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
`docs/superpowers/done/specs/2026-06-14-search-query-translation-design.md`.
|
||||
**Builds on:** the `SearchQueryError(ValueError)` base in
|
||||
`documents/search/_translate.py` and the single `except SearchQueryError` handler
|
||||
in `UnifiedSearchViewSet.list` (`documents/views.py:2477`), which re-raises as DRF
|
||||
in `UnifiedSearchViewSet.list` (`documents/views.py:2612`), which re-raises as DRF
|
||||
`ValidationError({"query": [msg]})`. Any new subclass surfaces through that one
|
||||
handler automatically, so this work is purely additive.
|
||||
|
||||
@@ -15,7 +15,7 @@ handler automatically, so this work is purely additive.
|
||||
Every advanced-search failure other than the now-handled invalid date lands in
|
||||
the view's generic `except Exception` and returns
|
||||
`HttpResponseBadRequest("Error listing search results, check logs for more
|
||||
detail.")` (`views.py:2479-2482`). `index.parse_query(...)` runs _outside_ the
|
||||
detail.")` (`views.py:2617-2621`). `index.parse_query(...)` runs _outside_ the
|
||||
`translate_query` try/except in `parse_user_query` (`_query.py:220-235`), so
|
||||
anything Tantivy rejects bypasses `SearchQueryError` entirely and gets the
|
||||
unhelpful generic 400. Some Tantivy errors also leak Rust internals (e.g.
|
||||
|
||||
@@ -131,7 +131,9 @@ Both postdate the `0.26.0` wheel.
|
||||
- Tantivy side (does a translated string parse?): build a real index via
|
||||
`documents.search._schema.build_schema` + `register_tokenizers`, then
|
||||
`index.parse_query(translate_query(q, tz), DEFAULT_SEARCH_FIELDS, field_boosts=…)`.
|
||||
- Whoosh side (what did v2 do?): the old `get_schema()` + `MultifieldParser([...]) +
|
||||
DateParserPlugin(...)` still exists on `main` (`src/documents/index.py`); run a query
|
||||
through it to get the ground-truth `Query`.
|
||||
- Whoosh side (what did v2 do?): `src/documents/index.py` (the old `get_schema()` +
|
||||
`MultifieldParser([...]) + DateParserPlugin(...)`) was deleted from `main` in
|
||||
`aed9abe48` (#12471, 2026-04-02); check it out at `git show aed9abe48^:src/documents/index.py`
|
||||
(or `git checkout aed9abe48^ -- src/documents/index.py`) and run a query through it
|
||||
to get the ground-truth `Query`.
|
||||
- A fuller empirical gap matrix lives in `SEARCH_TANTIVY_WHOOSH_COMPAT.md`.
|
||||
|
||||
@@ -11,11 +11,11 @@ Every document that enters paperless converges on one operation: build a
|
||||
and dispatch the `consume_file` Celery task with a `trigger_source` header. That
|
||||
operation is hand-rolled at **five** sites today, plus a sixth internal one:
|
||||
|
||||
- consume-folder watcher — `document_consumer.py:342`
|
||||
- API upload + Web UI — `views.py:3181` (one endpoint, two `DocumentSource` values)
|
||||
- document-version upload — `views.py:1964`
|
||||
- mail attachment — `mail.py:899`
|
||||
- mail `.eml` whole-message — `mail.py:987`
|
||||
- consume-folder watcher — `document_consumer.py:346`
|
||||
- API upload + Web UI — `views.py:3327` (one endpoint, two `DocumentSource` values)
|
||||
- document-version upload — `views.py:2086`
|
||||
- mail attachment — `mail.py:993`
|
||||
- mail `.eml` whole-message — `mail.py:1084`
|
||||
- barcode split children (internal re-enqueue) — `barcodes.py:190`/`227`
|
||||
|
||||
The duplication causes three concrete problems:
|
||||
@@ -29,10 +29,10 @@ The duplication causes three concrete problems:
|
||||
2. **A scratch leak from split staging/cleanup ownership.** Staged sources create
|
||||
scratch input under `SCRATCH_DIR` that nothing ever fully removes:
|
||||
`ConsumerPlugin` unlinks only the input **file**, and only on the success path
|
||||
(`consumer.py:742`). The exact leak shape varies by site — mail attachments and
|
||||
(`consumer.py:738`). The exact leak shape varies by site — mail attachments and
|
||||
API/version use `mkdtemp` + a file inside, so the **directory** is orphaned
|
||||
(empty after success, dir-with-file on failure); the mail `.eml` path uses
|
||||
`mkstemp` (`mail.py:~955`), so it leaks a **file** directly in `SCRATCH_DIR` on
|
||||
`mkstemp` (`mail.py:~1034`), so it leaks a **file** directly in `SCRATCH_DIR` on
|
||||
failure. Either way there is no owner that removes the staged input on every
|
||||
terminal path.
|
||||
|
||||
@@ -46,7 +46,7 @@ The duplication causes three concrete problems:
|
||||
Separately, the consumption task already has **two** working temp directories that
|
||||
duplicate each other: `consume_file` opens one `TemporaryDirectory` and passes it
|
||||
to every plugin (`tasks.py:220`), but `ConsumerPlugin` ignores that and opens its
|
||||
_own_ second `TemporaryDirectory` (`consumer.py:417`).
|
||||
_own_ second `TemporaryDirectory` (`consumer.py:408`).
|
||||
|
||||
## Goal
|
||||
|
||||
@@ -194,7 +194,7 @@ with a derived work_root:
|
||||
|
||||
The per-task working directory passed to plugins becomes a **subfolder of
|
||||
work_root**, and `ConsumerPlugin` uses that handed-in directory for its working
|
||||
copy instead of opening its own second `TemporaryDirectory` (`consumer.py:417`).
|
||||
copy instead of opening its own second `TemporaryDirectory` (`consumer.py:408`).
|
||||
One tree per document; one cleanup.
|
||||
|
||||
### Barcode split children (`barcodes.py`)
|
||||
@@ -216,7 +216,7 @@ independently cleanable when the parent stops.
|
||||
Mail is the one source that does **not** dispatch per file: `_handle_message`
|
||||
collects N attachment signatures (and optionally the `.eml` signature), then
|
||||
`queue_consumption_tasks` wraps them in a single `chord(...).delay()` _after_ the
|
||||
loop (`mail.py:919`). A per-file `release()` is therefore wrong — if `release()`
|
||||
loop (`mail.py:1014`). A per-file `release()` is therefore wrong — if `release()`
|
||||
ran per attachment and the later chord dispatch threw, every staged file would be
|
||||
orphaned, reopening the leak. **The ownership boundary is the whole message:**
|
||||
|
||||
@@ -237,7 +237,7 @@ def _handle_message(...):
|
||||
`queue_consumption_tasks` itself is unchanged. `build_consume_signature` **must
|
||||
pass `input_doc`/`overrides` as keyword args** (`consume_file.s(input_doc=...,
|
||||
overrides=...)`) so the resulting `Signature.kwargs` keeps the shape mail tests
|
||||
assert on (`test_mail.py:365-366`).
|
||||
assert on (`test_mail.py:389-390`).
|
||||
|
||||
### Call-site refactor (the external sites)
|
||||
|
||||
@@ -298,7 +298,7 @@ consume_file task (async, later)
|
||||
extend to it; (b) the plan must verify the move-precedes-stop ordering, since it
|
||||
is load-bearing for the cleanup rule.
|
||||
- **`ConsumerPlugin`'s own cleanup becomes partly redundant.** On success it
|
||||
unlinks `original_file` and `working_copy` (`consumer.py:742/744`), both of
|
||||
unlinks `original_file` and `working_copy` (`consumer.py:738/740`), both of
|
||||
which now live inside work_root that the task `finally` `rmtree`s. The redundant
|
||||
unlinks are harmless but the plan should remove them for clarity, while keeping
|
||||
the qpdf `--replace-input` recovery (`unmodified_original`, `consumer.py:452+`)
|
||||
|
||||
@@ -11,7 +11,7 @@ The archive-wide AI chat (`ChatStreamingView` with no `document_id`, backed by
|
||||
purely via dense-vector similarity search: `VectorIndexRetriever` embeds the
|
||||
user's question and does cosine-similarity nearest-neighbor search over chunk
|
||||
embeddings, with a hardcoded `similarity_top_k` of 5 (`CHAT_RETRIEVER_TOP_K`,
|
||||
`chat.py:20`) and no similarity cutoff.
|
||||
`chat.py:22`) and no similarity cutoff.
|
||||
|
||||
Dense embeddings are known to perform poorly on exact keyword, rare/foreign
|
||||
word, and numeric-string matching (e.g. a compound German word like
|
||||
@@ -60,9 +60,9 @@ already computed for vector search wherever possible.
|
||||
caller's permitted document IDs.
|
||||
2. Call `documents.search.get_backend().search_ids(query_str, user=user,
|
||||
search_mode=SearchMode.TEXT, limit=CHAT_LEXICAL_TOP_K)` — the same idiom
|
||||
already used in `views.py:3522` — to get lexical document-ID matches.
|
||||
already used in `views.py:3588` — to get lexical document-ID matches.
|
||||
`user` is `None` for superusers and `request.user` otherwise, matching the
|
||||
existing permission pattern (`views.py:3521`). Intersect the returned IDs
|
||||
existing permission pattern (`views.py:3587`). Intersect the returned IDs
|
||||
with the caller-provided `documents` set so results never exceed what the
|
||||
caller already permission-scoped (this is what makes the retriever safe
|
||||
to use for both the archive-wide and single-document cases).
|
||||
@@ -72,7 +72,7 @@ search_mode=SearchMode.TEXT, limit=CHAT_LEXICAL_TOP_K)` — the same idiom
|
||||
and a metadata filter restricted to that one document ID, reusing the
|
||||
same query embedding. This is deliberate: `PaperlessSqliteVecVectorStore
|
||||
.query()` runs a single global `vec0` KNN search over the WHERE-filtered
|
||||
rows (`vector_store.py:409-434`) — it does not partition top-k per
|
||||
rows (`vector_store.py:491-527`) — it does not partition top-k per
|
||||
document — so a single batched call across N lexical-hit documents with
|
||||
`top_k=N` could return several chunks from one document and none from
|
||||
another. Per-document calls are the only way to guarantee each lexical
|
||||
@@ -122,7 +122,7 @@ search_mode=SearchMode.TEXT, limit=CHAT_LEXICAL_TOP_K)` — the same idiom
|
||||
internally exactly as today and used as step 1 of the hybrid flow.
|
||||
- **Changed:** `stream_chat_with_documents()` / `_stream_chat_with_documents()`
|
||||
gain a `user` parameter, threaded from `ChatStreamingView.post`
|
||||
(`views.py:2268`) using the same `None`-for-superuser convention already
|
||||
(`views.py:2303`) using the same `None`-for-superuser convention already
|
||||
used elsewhere in `views.py`.
|
||||
|
||||
### Scope: applies to both chat modes
|
||||
@@ -140,7 +140,7 @@ that vector similarity alone might miss.
|
||||
reason, the lexical step should degrade gracefully to vector-only results
|
||||
(log and continue) rather than failing the whole chat response — chat
|
||||
already wraps everything in a try/except at the `stream_chat_with_documents`
|
||||
level (`chat.py:82-87`), but the lexical addition should not, by itself,
|
||||
level (`chat.py:138-146`), but the lexical addition should not, by itself,
|
||||
turn a previously-working vector-only answer into an error.
|
||||
|
||||
### Testing
|
||||
|
||||
Reference in New Issue
Block a user