Compare commits

..
17 changed files with 193 additions and 67 deletions
+1 -2
View File
@@ -684,8 +684,7 @@ It requires [AI features](configuration.md#ai) to be enabled. You can specify:
never replace the document's existing tags. never replace the document's existing tags.
The action works with every trigger **except Consumption Started**, because suggestions are made from The action works with every trigger **except Consumption Started**, because suggestions are made from
the document's text, which does not exist until after the document has been processed. Documents whose the document's text, which does not exist until after the document has been processed.
processed text is empty or contains only whitespace are skipped.
Because the query to the AI service is slow, the action is queued and runs in the background rather Because the query to the AI service is slow, the action is queued and runs in the background rather
than as part of the workflow run itself. The document is updated once the suggestions come back. than as part of the workflow run itself. The document is updated once the suggestions come back.
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"python-ipware~=3.0.0", "python-ipware~=3.0.0",
"python-magic~=0.4.27", "python-magic~=0.4.27",
"rapidfuzz~=3.14.5", "rapidfuzz~=3.14.5",
"redis[hiredis]~=5.2.1", "redis[hiredis]~=6.4.0",
"regex>=2026.7.19", "regex>=2026.7.19",
"scikit-learn~=1.9.0", "scikit-learn~=1.9.0",
"sentence-transformers>=5.6.1", "sentence-transformers>=5.6.1",
@@ -23,17 +23,30 @@
<div class="col"> <div class="col">
<div class="card bg-light"> <div class="card bg-light">
<div class="card-body"> <div class="card-body">
<div class="card-title d-flex align-items-center"> <div class="card-title d-flex align-items-center flex-wrap">
<h6 class="mb-0"> <h6 class="mb-0">
{{option.title}} {{option.title}}
</h6> </h6>
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer"> <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
<i-bs name="info-circle"></i-bs> <i-bs name="info-circle"></i-bs>
</a> </a>
@if (isExternallyConfigured(option.config_key)) {
@if (isSet(option.key)) {
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="This value overrides {{option.config_key}}, which is set outside Paperless." i18n-title>Overrides external</span>
} @else {
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="{{option.config_key}} is set outside Paperless. Enter a value here to override it." i18n-title>Set externally</span>
}
}
@if (isSet(option.key)) { @if (isSet(option.key)) {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)"> @if (isExternallyConfigured(option.config_key)) {
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container> <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Use the externally configured value" i18n-title (click)="resetOption(option.key)">
</button> <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset to external</ng-container>
</button>
} @else {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
</button>
}
} }
</div> </div>
<div class="mb-n3"> <div class="mb-n3">
@@ -163,6 +163,19 @@ describe('ConfigComponent', () => {
expect(component.configForm.get('barcodes_enabled').value).toBeNull() expect(component.configForm.get('barcodes_enabled').value).toBeNull()
}) })
it('should identify externally configured options', () => {
component.externallyConfiguredVariables = new Set([
'PAPERLESS_OCR_LANGUAGE',
])
expect(
component.isExternallyConfigured('PAPERLESS_OCR_LANGUAGE')
).toBeTruthy()
expect(
component.isExternallyConfigured('PAPERLESS_OCR_OUTPUT_TYPE')
).toBeFalsy()
})
it('should group options into sections within a category, or not', () => { it('should group options into sections within a category, or not', () => {
const sections = component.getCategorySections(ConfigCategory.OCR) const sections = component.getCategorySections(ConfigCategory.OCR)
expect(sections).toEqual([null, ConfigSection.RemoteOCR]) expect(sections).toEqual([null, ConfigSection.RemoteOCR])
@@ -69,6 +69,7 @@ export class ConfigComponent
public configForm = new FormGroup({}) public configForm = new FormGroup({})
public errors = {} public errors = {}
public externallyConfiguredVariables = new Set<string>()
get optionCategories(): string[] { get optionCategories(): string[] {
return Object.values(ConfigCategory) return Object.values(ConfigCategory)
@@ -152,6 +153,9 @@ export class ConfigComponent
} }
private initialize(config: PaperlessConfig) { private initialize(config: PaperlessConfig) {
this.externallyConfiguredVariables = new Set(
config.externally_configured_variables ?? []
)
if (!this.store) { if (!this.store) {
this.store = new BehaviorSubject(config) this.store = new BehaviorSubject(config)
@@ -162,7 +166,9 @@ export class ConfigComponent
this.configForm.patchValue(state, { emitEvent: false }) this.configForm.patchValue(state, { emitEvent: false })
}) })
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable()) this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable(), {
excludeKeys: ['externally_configured_variables'],
})
} }
this.configForm.patchValue(config) this.configForm.patchValue(config)
@@ -227,6 +233,10 @@ export class ConfigComponent
return this.configForm.get(key).value != null return this.configForm.get(key).value != null
} }
public isExternallyConfigured(configKey: string): boolean {
return this.externallyConfiguredVariables.has(configKey)
}
public resetOption(key: string) { public resetOption(key: string) {
this.configForm.get(key).setValue(null) this.configForm.get(key).setValue(null)
} }
@@ -22,7 +22,7 @@
} }
::ng-deep .pngx-pdf-viewer-container { ::ng-deep .pngx-pdf-viewer-container {
overflow: hidden; overflow: hidden !important;
} }
.hover-actions { .hover-actions {
+1
View File
@@ -422,6 +422,7 @@ export const PaperlessConfigOptions: ConfigOption[] = [
] ]
export interface PaperlessConfig extends ObjectWithId { export interface PaperlessConfig extends ObjectWithId {
externally_configured_variables: string[]
output_type: OutputTypeConfig output_type: OutputTypeConfig
pages: number pages: number
language: string language: string
+49 -2
View File
@@ -35,7 +35,8 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
THEN: THEN:
- Existing config - Existing config
""" """
response = self.client.get(self.ENDPOINT, format="json") with patch.dict("os.environ", {}, clear=True):
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -45,6 +46,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
response.data[0], response.data[0],
{ {
"id": 1, "id": 1,
"externally_configured_variables": [],
"output_type": None, "output_type": None,
"pages": None, "pages": None,
"language": None, "language": None,
@@ -76,7 +78,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
"remote_ocr_api_key": None, "remote_ocr_api_key": None,
"remote_ocr_endpoint": None, "remote_ocr_endpoint": None,
"remote_ocr_mode": None, "remote_ocr_mode": None,
"ai_enabled": False, "ai_enabled": None,
"llm_embedding_backend": None, "llm_embedding_backend": None,
"llm_embedding_model": None, "llm_embedding_model": None,
"llm_embedding_endpoint": None, "llm_embedding_endpoint": None,
@@ -91,6 +93,31 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
}, },
) )
def test_api_get_config_reports_external_configuration_without_values(self) -> None:
with patch.dict(
"os.environ",
{
"PAPERLESS_OCR_LANGUAGE": "eng",
"PAPERLESS_REMOTE_OCR_API_KEY": "secret-value",
"PAPERLESS_FUTURE_SETTING": "future-value",
"UNRELATED_SETTING": "unrelated-value",
},
clear=True,
):
response = self.client.get(self.ENDPOINT, format="json")
self.assertCountEqual(
response.data[0]["externally_configured_variables"],
[
"PAPERLESS_FUTURE_SETTING",
"PAPERLESS_OCR_LANGUAGE",
"PAPERLESS_REMOTE_OCR_API_KEY",
],
)
self.assertNotContains(response, "secret-value")
self.assertNotContains(response, "future-value")
self.assertNotContains(response, "UNRELATED_SETTING")
def test_api_get_ui_settings_with_config(self) -> None: def test_api_get_ui_settings_with_config(self) -> None:
""" """
GIVEN: GIVEN:
@@ -949,6 +976,26 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
) )
mock_update.assert_called_once() mock_update.assert_called_once()
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND=None)
def test_external_ai_setting_triggers_index_update(self) -> None:
config = ApplicationConfiguration.objects.first()
assert config is not None
config.ai_enabled = None
config.llm_embedding_backend = None
config.save()
with (
patch("documents.tasks.llmindex_index.apply_async") as mock_update,
patch("paperless.views.llm_index_exists", return_value=False),
):
self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps({"llm_embedding_backend": "openai-like"}),
content_type="application/json",
)
mock_update.assert_called_once()
def test_update_llm_embedding_chunk_size_triggers_rebuild(self) -> None: def test_update_llm_embedding_chunk_size_triggers_rebuild(self) -> None:
config = ApplicationConfiguration.objects.first() config = ApplicationConfiguration.objects.first()
assert config is not None assert config is not None
-33
View File
@@ -5711,39 +5711,6 @@ class TestApplyAISuggestionsWorkflowAction(
self.assertEqual(changed, []) self.assertEqual(changed, [])
self.assertIn("AI is not enabled", "".join(cm.output)) self.assertIn("AI is not enabled", "".join(cm.output))
def test_document_without_content_does_nothing(self) -> None:
"""
GIVEN:
- A document whose OCR content is empty or whitespace-only
WHEN:
- AI suggestions are applied by a workflow
THEN:
- The classifier is not called and the document is left unchanged
"""
action = self.make_action(ai_overwrite_existing=True)
for content in ("", " \n\t"):
with self.subTest(content=content):
self.doc.content = content
self.doc.save(update_fields=["content"])
with (
mock.patch(
"documents.workflows.ai.get_ai_document_classification",
) as get_classification,
self.assertLogs(
"paperless.workflows.ai",
level="WARNING",
) as cm,
):
changed = apply_ai_suggestions_to_document(action, self.doc)
self.assertEqual(changed, [])
get_classification.assert_not_called()
self.assertIn("has no content", "".join(cm.output))
self.doc.refresh_from_db()
self.assertEqual(self.doc.title, "original.pdf")
def test_invalid_configuration_leaves_document_untouched(self) -> None: def test_invalid_configuration_leaves_document_untouched(self) -> None:
""" """
GIVEN: GIVEN:
-10
View File
@@ -138,16 +138,6 @@ def apply_ai_suggestions_to_document(
) )
return [] return []
if not document.content.strip():
logger.warning(
"Document %s has no content, skipping AI suggestions for workflow "
"action %s",
document.pk,
action.pk,
extra={"group": logging_group},
)
return []
# Workflows run without a user, so we use the document owner # Workflows run without a user, so we use the document owner
owner = document.owner owner = document.owner
+21 -7
View File
@@ -133,21 +133,27 @@ class BarcodeConfig(BaseConfig):
app_config = self._get_config_instance() app_config = self._get_config_instance()
self.barcodes_enabled = ( self.barcodes_enabled = (
app_config.barcodes_enabled or settings.CONSUMER_ENABLE_BARCODES app_config.barcodes_enabled
if app_config.barcodes_enabled is not None
else settings.CONSUMER_ENABLE_BARCODES
) )
self.barcode_enable_tiff_support = ( self.barcode_enable_tiff_support = (
app_config.barcode_enable_tiff_support app_config.barcode_enable_tiff_support
or settings.CONSUMER_BARCODE_TIFF_SUPPORT if app_config.barcode_enable_tiff_support is not None
else settings.CONSUMER_BARCODE_TIFF_SUPPORT
) )
self.barcode_string = ( self.barcode_string = (
app_config.barcode_string or settings.CONSUMER_BARCODE_STRING app_config.barcode_string or settings.CONSUMER_BARCODE_STRING
) )
self.barcode_retain_split_pages = ( self.barcode_retain_split_pages = (
app_config.barcode_retain_split_pages app_config.barcode_retain_split_pages
or settings.CONSUMER_BARCODE_RETAIN_SPLIT_PAGES if app_config.barcode_retain_split_pages is not None
else settings.CONSUMER_BARCODE_RETAIN_SPLIT_PAGES
) )
self.barcode_enable_asn = ( self.barcode_enable_asn = (
app_config.barcode_enable_asn or settings.CONSUMER_ENABLE_ASN_BARCODE app_config.barcode_enable_asn
if app_config.barcode_enable_asn is not None
else settings.CONSUMER_ENABLE_ASN_BARCODE
) )
self.barcode_asn_prefix = ( self.barcode_asn_prefix = (
app_config.barcode_asn_prefix or settings.CONSUMER_ASN_BARCODE_PREFIX app_config.barcode_asn_prefix or settings.CONSUMER_ASN_BARCODE_PREFIX
@@ -160,13 +166,17 @@ class BarcodeConfig(BaseConfig):
app_config.barcode_max_pages or settings.CONSUMER_BARCODE_MAX_PAGES app_config.barcode_max_pages or settings.CONSUMER_BARCODE_MAX_PAGES
) )
self.barcode_enable_tag = ( self.barcode_enable_tag = (
app_config.barcode_enable_tag or settings.CONSUMER_ENABLE_TAG_BARCODE app_config.barcode_enable_tag
if app_config.barcode_enable_tag is not None
else settings.CONSUMER_ENABLE_TAG_BARCODE
) )
self.barcode_tag_mapping = ( self.barcode_tag_mapping = (
app_config.barcode_tag_mapping or settings.CONSUMER_TAG_BARCODE_MAPPING app_config.barcode_tag_mapping or settings.CONSUMER_TAG_BARCODE_MAPPING
) )
self.barcode_tag_split = ( self.barcode_tag_split = (
app_config.barcode_tag_split or settings.CONSUMER_TAG_BARCODE_SPLIT app_config.barcode_tag_split
if app_config.barcode_tag_split is not None
else settings.CONSUMER_TAG_BARCODE_SPLIT
) )
@@ -248,7 +258,11 @@ class AIConfig(BaseConfig):
def __post_init__(self) -> None: def __post_init__(self) -> None:
app_config = self._get_config_instance() app_config = self._get_config_instance()
self.ai_enabled = app_config.ai_enabled or settings.AI_ENABLED self.ai_enabled = (
app_config.ai_enabled
if app_config.ai_enabled is not None
else settings.AI_ENABLED
)
self.llm_embedding_backend = ( self.llm_embedding_backend = (
app_config.llm_embedding_backend or settings.LLM_EMBEDDING_BACKEND app_config.llm_embedding_backend or settings.LLM_EMBEDDING_BACKEND
) )
@@ -0,0 +1,28 @@
from django.db import migrations
from django.db import models
def normalize_ai_enabled(apps, schema_editor):
application_configuration = apps.get_model(
"paperless",
"ApplicationConfiguration",
)
application_configuration.objects.filter(ai_enabled=False).update(ai_enabled=None)
class Migration(migrations.Migration):
dependencies = [
("paperless", "0015_applicationconfiguration_remote_ocr_mode"),
]
operations = [
migrations.AlterField(
model_name="applicationconfiguration",
name="ai_enabled",
field=models.BooleanField(
null=True,
verbose_name="Enables AI features",
),
),
migrations.RunPython(normalize_ai_enabled, migrations.RunPython.noop),
]
-1
View File
@@ -348,7 +348,6 @@ class ApplicationConfiguration(AbstractSingletonModel):
ai_enabled = models.BooleanField( ai_enabled = models.BooleanField(
verbose_name=_("Enables AI features"), verbose_name=_("Enables AI features"),
null=True, null=True,
default=False,
) )
llm_embedding_backend = models.CharField( llm_embedding_backend = models.CharField(
+8
View File
@@ -1,4 +1,5 @@
import logging import logging
import os
from io import BytesIO from io import BytesIO
import magic import magic
@@ -212,6 +213,7 @@ class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer[Use
class ApplicationConfigurationSerializer( class ApplicationConfigurationSerializer(
serializers.ModelSerializer[ApplicationConfiguration], serializers.ModelSerializer[ApplicationConfiguration],
): ):
externally_configured_variables = serializers.SerializerMethodField()
user_args = serializers.JSONField(binary=True, allow_null=True) user_args = serializers.JSONField(binary=True, allow_null=True)
barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True) barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
llm_api_key = ObfuscatedPasswordField( llm_api_key = ObfuscatedPasswordField(
@@ -227,6 +229,12 @@ class ApplicationConfigurationSerializer(
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key") OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
def get_externally_configured_variables(
self,
instance: ApplicationConfiguration,
) -> list[str]:
return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
def run_validation(self, data): def run_validation(self, data):
# Empty strings treated as None to avoid unexpected behavior # Empty strings treated as None to avoid unexpected behavior
if "user_args" in data and data["user_args"] == "": if "user_args" in data and data["user_args"] == "":
@@ -0,0 +1,32 @@
from django.test import TestCase
from django.test import override_settings
from paperless.config import AIConfig
from paperless.config import BarcodeConfig
from paperless.models import ApplicationConfiguration
class TestBooleanConfigPrecedence(TestCase):
@override_settings(CONSUMER_ENABLE_BARCODES=True)
def test_database_false_overrides_barcode_environment_setting(self) -> None:
config, _ = ApplicationConfiguration.objects.get_or_create()
config.barcodes_enabled = False
config.save()
self.assertFalse(BarcodeConfig().barcodes_enabled)
@override_settings(AI_ENABLED=True)
def test_database_false_overrides_ai_environment_setting(self) -> None:
config, _ = ApplicationConfiguration.objects.get_or_create()
config.ai_enabled = False
config.save()
self.assertFalse(AIConfig().ai_enabled)
@override_settings(AI_ENABLED=True)
def test_null_ai_setting_uses_environment_setting(self) -> None:
config, _ = ApplicationConfiguration.objects.get_or_create()
config.ai_enabled = None
config.save()
self.assertTrue(AIConfig().ai_enabled)
+6 -1
View File
@@ -443,8 +443,13 @@ class ApplicationConfigurationViewSet(ModelViewSet[ApplicationConfiguration]):
new_llm_embedding_backend = ( new_llm_embedding_backend = (
new_instance.llm_embedding_backend or settings.LLM_EMBEDDING_BACKEND new_instance.llm_embedding_backend or settings.LLM_EMBEDDING_BACKEND
) )
new_ai_enabled = (
new_instance.ai_enabled
if new_instance.ai_enabled is not None
else settings.AI_ENABLED
)
new_ai_index_enabled = bool( new_ai_index_enabled = bool(
new_instance.ai_enabled and new_llm_embedding_backend, new_ai_enabled and new_llm_embedding_backend,
) )
new_llm_embedding_chunk_size = ( new_llm_embedding_chunk_size = (
new_instance.llm_embedding_chunk_size or settings.LLM_EMBEDDING_CHUNK_SIZE new_instance.llm_embedding_chunk_size or settings.LLM_EMBEDDING_CHUNK_SIZE
Generated
+4 -4
View File
@@ -3070,7 +3070,7 @@ requires-dist = [
{ name = "python-ipware", specifier = "~=3.0.0" }, { name = "python-ipware", specifier = "~=3.0.0" },
{ name = "python-magic", specifier = "~=0.4.27" }, { name = "python-magic", specifier = "~=0.4.27" },
{ name = "rapidfuzz", specifier = "~=3.14.5" }, { name = "rapidfuzz", specifier = "~=3.14.5" },
{ name = "redis", extras = ["hiredis"], specifier = "~=5.2.1" }, { name = "redis", extras = ["hiredis"], specifier = "~=6.4.0" },
{ name = "regex", specifier = ">=2026.7.19" }, { name = "regex", specifier = ">=2026.7.19" },
{ name = "scikit-learn", specifier = "~=1.9.0" }, { name = "scikit-learn", specifier = "~=1.9.0" },
{ name = "sentence-transformers", specifier = ">=5.6.1" }, { name = "sentence-transformers", specifier = ">=5.6.1" },
@@ -4150,14 +4150,14 @@ wheels = [
[[package]] [[package]]
name = "redis" name = "redis"
version = "5.2.1" version = "6.4.0"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "async-timeout", marker = "python_full_version < '3.11.3'" }, { name = "async-timeout", marker = "python_full_version < '3.11.3'" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355, upload-time = "2024-12-06T09:50:41.956Z" } sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502, upload-time = "2024-12-06T09:50:39.656Z" }, { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" },
] ]
[package.optional-dependencies] [package.optional-dependencies]