mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-03-12 04:01:22 +00:00
Compare commits
5 Commits
feature-tr
...
fix-migrat
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a0a412a51 | ||
|
|
86fa74c115 | ||
|
|
b7b9e83f37 | ||
|
|
217b5df591 | ||
|
|
3efc9a5733 |
@@ -56,6 +56,7 @@ services:
|
||||
environment:
|
||||
PAPERLESS_REDIS: redis://broker:6379
|
||||
PAPERLESS_DBHOST: db
|
||||
PAPERLESS_DBENGINE: postgres
|
||||
env_file:
|
||||
- stack.env
|
||||
volumes:
|
||||
|
||||
@@ -62,6 +62,7 @@ services:
|
||||
environment:
|
||||
PAPERLESS_REDIS: redis://broker:6379
|
||||
PAPERLESS_DBHOST: db
|
||||
PAPERLESS_DBENGINE: postgresql
|
||||
PAPERLESS_TIKA_ENABLED: 1
|
||||
PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
|
||||
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
|
||||
|
||||
@@ -56,6 +56,7 @@ services:
|
||||
environment:
|
||||
PAPERLESS_REDIS: redis://broker:6379
|
||||
PAPERLESS_DBHOST: db
|
||||
PAPERLESS_DBENGINE: postgresql
|
||||
volumes:
|
||||
data:
|
||||
media:
|
||||
|
||||
@@ -51,6 +51,7 @@ services:
|
||||
env_file: docker-compose.env
|
||||
environment:
|
||||
PAPERLESS_REDIS: redis://broker:6379
|
||||
PAPERLESS_DBENGINE: sqlite
|
||||
PAPERLESS_TIKA_ENABLED: 1
|
||||
PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
|
||||
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
|
||||
|
||||
@@ -42,6 +42,7 @@ services:
|
||||
env_file: docker-compose.env
|
||||
environment:
|
||||
PAPERLESS_REDIS: redis://broker:6379
|
||||
PAPERLESS_DBENGINE: sqlite
|
||||
volumes:
|
||||
data:
|
||||
media:
|
||||
|
||||
@@ -10,8 +10,10 @@ cd "${PAPERLESS_SRC_DIR}"
|
||||
|
||||
# The whole migrate, with flock, needs to run as the right user
|
||||
if [[ -n "${USER_IS_NON_ROOT}" ]]; then
|
||||
python3 manage.py check --tag compatibility paperless
|
||||
exec s6-setlock -n "${data_dir}/migration_lock" python3 manage.py migrate --skip-checks --no-input
|
||||
else
|
||||
s6-setuidgid paperless python3 manage.py check --tag compatibility paperless
|
||||
exec s6-setuidgid paperless \
|
||||
s6-setlock -n "${data_dir}/migration_lock" \
|
||||
python3 manage.py migrate --skip-checks --no-input
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { DatePipe } from '@angular/common'
|
||||
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'
|
||||
import {
|
||||
HttpTestingController,
|
||||
@@ -138,6 +139,7 @@ describe('BulkEditorComponent', () => {
|
||||
},
|
||||
},
|
||||
FilterPipe,
|
||||
DatePipe,
|
||||
SettingsService,
|
||||
{
|
||||
provide: UserService,
|
||||
|
||||
@@ -169,7 +169,7 @@ def match_storage_paths(document: Document, classifier: DocumentClassifier, user
|
||||
def matches(matching_model: MatchingModel, document: Document):
|
||||
search_flags = 0
|
||||
|
||||
document_content = document.content
|
||||
document_content = document.get_effective_content() or ""
|
||||
|
||||
# Check that match is not empty
|
||||
if not matching_model.match.strip():
|
||||
|
||||
@@ -5,7 +5,7 @@ from django.db import migrations
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0003_workflowaction_order"),
|
||||
("documents", "0002_squashed"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -1,18 +0,0 @@
|
||||
# Generated by Django 5.2.9 on 2026-01-20 20:06
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0002_squashed"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="order",
|
||||
field=models.PositiveIntegerField(default=0, verbose_name="order"),
|
||||
),
|
||||
]
|
||||
@@ -6,7 +6,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0004_remove_document_storage_type"),
|
||||
("documents", "0003_remove_document_storage_type"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -6,7 +6,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0005_workflowtrigger_filter_has_any_correspondents_and_more"),
|
||||
("documents", "0004_workflowtrigger_filter_has_any_correspondents_and_more"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -7,7 +7,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0006_alter_document_checksum_unique"),
|
||||
("documents", "0005_alter_document_checksum_unique"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -46,7 +46,7 @@ def revoke_share_link_bundle_permissions(apps, schema_editor):
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
("documents", "0007_document_content_length"),
|
||||
("documents", "0006_document_content_length"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -6,7 +6,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0008_sharelinkbundle"),
|
||||
("documents", "0007_sharelinkbundle"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -7,7 +7,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0009_workflowaction_passwords_alter_workflowaction_type"),
|
||||
("documents", "0008_workflowaction_passwords_alter_workflowaction_type"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -7,7 +7,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0010_alter_document_content_length"),
|
||||
("documents", "0009_alter_document_content_length"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -6,7 +6,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0011_optimize_integer_field_sizes"),
|
||||
("documents", "0010_optimize_integer_field_sizes"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -7,7 +7,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0012_alter_workflowaction_type"),
|
||||
("documents", "0011_alter_workflowaction_type"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -6,7 +6,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0013_document_root_document"),
|
||||
("documents", "0012_document_root_document"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -124,7 +124,7 @@ def _restore_visibility_fields(apps, schema_editor):
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0014_alter_paperlesstask_task_name"),
|
||||
("documents", "0013_alter_paperlesstask_task_name"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -7,7 +7,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0015_savedview_visibility_to_ui_settings"),
|
||||
("documents", "0014_savedview_visibility_to_ui_settings"),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
@@ -361,6 +361,42 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
res += f" {self.title}"
|
||||
return res
|
||||
|
||||
def get_effective_content(self) -> str | None:
|
||||
"""
|
||||
Returns the effective content for the document.
|
||||
|
||||
For root documents, this is the latest version's content when available.
|
||||
For version documents, this is always the document's own content.
|
||||
If the queryset already annotated ``effective_content``, that value is used.
|
||||
"""
|
||||
if hasattr(self, "effective_content"):
|
||||
return getattr(self, "effective_content")
|
||||
|
||||
if self.root_document_id is not None or self.pk is None:
|
||||
return self.content
|
||||
|
||||
prefetched_cache = getattr(self, "_prefetched_objects_cache", None)
|
||||
prefetched_versions = (
|
||||
prefetched_cache.get("versions")
|
||||
if isinstance(prefetched_cache, dict)
|
||||
else None
|
||||
)
|
||||
if prefetched_versions:
|
||||
latest_prefetched = max(prefetched_versions, key=lambda doc: doc.id)
|
||||
return latest_prefetched.content
|
||||
|
||||
latest_version_content = (
|
||||
Document.objects.filter(root_document=self)
|
||||
.order_by("-id")
|
||||
.values_list("content", flat=True)
|
||||
.first()
|
||||
)
|
||||
return (
|
||||
latest_version_content
|
||||
if latest_version_content is not None
|
||||
else self.content
|
||||
)
|
||||
|
||||
@property
|
||||
def suggestion_content(self):
|
||||
"""
|
||||
@@ -373,15 +409,21 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
This improves processing speed for large documents while keeping
|
||||
enough context for accurate suggestions.
|
||||
"""
|
||||
if not self.content or len(self.content) <= 1200000:
|
||||
return self.content
|
||||
effective_content = self.get_effective_content()
|
||||
if not effective_content or len(effective_content) <= 1200000:
|
||||
return effective_content
|
||||
else:
|
||||
# Use 80% from the start and 20% from the end
|
||||
# to preserve both opening and closing context.
|
||||
head_len = 800000
|
||||
tail_len = 200000
|
||||
|
||||
return " ".join((self.content[:head_len], self.content[-tail_len:]))
|
||||
return " ".join(
|
||||
(
|
||||
effective_content[:head_len],
|
||||
effective_content[-tail_len:],
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def source_path(self) -> Path:
|
||||
|
||||
@@ -156,6 +156,46 @@ class TestDocument(TestCase):
|
||||
)
|
||||
self.assertEqual(doc.get_public_filename(), "2020-12-25 test")
|
||||
|
||||
def test_suggestion_content_uses_latest_version_content_for_root_documents(
|
||||
self,
|
||||
) -> None:
|
||||
root = Document.objects.create(
|
||||
title="root",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
content="outdated root content",
|
||||
)
|
||||
version = Document.objects.create(
|
||||
title="v1",
|
||||
checksum="v1",
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
content="latest version content",
|
||||
)
|
||||
|
||||
self.assertEqual(root.suggestion_content, version.content)
|
||||
|
||||
def test_content_length_is_per_document_row_for_versions(self) -> None:
|
||||
root = Document.objects.create(
|
||||
title="root",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
content="abc",
|
||||
)
|
||||
version = Document.objects.create(
|
||||
title="v1",
|
||||
checksum="v1",
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
content="abcdefgh",
|
||||
)
|
||||
|
||||
root.refresh_from_db()
|
||||
version.refresh_from_db()
|
||||
|
||||
self.assertEqual(root.content_length, 3)
|
||||
self.assertEqual(version.content_length, 8)
|
||||
|
||||
|
||||
def test_suggestion_content() -> None:
|
||||
"""
|
||||
|
||||
@@ -48,6 +48,52 @@ class _TestMatchingBase(TestCase):
|
||||
|
||||
|
||||
class TestMatching(_TestMatchingBase):
|
||||
def test_matches_uses_latest_version_content_for_root_documents(self) -> None:
|
||||
root = Document.objects.create(
|
||||
title="root",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
content="root content without token",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="v1",
|
||||
checksum="v1",
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
content="latest version contains keyword",
|
||||
)
|
||||
tag = Tag.objects.create(
|
||||
name="tag",
|
||||
match="keyword",
|
||||
matching_algorithm=Tag.MATCH_ANY,
|
||||
)
|
||||
|
||||
self.assertTrue(matching.matches(tag, root))
|
||||
|
||||
def test_matches_does_not_fall_back_to_root_content_when_version_exists(
|
||||
self,
|
||||
) -> None:
|
||||
root = Document.objects.create(
|
||||
title="root",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
content="root contains keyword",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="v1",
|
||||
checksum="v1",
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
content="latest version without token",
|
||||
)
|
||||
tag = Tag.objects.create(
|
||||
name="tag",
|
||||
match="keyword",
|
||||
matching_algorithm=Tag.MATCH_ANY,
|
||||
)
|
||||
|
||||
self.assertFalse(matching.matches(tag, root))
|
||||
|
||||
def test_match_none(self) -> None:
|
||||
self._test_matching(
|
||||
"",
|
||||
|
||||
@@ -6,8 +6,8 @@ SIDEBAR_VIEWS_VISIBLE_IDS_KEY = "sidebar_views_visible_ids"
|
||||
|
||||
|
||||
class TestMigrateSavedViewVisibilityToUiSettings(TestMigrations):
|
||||
migrate_from = "0013_document_root_document"
|
||||
migrate_to = "0015_savedview_visibility_to_ui_settings"
|
||||
migrate_from = "0013_alter_paperlesstask_task_name"
|
||||
migrate_to = "0014_savedview_visibility_to_ui_settings"
|
||||
|
||||
def setUpBeforeMigration(self, apps) -> None:
|
||||
User = apps.get_model("auth", "User")
|
||||
@@ -132,8 +132,8 @@ class TestMigrateSavedViewVisibilityToUiSettings(TestMigrations):
|
||||
|
||||
|
||||
class TestReverseMigrateSavedViewVisibilityFromUiSettings(TestMigrations):
|
||||
migrate_from = "0015_savedview_visibility_to_ui_settings"
|
||||
migrate_to = "0014_alter_paperlesstask_task_name"
|
||||
migrate_from = "0014_savedview_visibility_to_ui_settings"
|
||||
migrate_to = "0013_alter_paperlesstask_task_name"
|
||||
|
||||
def setUpBeforeMigration(self, apps) -> None:
|
||||
User = apps.get_model("auth", "User")
|
||||
|
||||
@@ -2,8 +2,8 @@ from documents.tests.utils import TestMigrations
|
||||
|
||||
|
||||
class TestMigrateShareLinkBundlePermissions(TestMigrations):
|
||||
migrate_from = "0007_document_content_length"
|
||||
migrate_to = "0008_sharelinkbundle"
|
||||
migrate_from = "0006_document_content_length"
|
||||
migrate_to = "0007_sharelinkbundle"
|
||||
|
||||
def setUpBeforeMigration(self, apps) -> None:
|
||||
User = apps.get_model("auth", "User")
|
||||
@@ -24,8 +24,8 @@ class TestMigrateShareLinkBundlePermissions(TestMigrations):
|
||||
|
||||
|
||||
class TestReverseMigrateShareLinkBundlePermissions(TestMigrations):
|
||||
migrate_from = "0008_sharelinkbundle"
|
||||
migrate_to = "0007_document_content_length"
|
||||
migrate_from = "0007_sharelinkbundle"
|
||||
migrate_to = "0006_document_content_length"
|
||||
|
||||
def setUpBeforeMigration(self, apps) -> None:
|
||||
User = apps.get_model("auth", "User")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.checks import Error
|
||||
from django.core.checks import Tags
|
||||
from django.core.checks import Warning
|
||||
from django.core.checks import register
|
||||
from django.db import connections
|
||||
@@ -204,15 +205,16 @@ def audit_log_check(app_configs, **kwargs):
|
||||
return result
|
||||
|
||||
|
||||
@register()
|
||||
@register(Tags.compatibility)
|
||||
def check_v3_minimum_upgrade_version(
|
||||
app_configs: object,
|
||||
**kwargs: object,
|
||||
) -> list[Error]:
|
||||
"""Enforce that upgrades to v3 must start from v2.20.9.
|
||||
"""
|
||||
Enforce that upgrades to v3 must start from v2.20.10.
|
||||
|
||||
v3 squashes all prior migrations into 0001_squashed and 0002_squashed.
|
||||
If a user skips v2.20.9, the data migration in 1075_workflowaction_order
|
||||
If a user skips v2.20.10, the data migration in 1075_workflowaction_order
|
||||
never runs and the squash may apply schema changes against an incomplete
|
||||
database state.
|
||||
"""
|
||||
@@ -239,7 +241,7 @@ def check_v3_minimum_upgrade_version(
|
||||
if {"0001_squashed", "0002_squashed"} & applied:
|
||||
return []
|
||||
|
||||
# On v2.20.9 exactly — squash will pick up cleanly from here
|
||||
# On v2.20.10 exactly — squash will pick up cleanly from here
|
||||
if "1075_workflowaction_order" in applied:
|
||||
return []
|
||||
|
||||
@@ -250,8 +252,8 @@ def check_v3_minimum_upgrade_version(
|
||||
Error(
|
||||
"Cannot upgrade to Paperless-ngx v3 from this version.",
|
||||
hint=(
|
||||
"Upgrading to v3 can only be performed from v2.20.9."
|
||||
"Please upgrade to v2.20.9, run migrations, then upgrade to v3."
|
||||
"Upgrading to v3 can only be performed from v2.20.10."
|
||||
"Please upgrade to v2.20.10, run migrations, then upgrade to v3."
|
||||
"See https://docs.paperless-ngx.com/setup/#upgrading for details."
|
||||
),
|
||||
id="paperless.E002",
|
||||
|
||||
@@ -209,7 +209,6 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
engine = get_choice_from_env(
|
||||
"PAPERLESS_DBENGINE",
|
||||
{"sqlite", "postgresql", "mariadb"},
|
||||
default="sqlite",
|
||||
)
|
||||
except ValueError:
|
||||
# MariaDB users already had to set PAPERLESS_DBENGINE, so it was picked up above
|
||||
|
||||
@@ -581,11 +581,11 @@ class TestV3MinimumUpgradeVersionCheck:
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- DB is on an old v2 version (pre-v2.20.9)
|
||||
- DB is on an old v2 version (pre-v2.20.10)
|
||||
WHEN:
|
||||
- The v3 upgrade check runs
|
||||
THEN:
|
||||
- The error hint explicitly references v2.20.9 so users know what to do
|
||||
- The error hint explicitly references v2.20.10 so users know what to do
|
||||
"""
|
||||
mocker.patch.dict(
|
||||
"paperless.checks.connections",
|
||||
@@ -593,7 +593,7 @@ class TestV3MinimumUpgradeVersionCheck:
|
||||
)
|
||||
result = check_v3_minimum_upgrade_version(None)
|
||||
assert len(result) == 1
|
||||
assert "v2.20.9" in result[0].hint
|
||||
assert "v2.20.10" in result[0].hint
|
||||
|
||||
def test_db_error_is_swallowed(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user