Bumps version to 1.16.5

Fix: owner removed when set_permissions passed on object create

docs/changelog.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## paperless-ngx 1.16.4
+
+### Bug Fixes
+
+- Fix: prevent button wrapping when sidebar narrows in MS Edge [@shamoon](https://github.com/shamoon) ([#3682](https://github.com/paperless-ngx/paperless-ngx/pull/3682))
+- Fix: Handling for filenames with non-ascii and no content attribute [@stumpylog](https://github.com/stumpylog) ([#3695](https://github.com/paperless-ngx/paperless-ngx/pull/3695))
+- Fix: Generation of thumbnails for existing stored emails [@stumpylog](https://github.com/stumpylog) ([#3696](https://github.com/paperless-ngx/paperless-ngx/pull/3696))
+- Fix: Use row gap for filter editor [@kleinweby](https://github.com/kleinweby) ([#3662](https://github.com/paperless-ngx/paperless-ngx/pull/3662))
+
+### Documentation
+
+- Documentation: update API docs re permissions [@shamoon](https://github.com/shamoon) ([#3697](https://github.com/paperless-ngx/paperless-ngx/pull/3697))
+
+### Maintenance
+
+- Chore: Updates codecov configuration for the flag settings and notification delay [@stumpylog](https://github.com/stumpylog) ([#3656](https://github.com/paperless-ngx/paperless-ngx/pull/3656))
+
+### All App Changes
+
+<details>
+<summary>4 changes</summary>
+
+- Fix: prevent button wrapping when sidebar narrows in MS Edge [@shamoon](https://github.com/shamoon) ([#3682](https://github.com/paperless-ngx/paperless-ngx/pull/3682))
+- Fix: Handling for filenames with non-ascii and no content attribute [@stumpylog](https://github.com/stumpylog) ([#3695](https://github.com/paperless-ngx/paperless-ngx/pull/3695))
+- Fix: Generation of thumbnails for existing stored emails [@stumpylog](https://github.com/stumpylog) ([#3696](https://github.com/paperless-ngx/paperless-ngx/pull/3696))
+- Fix: Use row gap for filter editor [@kleinweby](https://github.com/kleinweby) ([#3662](https://github.com/paperless-ngx/paperless-ngx/pull/3662))
+</details>
+
 ## paperless-ngx 1.16.3
 
 ### Bug Fixes

docs/configuration.md

@@ -1095,6 +1095,27 @@ barcode.
 
     Defaults to "ASN"
 
+`PAPERLESS_CONSUMER_BARCODE_UPSCALE=<float>`
+
+: Defines the upscale factor used in barcode detection.
+Improves the detection of small barcodes, i.e. with a value of 1.5 by
+upscaling the document beforce the detection process. Upscaling will
+only take place if value is bigger than 1.0. Otherwise upscaling will
+not be performed to save resources. Try using in combination with
+PAPERLESS_CONSUMER_BARCODE_DPI set to a value higher than default.
+
+    Defaults to 0.0
+
+`PAPERLESS_CONSUMER_BARCODE_DPI=<int>`
+
+: During barcode detection every page from a PDF document needs
+to be converted to an image. A dpi value can be specified in the
+conversion process. Default is 300. If the detection of small barcodes
+fails a bigger dpi value i.e. 600 can fix the issue. Try using in
+combination with PAPERLESS_CONSUMER_BARCODE_UPSCALE bigger than 1.0.
+
+    Defaults to "300"
+
 ## Binaries
 
 There are a few external software packages that Paperless expects to

paperless.conf.example

@@ -66,6 +66,8 @@
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
 #PAPERLESS_CONSUMER_ENABLE_BARCODES=false
 #PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
+#PAPERLESS_CONSUMER_BARCODE_UPSCALE=0.0
+#PAPERLESS_CONSUMER_BARCODE_DPI=300
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD

src-ui/src/environments/environment.prod.ts

@@ -5,7 +5,7 @@ export const environment = {
   apiBaseUrl: document.baseURI + 'api/',
   apiVersion: '3',
   appTitle: 'Paperless-ngx',
-  version: '1.16.4',
+  version: '1.16.5',
   webSocketHost: window.location.host,
   webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
   webSocketBaseUrl: base_url.pathname + 'ws/',

src/documents/barcodes.py

@@ -203,11 +203,21 @@ class BarcodeReader:
         try:
             pages_from_path = convert_from_path(
                 self.pdf_file,
-                dpi=300,
+                dpi=settings.CONSUMER_BARCODE_DPI,
                 output_folder=self.temp_dir.name,
             )
 
             for current_page_number, page in enumerate(pages_from_path):
+                factor = settings.CONSUMER_BARCODE_UPSCALE
+                if factor > 1.0:
+                    logger.debug(
+                        f"Upscaling image by {factor} for better barcode detection",
+                    )
+                    x, y = page.size
+                    page = page.resize(
+                        (int(round(x * factor)), (int(round(y * factor)))),
+                    )
+
                 for barcode_value in reader(page):
                     self.barcodes.append(
                         Barcode(current_page_number, barcode_value),

src/documents/serialisers.py

@@ -213,15 +213,12 @@ class OwnedObjectSerializer(serializers.ModelSerializer, SetPermissionsMixin):
     # other methods in mixin
 
     def create(self, validated_data):
-        if self.user and (
-            "owner" not in validated_data or validated_data["owner"] is None
-        ):
+        # default to current user if not set
+        if "owner" not in validated_data and self.user:
             validated_data["owner"] = self.user
         permissions = None
         if "set_permissions" in validated_data:
             permissions = validated_data.pop("set_permissions")
-            if "user" not in permissions or permissions["user"] is None:
-                validated_data["owner"] = None
         instance = super().create(validated_data)
         if permissions is not None:
             self._set_permissions(permissions, instance)

src/documents/tests/test_api.py

@@ -28,6 +28,7 @@ from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
 from guardian.shortcuts import assign_perm
+from guardian.shortcuts import get_perms
 from rest_framework import status
 from rest_framework.test import APITestCase
 from whoosh.writing import AsyncWriter
@@ -3855,7 +3856,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             status.HTTP_200_OK,
         )
 
-    def test_object_permissions(self):
+    def test_api_get_object_permissions(self):
         user1 = User.objects.create_user(username="test1")
         user2 = User.objects.create_user(username="test2")
         user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
@@ -3886,18 +3887,16 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             status.HTTP_404_NOT_FOUND,
         )
 
-    def test_api_set_permissions(self):
+    def test_api_default_owner(self):
         """
         GIVEN:
-            - API request to create an object (Tag) that supplies set_permissions object
+            - API request to create an object (Tag)
         WHEN:
-            - owner is passed as null or as a user id
-            - view > users is set
+            - owner is not set at all
         THEN:
-            - Object permissions are set appropriately
+            - Object created with current user as owner
         """
         user1 = User.objects.create_superuser(username="user1")
-        user2 = User.objects.create(username="user2")
 
         self.client.force_authenticate(user1)
 
@@ -3907,11 +3906,73 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
                 {
                     "name": "test1",
                     "matching_algorithm": MatchingModel.MATCH_AUTO,
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        tag1 = Tag.objects.filter(name="test1").first()
+        self.assertEqual(tag1.owner, user1)
+
+    def test_api_set_no_owner(self):
+        """
+        GIVEN:
+            - API request to create an object (Tag)
+        WHEN:
+            - owner is passed as None
+        THEN:
+            - Object created with no owner
+        """
+        user1 = User.objects.create_superuser(username="user1")
+
+        self.client.force_authenticate(user1)
+
+        response = self.client.post(
+            "/api/tags/",
+            json.dumps(
+                {
+                    "name": "test1",
+                    "matching_algorithm": MatchingModel.MATCH_AUTO,
+                    "owner": None,
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        tag1 = Tag.objects.filter(name="test1").first()
+        self.assertEqual(tag1.owner, None)
+
+    def test_api_set_owner_w_permissions(self):
+        """
+        GIVEN:
+            - API request to create an object (Tag) that supplies set_permissions object
+        WHEN:
+            - owner is passed as user id
+            - view > users is set & view > groups is set
+        THEN:
+            - Object permissions are set appropriately
+        """
+        user1 = User.objects.create_superuser(username="user1")
+        user2 = User.objects.create(username="user2")
+        group1 = Group.objects.create(name="group1")
+
+        self.client.force_authenticate(user1)
+
+        response = self.client.post(
+            "/api/tags/",
+            json.dumps(
+                {
+                    "name": "test1",
+                    "matching_algorithm": MatchingModel.MATCH_AUTO,
+                    "owner": user1.id,
                     "set_permissions": {
-                        "owner": None,
                         "view": {
-                            "users": None,
-                            "groups": None,
+                            "users": [user2.id],
+                            "groups": [group1.id],
                         },
                         "change": {
                             "users": None,
@@ -3926,19 +3987,43 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
 
         tag1 = Tag.objects.filter(name="test1").first()
-        self.assertEqual(tag1.owner, None)
 
-        response = self.client.post(
-            "/api/tags/",
+        from guardian.core import ObjectPermissionChecker
+
+        checker = ObjectPermissionChecker(user2)
+        self.assertEqual(checker.has_perm("view_tag", tag1), True)
+        self.assertIn("view_tag", get_perms(group1, tag1))
+
+    def test_api_set_doc_permissions(self):
+        """
+        GIVEN:
+            - API request to update doc permissions and owner
+        WHEN:
+            - owner is set
+            - view > users is set & view > groups is set
+        THEN:
+            - Object permissions are set appropriately
+        """
+        doc = Document.objects.create(
+            title="test",
+            mime_type="application/pdf",
+            content="this is a document",
+        )
+        user1 = User.objects.create_superuser(username="user1")
+        user2 = User.objects.create(username="user2")
+        group1 = Group.objects.create(name="group1")
+
+        self.client.force_authenticate(user1)
+
+        response = self.client.patch(
+            f"/api/documents/{doc.id}/",
             json.dumps(
                 {
-                    "name": "test2",
-                    "matching_algorithm": MatchingModel.MATCH_AUTO,
+                    "owner": user1.id,
                     "set_permissions": {
-                        "owner": user1.id,
                         "view": {
                             "users": [user2.id],
-                            "groups": None,
+                            "groups": [group1.id],
                         },
                         "change": {
                             "users": None,
@@ -3950,12 +4035,15 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             content_type="application/json",
         )
 
-        tag2 = Tag.objects.filter(name="test2").first()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        doc = Document.objects.get(pk=doc.id)
 
+        self.assertEqual(doc.owner, user1)
         from guardian.core import ObjectPermissionChecker
 
         checker = ObjectPermissionChecker(user2)
-        self.assertEqual(checker.has_perm("view_tag", tag2), True)
+        self.assertTrue(checker.has_perm("view_document", doc))
+        self.assertIn("view_document", get_perms(group1, doc))
 
     def test_dynamic_permissions_fields(self):
         user1 = User.objects.create_user(username="user1")

src/documents/tests/test_barcodes.py

@@ -906,6 +906,47 @@ class TestAsnBarcode(DirectoriesMixin, TestCase):
                 input_doc,
             )
 
+    @override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR")
+    def test_scan_file_for_qrcode_without_upscale(self):
+        """
+        GIVEN:
+            - A printed and scanned PDF document with a rather small QR code
+        WHEN:
+            - ASN barcode detection is run with default settings
+            - pyzbar is used for detection, as zxing would behave differently, and detect the QR code
+        THEN:
+            - ASN is not detected
+        """
+
+        test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf"
+
+        with BarcodeReader(test_file, "application/pdf") as reader:
+            reader.detect()
+            self.assertEqual(len(reader.barcodes), 0)
+
+    @override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR")
+    @override_settings(CONSUMER_BARCODE_DPI=600)
+    @override_settings(CONSUMER_BARCODE_UPSCALE=1.5)
+    def test_scan_file_for_qrcode_with_upscale(self):
+        """
+        GIVEN:
+            - A printed and scanned PDF document with a rather small QR code
+        WHEN:
+            - ASN barcode detection is run with 600dpi and an upscale factor of 1.5 and pyzbar
+            - pyzbar is used for detection, as zxing would behave differently.
+              Upscaling is a workaround for detection problems with pyzbar,
+              when you cannot switch to zxing (aarch64 build problems of zxing)
+        THEN:
+            - ASN 123 is detected
+        """
+
+        test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf"
+
+        with BarcodeReader(test_file, "application/pdf") as reader:
+            reader.detect()
+            self.assertEqual(len(reader.barcodes), 1)
+            self.assertEqual(reader.asn, 123)
+
 
 @pytest.mark.skipif(
     not HAS_ZXING_LIB,

src/paperless/settings.py

@@ -781,6 +781,16 @@ CONSUMER_ASN_BARCODE_PREFIX: Final[str] = os.getenv(
 )
 
 
+CONSUMER_BARCODE_UPSCALE: Final[float] = float(
+    os.getenv("PAPERLESS_CONSUMER_BARCODE_UPSCALE", 0.0),
+)
+
+
+CONSUMER_BARCODE_DPI: Final[str] = int(
+    os.getenv("PAPERLESS_CONSUMER_BARCODE_DPI", 300),
+)
+
+
 OCR_PAGES = int(os.getenv("PAPERLESS_OCR_PAGES", 0))
 
 # The default language that tesseract will attempt to use when parsing

src/paperless/version.py

@@ -1,7 +1,7 @@
 from typing import Final
 from typing import Tuple
 
-__version__: Final[Tuple[int, int, int]] = (1, 16, 4)
+__version__: Final[Tuple[int, int, int]] = (1, 16, 5)
 # Version string like X.Y.Z
 __full_version_str__: Final[str] = ".".join(map(str, __version__))
 # Version string like X.Y