mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-08 20:03:18 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cfd5869f82 | ||
|
|
b0e0e8a353 | ||
|
|
fc242bb570 | ||
|
|
b192a419fd | ||
|
|
3986150f95 | ||
|
|
ee5588ade3 | ||
|
|
2635a12281 | ||
|
|
1f396e51f3 | ||
|
|
3eb784b34d | ||
|
|
cb03a0b33e | ||
|
|
a96d0b15b8 | ||
|
|
376b61938f | ||
|
|
d1f5eb0335 | ||
|
|
d283205f57 | ||
|
|
42908ad2b9 | ||
|
|
db6842e710 | ||
|
|
d5f8cd59fb | ||
|
|
91d6741b4f | ||
|
|
972758fc72 | ||
|
|
15d9829b6d | ||
|
|
5ad34dfe03 | ||
|
|
52a0484f74 | ||
|
|
71e2f86f70 | ||
|
|
1824e5fafd | ||
|
|
7b9e56ef22 | ||
|
|
2b9bed749d | ||
|
|
ef49414162 | ||
|
|
a488ba6f90 | ||
|
|
47727de116 | ||
|
|
e86adb6c36 | ||
|
|
74aa10f31f | ||
|
|
2ea4a792fe | ||
|
|
3f5c770af4 | ||
|
|
765313926f | ||
|
|
1f1725ba56 | ||
|
|
c6cecd3c4e | ||
|
|
5e54259db9 | ||
|
|
3a484a3ff2 | ||
|
|
92a1b24583 | ||
|
|
8709dd807b | ||
|
|
5a5a0ffd21 | ||
|
|
d7aa0cb862 | ||
|
|
eafb494066 | ||
|
|
4ca1863dee | ||
|
|
d5d22c2c0f | ||
|
|
991d99bb7c | ||
|
|
30f11814ff | ||
|
|
0ca5c66ca7 | ||
|
|
dd99c2289d | ||
|
|
23c11f49d0 | ||
|
|
cfa1d3b058 | ||
|
|
68bd8f8f63 | ||
|
|
41d5d86637 | ||
|
|
2a50425902 | ||
|
|
b67ac5a7d7 | ||
|
|
b48c434266 | ||
|
|
92baa89638 | ||
|
|
4c7b3dd307 | ||
|
|
fe48752a6f | ||
|
|
4692de5595 | ||
|
|
141d74464b | ||
|
|
c05d069e97 | ||
|
|
f5e47a86c3 | ||
|
|
fe196eaf0e | ||
|
|
7fda0e877b | ||
|
|
5b8f4db1ca | ||
|
|
08ab98f3fc |
+37
-35
@@ -1,49 +1,51 @@
|
||||
categories:
|
||||
- type: 'pre-include'
|
||||
when:
|
||||
- label: 'enhancement'
|
||||
- label: 'bug'
|
||||
- label: 'chore'
|
||||
- label: 'deployment'
|
||||
- label: 'translation'
|
||||
- label: 'dependencies'
|
||||
- label: 'documentation'
|
||||
- label: 'frontend'
|
||||
- label: 'backend'
|
||||
- label: 'ci-cd'
|
||||
- label: 'breaking-change'
|
||||
- label: 'notable'
|
||||
- type: 'pre-exclude'
|
||||
when:
|
||||
- label: 'skip-changelog'
|
||||
- title: 'Breaking Changes'
|
||||
labels:
|
||||
- 'breaking-change'
|
||||
when:
|
||||
- label: 'breaking-change'
|
||||
- title: 'Notable Changes'
|
||||
labels:
|
||||
- 'notable'
|
||||
when:
|
||||
- label: 'notable'
|
||||
- title: 'Features / Enhancements'
|
||||
labels:
|
||||
- 'enhancement'
|
||||
when:
|
||||
- label: 'enhancement'
|
||||
- title: 'Bug Fixes'
|
||||
labels:
|
||||
- 'bug'
|
||||
when:
|
||||
- label: 'bug'
|
||||
- title: 'Documentation'
|
||||
labels:
|
||||
- 'documentation'
|
||||
when:
|
||||
- label: 'documentation'
|
||||
- title: 'Maintenance'
|
||||
labels:
|
||||
- 'chore'
|
||||
- 'deployment'
|
||||
- 'translation'
|
||||
- 'ci-cd'
|
||||
when:
|
||||
- label: 'chore'
|
||||
- label: 'deployment'
|
||||
- label: 'translation'
|
||||
- label: 'ci-cd'
|
||||
- title: 'Dependencies'
|
||||
collapse-after: 3
|
||||
labels:
|
||||
- 'dependencies'
|
||||
when:
|
||||
- label: 'dependencies'
|
||||
- title: 'All App Changes'
|
||||
labels:
|
||||
- 'frontend'
|
||||
- 'backend'
|
||||
collapse-after: 1
|
||||
include-labels:
|
||||
- 'enhancement'
|
||||
- 'bug'
|
||||
- 'chore'
|
||||
- 'deployment'
|
||||
- 'translation'
|
||||
- 'dependencies'
|
||||
- 'documentation'
|
||||
- 'frontend'
|
||||
- 'backend'
|
||||
- 'ci-cd'
|
||||
- 'breaking-change'
|
||||
- 'notable'
|
||||
exclude-labels:
|
||||
- 'skip-changelog'
|
||||
when:
|
||||
- label: 'frontend'
|
||||
- label: 'backend'
|
||||
filter-by-commitish: true
|
||||
category-template: '### $TITLE'
|
||||
change-template: '- $TITLE @$AUTHOR ([#$NUMBER]($URL))'
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
backend_changed: ${{ steps.force.outputs.run_all == 'true' || steps.filter.outputs.backend == 'true' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
- name: Detect changes
|
||||
id: filter
|
||||
if: steps.force.outputs.run_all != 'true'
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
base: ${{ steps.range.outputs.base }}
|
||||
ref: ${{ steps.range.outputs.ref }}
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
fail-fast: false
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Start containers
|
||||
@@ -96,11 +96,11 @@ jobs:
|
||||
docker compose --file docker/compose/docker-compose.ci-test.yml up --detach
|
||||
- name: Set up Python
|
||||
id: setup-python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "${{ matrix.python-version }}"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -169,16 +169,16 @@ jobs:
|
||||
PAPERLESS_SECRET_KEY: "ci-typing-not-a-real-secret"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Python
|
||||
id: setup-python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "${{ env.DEFAULT_PYTHON }}"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
ref-name: ${{ steps.ref.outputs.name }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Determine ref name
|
||||
@@ -106,9 +106,9 @@ jobs:
|
||||
echo "repository=${repo_name}"
|
||||
echo "name=${repo_name}" >> $GITHUB_OUTPUT
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
- name: Docker metadata
|
||||
id: docker-meta
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ steps.repo.outputs.name }}
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
@@ -182,29 +182,29 @@ jobs:
|
||||
echo "Downloaded digests:"
|
||||
ls -la /tmp/digests/
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Login to Docker Hub
|
||||
if: needs.build-arch.outputs.push-external == 'true'
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Login to Quay.io
|
||||
if: needs.build-arch.outputs.push-external == 'true'
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: quay.io
|
||||
username: ${{ secrets.QUAY_USERNAME }}
|
||||
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
|
||||
- name: Docker metadata
|
||||
id: docker-meta
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ needs.build-arch.outputs.repository }}
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
docs_changed: ${{ steps.force.outputs.run_all == 'true' || steps.filter.outputs.docs == 'true' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
- name: Detect changes
|
||||
id: filter
|
||||
if: steps.force.outputs.run_all != 'true'
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
base: ${{ steps.range.outputs.base }}
|
||||
ref: ${{ steps.range.outputs.ref }}
|
||||
@@ -69,16 +69,16 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Python
|
||||
id: setup-python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
frontend_changed: ${{ steps.force.outputs.run_all == 'true' || steps.filter.outputs.frontend == 'true' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
- name: Detect changes
|
||||
id: filter
|
||||
if: steps.force.outputs.run_all != 'true'
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
base: ${{ steps.range.outputs.base }}
|
||||
ref: ${{ steps.range.outputs.ref }}
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
cache: 'pnpm'
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
cache: 'pnpm'
|
||||
@@ -129,8 +129,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Run lint
|
||||
run: cd src-ui && pnpm run lint
|
||||
unit-tests:
|
||||
@@ -148,7 +148,7 @@ jobs:
|
||||
shard-count: [4]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
@@ -156,7 +156,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
cache: 'pnpm'
|
||||
@@ -168,8 +168,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Run Jest unit tests
|
||||
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
- name: Upload test results to Codecov
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
container: mcr.microsoft.com/playwright:v1.61.1-noble
|
||||
container: mcr.microsoft.com/playwright:v1.62.0-noble
|
||||
env:
|
||||
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
|
||||
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
|
||||
@@ -203,7 +203,7 @@ jobs:
|
||||
shard-count: [2]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
@@ -211,7 +211,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
cache: 'pnpm'
|
||||
@@ -223,23 +223,20 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --no-frozen-lockfile
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Run Playwright E2E tests
|
||||
run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
bundle-analysis:
|
||||
name: Bundle Analysis
|
||||
frontend-build:
|
||||
name: Frontend Build
|
||||
needs: [changes, unit-tests, e2e-tests]
|
||||
if: needs.changes.outputs.frontend_changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
environment: bundle-analysis
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
@@ -248,7 +245,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
cache: 'pnpm'
|
||||
@@ -260,21 +257,19 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Build and analyze
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Build
|
||||
run: cd src-ui && pnpm run build --configuration=production
|
||||
gate:
|
||||
name: Frontend CI Gate
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, bundle-analysis]
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, frontend-build]
|
||||
if: always()
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Check gate
|
||||
env:
|
||||
BUNDLE_ANALYSIS_RESULT: ${{ needs['bundle-analysis'].result }}
|
||||
BUILD_RESULT: ${{ needs['frontend-build'].result }}
|
||||
E2E_RESULT: ${{ needs['e2e-tests'].result }}
|
||||
FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }}
|
||||
INSTALL_RESULT: ${{ needs['install-dependencies'].result }}
|
||||
@@ -306,8 +301,8 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${BUNDLE_ANALYSIS_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend bundle-analysis job result: ${BUNDLE_ANALYSIS_RESULT}"
|
||||
if [[ "${BUILD_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend build job result: ${BUILD_RESULT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -17,12 +17,12 @@ jobs:
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- name: Run prek
|
||||
uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4
|
||||
uses: j178/prek-action@5337cb91e0fa35a7ff31b9ca345126d8bbbcdf16 # v2.0.6
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Wait for Docker build
|
||||
uses: lewagon/wait-on-check-action@96d9100b431964d10e0136aff8b9ccb92470505e # v1.8.0
|
||||
uses: lewagon/wait-on-check-action@2271c86c146b96545b4e871b855e10ffa6f50773 # v1.9.0
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
check-name: 'Merge and Push Manifest'
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# ---- Frontend Build ----
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
package-manager-cache: false
|
||||
@@ -55,11 +55,11 @@ jobs:
|
||||
# ---- Backend Setup ----
|
||||
- name: Set up Python
|
||||
id: setup-python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: false
|
||||
@@ -171,7 +171,7 @@ jobs:
|
||||
fi
|
||||
- name: Create release and changelog
|
||||
id: create-release
|
||||
uses: release-drafter/release-drafter@4d75298e00d9e34c483e5ff8c68d0ea1c1940c1e # v7.5.1
|
||||
uses: release-drafter/release-drafter@eada3c96a64734dd381cfbda23511034e328ddb0 # v7.6.0
|
||||
with:
|
||||
name: Paperless-ngx ${{ steps.get-version.outputs.version }}
|
||||
tag: ${{ steps.get-version.outputs.version }}
|
||||
@@ -202,17 +202,17 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: main
|
||||
persist-credentials: true # for pushing changelog branch
|
||||
- name: Set up Python
|
||||
id: setup-python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: false
|
||||
|
||||
@@ -22,11 +22,11 @@ jobs:
|
||||
security-events: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7
|
||||
uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1
|
||||
semgrep:
|
||||
name: Semgrep CE
|
||||
runs-on: ubuntu-24.04
|
||||
@@ -38,13 +38,13 @@ jobs:
|
||||
security-events: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Run Semgrep
|
||||
run: semgrep scan --config auto --sarif-output results.sarif
|
||||
- name: Upload results to GitHub code scanning
|
||||
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -34,12 +34,12 @@ jobs:
|
||||
# Learn more about CodeQL language support at https://git.io/codeql-language-support
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -47,4 +47,4 @@ jobs:
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
|
||||
@@ -17,12 +17,12 @@ jobs:
|
||||
environment: translation-sync
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
token: ${{ secrets.PNGX_BOT_PAT }}
|
||||
persist-credentials: false
|
||||
- name: crowdin action
|
||||
uses: crowdin/github-action@52aa776766211d83d975df51f3b9c53c2f8ba35f # v2.16.3
|
||||
uses: crowdin/github-action@c7af9bc98b01694653031fef2a0dc6c7888ce9bc # v2.17.0
|
||||
with:
|
||||
upload_translations: false
|
||||
download_translations: true
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
steps:
|
||||
- name: Label PR by file path or branch name
|
||||
# see .github/labeler.yml for the labeler config
|
||||
uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0
|
||||
uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Label by size
|
||||
|
||||
@@ -19,6 +19,6 @@ jobs:
|
||||
if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login != 'dependabot'
|
||||
steps:
|
||||
- name: Label PR with release-drafter
|
||||
uses: release-drafter/release-drafter@4d75298e00d9e34c483e5ff8c68d0ea1c1940c1e # v7.5.1
|
||||
uses: release-drafter/release-drafter@eada3c96a64734dd381cfbda23511034e328ddb0 # v7.6.0
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
|
||||
- uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
|
||||
with:
|
||||
days-before-stale: 7
|
||||
days-before-close: 14
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
env:
|
||||
GH_REF: ${{ github.ref }} # sonar rule:githubactions:S7630 - avoid injection
|
||||
with:
|
||||
@@ -23,13 +23,13 @@ jobs:
|
||||
persist-credentials: true # for pushing translation branch
|
||||
- name: Set up Python
|
||||
id: setup-python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -qq --no-install-recommends gettext
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24.x
|
||||
cache: 'pnpm'
|
||||
@@ -61,16 +61,13 @@ jobs:
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install frontend dependencies
|
||||
if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
|
||||
run: cd src-ui && pnpm install
|
||||
- name: Re-link Angular cli
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Generate frontend translation strings
|
||||
run: |
|
||||
cd src-ui
|
||||
pnpm run ng extract-i18n
|
||||
- name: Commit changes
|
||||
uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0
|
||||
uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0
|
||||
with:
|
||||
file_pattern: 'src-ui/messages.xlf src/locale/en_US/LC_MESSAGES/django.po'
|
||||
commit_message: "Auto translate strings"
|
||||
|
||||
@@ -29,7 +29,7 @@ repos:
|
||||
- id: check-case-conflict
|
||||
- id: detect-private-key
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.2
|
||||
rev: v2.4.3
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: [tomli]
|
||||
@@ -38,7 +38,7 @@ repos:
|
||||
- json
|
||||
# See https://github.com/prettier/prettier/issues/15742 for the fork reason
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: 'v3.9.4'
|
||||
rev: 'v3.9.6'
|
||||
hooks:
|
||||
- id: prettier
|
||||
types_or:
|
||||
@@ -46,16 +46,16 @@ repos:
|
||||
- ts
|
||||
- markdown
|
||||
additional_dependencies:
|
||||
- prettier@3.9.4
|
||||
- prettier@3.9.6
|
||||
- 'prettier-plugin-organize-imports@4.3.0'
|
||||
# Python hooks
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.20
|
||||
rev: v0.16.1
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
- id: ruff-format
|
||||
- repo: https://github.com/tox-dev/pyproject-fmt
|
||||
rev: "v2.25.1"
|
||||
rev: "v2.26.0"
|
||||
hooks:
|
||||
- id: pyproject-fmt
|
||||
additional_dependencies: [tomli]
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ RUN set -eux \
|
||||
# Purpose: Installs s6-overlay and rootfs
|
||||
# Comments:
|
||||
# - Don't leave anything extra in here either
|
||||
FROM ghcr.io/astral-sh/uv:0.11.28-python3.12-trixie-slim AS s6-overlay-base
|
||||
FROM ghcr.io/astral-sh/uv:0.11.32-python3.12-trixie-slim AS s6-overlay-base
|
||||
|
||||
WORKDIR /usr/src/s6
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ services:
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
greenmail:
|
||||
image: docker.io/greenmail/standalone:2.1.9
|
||||
image: docker.io/greenmail/standalone:2.1.11
|
||||
hostname: greenmail
|
||||
container_name: greenmail
|
||||
environment:
|
||||
@@ -35,7 +35,7 @@ services:
|
||||
- "3143:3143" # IMAP
|
||||
restart: unless-stopped
|
||||
nginx:
|
||||
image: docker.io/nginx:1.31.2-alpine
|
||||
image: docker.io/nginx:1.31.3-alpine
|
||||
hostname: nginx
|
||||
container_name: nginx
|
||||
ports:
|
||||
|
||||
@@ -1,5 +1,65 @@
|
||||
# Changelog
|
||||
|
||||
## paperless-ngx 3.0.5
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix: accept Whoosh-era abbreviated relative-date units (yrs, mos, wks, etc) in search queries [@stumpylog](https://github.com/stumpylog) ([#13486](https://github.com/paperless-ngx/paperless-ngx/pull/13486))
|
||||
- Fix: fix edit dialog error change detection [@shamoon](https://github.com/shamoon) ([#13483](https://github.com/paperless-ngx/paperless-ngx/pull/13483))
|
||||
- Fix: key the AI suggestion cache by model and endpoint [@lunetics](https://github.com/lunetics) ([#13449](https://github.com/paperless-ngx/paperless-ngx/pull/13449))
|
||||
- Fix: validate custom field values in bulk operations [@shamoon](https://github.com/shamoon) ([#13457](https://github.com/paperless-ngx/paperless-ngx/pull/13457))
|
||||
- Fixhancement: better handle empty fields from AI suggestions [@shamoon](https://github.com/shamoon) ([#13454](https://github.com/paperless-ngx/paperless-ngx/pull/13454))
|
||||
- Fix: normalize monetary decimal symbol by locale [@shamoon](https://github.com/shamoon) ([#13427](https://github.com/paperless-ngx/paperless-ngx/pull/13427))
|
||||
- Fix: fold overflowing path text in sanity checker [@stumpylog](https://github.com/stumpylog) ([#13426](https://github.com/paperless-ngx/paperless-ngx/pull/13426))
|
||||
- Fix: correct delayed add field button for custom fields [@shamoon](https://github.com/shamoon) ([#13424](https://github.com/paperless-ngx/paperless-ngx/pull/13424))
|
||||
- Fix: hide attributes collapse / show button without UI settings [@shamoon](https://github.com/shamoon) ([#13425](https://github.com/paperless-ngx/paperless-ngx/pull/13425))
|
||||
- Fix: consolidate born-digital PDF detection between archive decision and OCR [@stumpylog](https://github.com/stumpylog) ([#13409](https://github.com/paperless-ngx/paperless-ngx/pull/13409))
|
||||
- Fix: prevent scale vs page loop in pngx PDF viewer [@shamoon](https://github.com/shamoon) ([#13406](https://github.com/paperless-ngx/paperless-ngx/pull/13406))
|
||||
- Fix: handle hidden line breaks in email subjects when sending email [@shamoon](https://github.com/shamoon) ([#13402](https://github.com/paperless-ngx/paperless-ngx/pull/13402))
|
||||
- Fix: avoid NotSupportedError from document\_importer on MariaDB [@stumpylog](https://github.com/stumpylog) ([#13400](https://github.com/paperless-ngx/paperless-ngx/pull/13400))
|
||||
- Fix: exclude next-period start from relative date-range filters [@stumpylog](https://github.com/stumpylog) ([#13381](https://github.com/paperless-ngx/paperless-ngx/pull/13381))
|
||||
- Fix: close non-atomic db connections in before\_task\_publish [@shamoon](https://github.com/shamoon) ([#13366](https://github.com/paperless-ngx/paperless-ngx/pull/13366))
|
||||
- Chore/fix: refactor frontend task service [@shamoon](https://github.com/shamoon) ([#13365](https://github.com/paperless-ngx/paperless-ngx/pull/13365))
|
||||
- Fix: fix Enter selection in search autocomplete [@shamoon](https://github.com/shamoon) ([#13361](https://github.com/paperless-ngx/paperless-ngx/pull/13361))
|
||||
|
||||
### Documentation
|
||||
|
||||
- Documentation: PAPERLESS\_CONSUMER\_IGNORE\_PATTERNS clarifications [@shamoon](https://github.com/shamoon) ([#13489](https://github.com/paperless-ngx/paperless-ngx/pull/13489))
|
||||
- Documentation: Add Password Removal workflow action documentation [@stumpylog](https://github.com/stumpylog) ([#13377](https://github.com/paperless-ngx/paperless-ngx/pull/13377))
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Chore(deps): Bump pymdown-extensions from 10.21.3 to 11.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#13378](https://github.com/paperless-ngx/paperless-ngx/pull/13378))
|
||||
|
||||
### All App Changes
|
||||
|
||||
<details>
|
||||
<summary>21 changes</summary>
|
||||
|
||||
- Fix: accept Whoosh-era abbreviated relative-date units (yrs, mos, wks, etc) in search queries [@stumpylog](https://github.com/stumpylog) ([#13486](https://github.com/paperless-ngx/paperless-ngx/pull/13486))
|
||||
- Fix: fix edit dialog error change detection [@shamoon](https://github.com/shamoon) ([#13483](https://github.com/paperless-ngx/paperless-ngx/pull/13483))
|
||||
- Performance: sqlite-vec point-delete for document chunks [@stumpylog](https://github.com/stumpylog) ([#13438](https://github.com/paperless-ngx/paperless-ngx/pull/13438))
|
||||
- Fix: key the AI suggestion cache by model and endpoint [@lunetics](https://github.com/lunetics) ([#13449](https://github.com/paperless-ngx/paperless-ngx/pull/13449))
|
||||
- Fix: validate custom field values in bulk operations [@shamoon](https://github.com/shamoon) ([#13457](https://github.com/paperless-ngx/paperless-ngx/pull/13457))
|
||||
- Fixhancement: better handle empty fields from AI suggestions [@shamoon](https://github.com/shamoon) ([#13454](https://github.com/paperless-ngx/paperless-ngx/pull/13454))
|
||||
- Performance: Use server side iterators during LLM index updating [@stumpylog](https://github.com/stumpylog) ([#13430](https://github.com/paperless-ngx/paperless-ngx/pull/13430))
|
||||
- Fix: normalize monetary decimal symbol by locale [@shamoon](https://github.com/shamoon) ([#13427](https://github.com/paperless-ngx/paperless-ngx/pull/13427))
|
||||
- Fix: fold overflowing path text in sanity checker [@stumpylog](https://github.com/stumpylog) ([#13426](https://github.com/paperless-ngx/paperless-ngx/pull/13426))
|
||||
- Fix: correct delayed add field button for custom fields [@shamoon](https://github.com/shamoon) ([#13424](https://github.com/paperless-ngx/paperless-ngx/pull/13424))
|
||||
- Fix: hide attributes collapse / show button without UI settings [@shamoon](https://github.com/shamoon) ([#13425](https://github.com/paperless-ngx/paperless-ngx/pull/13425))
|
||||
- Fix: consolidate born-digital PDF detection between archive decision and OCR [@stumpylog](https://github.com/stumpylog) ([#13409](https://github.com/paperless-ngx/paperless-ngx/pull/13409))
|
||||
- Fix: prevent scale vs page loop in pngx PDF viewer [@shamoon](https://github.com/shamoon) ([#13406](https://github.com/paperless-ngx/paperless-ngx/pull/13406))
|
||||
- Fix: handle hidden line breaks in email subjects when sending email [@shamoon](https://github.com/shamoon) ([#13402](https://github.com/paperless-ngx/paperless-ngx/pull/13402))
|
||||
- Fix: avoid NotSupportedError from document\_importer on MariaDB [@stumpylog](https://github.com/stumpylog) ([#13400](https://github.com/paperless-ngx/paperless-ngx/pull/13400))
|
||||
- Tweak: adjust doc details button toolbar flow [@shamoon](https://github.com/shamoon) ([#13382](https://github.com/paperless-ngx/paperless-ngx/pull/13382))
|
||||
- Fix: exclude next-period start from relative date-range filters [@stumpylog](https://github.com/stumpylog) ([#13381](https://github.com/paperless-ngx/paperless-ngx/pull/13381))
|
||||
- Chore(deps): Bump pymdown-extensions from 10.21.3 to 11.0 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#13378](https://github.com/paperless-ngx/paperless-ngx/pull/13378))
|
||||
- Fix: close non-atomic db connections in before\_task\_publish [@shamoon](https://github.com/shamoon) ([#13366](https://github.com/paperless-ngx/paperless-ngx/pull/13366))
|
||||
- Chore/fix: refactor frontend task service [@shamoon](https://github.com/shamoon) ([#13365](https://github.com/paperless-ngx/paperless-ngx/pull/13365))
|
||||
- Fix: fix Enter selection in search autocomplete [@shamoon](https://github.com/shamoon) ([#13361](https://github.com/paperless-ngx/paperless-ngx/pull/13361))
|
||||
|
||||
</details>
|
||||
|
||||
## paperless-ngx 3.0.4
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
+24
-24
@@ -21,7 +21,7 @@ dependencies = [
|
||||
"channels~=4.2",
|
||||
"channels-redis~=4.2",
|
||||
"concurrent-log-handler~=0.9.25",
|
||||
"dateparser~=1.2",
|
||||
"dateparser~=1.4",
|
||||
# WARNING: django does not use semver.
|
||||
# Only patch versions are guaranteed to not introduce breaking changes.
|
||||
"django~=5.2.13",
|
||||
@@ -32,33 +32,32 @@ dependencies = [
|
||||
"django-cors-headers~=4.9.0",
|
||||
"django-extensions~=4.1",
|
||||
"django-filter~=25.1",
|
||||
"django-guardian~=3.3.0",
|
||||
"django-guardian~=3.3.3",
|
||||
"django-multiselectfield~=1.0.1",
|
||||
"django-rich~=2.2.0",
|
||||
"django-soft-delete~=1.0.18",
|
||||
"django-treenode>=0.24",
|
||||
"djangorestframework~=3.16",
|
||||
"djangorestframework-guardian~=0.4.0",
|
||||
"drf-spectacular~=0.28",
|
||||
"drf-spectacular-sidecar~=2026.5.1",
|
||||
"drf-spectacular~=0.30",
|
||||
"drf-spectacular-sidecar~=2026.7.1",
|
||||
"drf-writable-nested~=0.7.1",
|
||||
"filelock~=3.29.0",
|
||||
"filelock~=3.32.0",
|
||||
"flower~=2.0.1",
|
||||
"gotenberg-client~=0.14.0",
|
||||
"httpx-oauth~=0.16",
|
||||
"ijson>=3.2",
|
||||
"imap-tools~=1.13.0",
|
||||
"httpx-oauth~=0.17",
|
||||
"ijson>=3.5.1",
|
||||
"imap-tools~=1.14.0",
|
||||
"jinja2~=3.1.5",
|
||||
"langdetect~=1.0.9",
|
||||
"llama-index-core>=0.14.22",
|
||||
"llama-index-core>=0.14.23",
|
||||
"llama-index-embeddings-huggingface>=0.6.1",
|
||||
"llama-index-embeddings-ollama>=0.9",
|
||||
"llama-index-embeddings-openai-like>=0.2.2",
|
||||
"llama-index-llms-ollama>=0.9.1",
|
||||
"llama-index-llms-openai-like>=0.7.1",
|
||||
"nltk~=3.10.0",
|
||||
"ocrmypdf~=17.4.2",
|
||||
"openai>=2.32",
|
||||
"ocrmypdf~=17.7.0",
|
||||
"openai>=2.48",
|
||||
"pathvalidate~=3.3.1",
|
||||
"pdf2image~=1.17.0",
|
||||
"python-dateutil~=2.9.0",
|
||||
@@ -68,17 +67,17 @@ dependencies = [
|
||||
"python-magic~=0.4.27",
|
||||
"rapidfuzz~=3.14.5",
|
||||
"redis[hiredis]~=5.2.1",
|
||||
"regex>=2026.4.4",
|
||||
"scikit-learn~=1.8.0",
|
||||
"sentence-transformers>=5.4.1",
|
||||
"regex>=2026.7.19",
|
||||
"scikit-learn~=1.9.0",
|
||||
"sentence-transformers>=5.6.1",
|
||||
"setproctitle~=1.3.4",
|
||||
"sqlite-vec==0.1.9",
|
||||
"tantivy~=0.26.0",
|
||||
"tika-client~=0.11.0",
|
||||
"torch~=2.13.0",
|
||||
"watchfiles>=1.1.1",
|
||||
"watchfiles>=1.2",
|
||||
"whitenoise~=6.11",
|
||||
"zxing-cpp~=3.0.0",
|
||||
"zxing-cpp~=3.1.0",
|
||||
]
|
||||
[project.optional-dependencies]
|
||||
mariadb = [
|
||||
@@ -101,25 +100,25 @@ dev = [
|
||||
{ include-group = "testing" },
|
||||
]
|
||||
docs = [
|
||||
"zensical>=0.0.47",
|
||||
"zensical>=0.0.51",
|
||||
]
|
||||
lint = [
|
||||
"prek~=0.3.10",
|
||||
"ruff~=0.15.20",
|
||||
"prek~=0.4.11",
|
||||
"ruff~=0.16.1",
|
||||
]
|
||||
testing = [
|
||||
"daphne",
|
||||
"factory-boy~=3.3.1",
|
||||
"faker~=40.15.0",
|
||||
"faker~=40.36.0",
|
||||
"imagehash",
|
||||
"pytest~=9.0.3",
|
||||
"pytest~=9.1.1",
|
||||
"pytest-cov~=7.1.0",
|
||||
"pytest-django~=4.12.0",
|
||||
"pytest-env~=1.6.0",
|
||||
"pytest-env~=1.7.0",
|
||||
"pytest-httpx",
|
||||
"pytest-mock~=3.15.1",
|
||||
# "pytest-randomly~=4.0.1",
|
||||
"pytest-rerunfailures~=16.1",
|
||||
"pytest-rerunfailures~=16.4",
|
||||
"pytest-sugar",
|
||||
"pytest-xdist~=3.8.0",
|
||||
"time-machine>=2.13",
|
||||
@@ -183,6 +182,7 @@ output-format = "grouped"
|
||||
line-ending = "lf"
|
||||
[tool.ruff.lint]
|
||||
# https://docs.astral.sh/ruff/rules/
|
||||
select = [ "E4", "E7", "E9", "F" ]
|
||||
extend-select = [
|
||||
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
|
||||
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
|
||||
|
||||
+12
-9
@@ -56,13 +56,13 @@
|
||||
},
|
||||
"architect": {
|
||||
"build": {
|
||||
"builder": "@angular-builders/custom-webpack:browser",
|
||||
"builder": "@angular/build:application",
|
||||
"options": {
|
||||
"customWebpackConfig": {
|
||||
"path": "./extra-webpack.config.ts"
|
||||
"outputPath": {
|
||||
"base": "dist/paperless-ui",
|
||||
"browser": ""
|
||||
},
|
||||
"outputPath": "dist/paperless-ui",
|
||||
"main": "src/main.ts",
|
||||
"browser": "src/main.ts",
|
||||
"outputHashing": "none",
|
||||
"index": "src/index.html",
|
||||
"polyfills": [
|
||||
@@ -97,6 +97,7 @@
|
||||
"scripts": [],
|
||||
"allowedCommonJsDependencies": [
|
||||
"file-saver",
|
||||
"mime-names",
|
||||
"utif"
|
||||
],
|
||||
"extractLicenses": false,
|
||||
@@ -117,11 +118,13 @@
|
||||
"with": "src/environments/environment.prod.ts"
|
||||
}
|
||||
],
|
||||
"outputPath": "../src/documents/static/frontend/",
|
||||
"outputPath": {
|
||||
"base": "../src/documents/static/frontend/",
|
||||
"browser": ""
|
||||
},
|
||||
"optimization": true,
|
||||
"outputHashing": "none",
|
||||
"sourceMap": false,
|
||||
"namedChunks": false,
|
||||
"extractLicenses": true,
|
||||
"budgets": [
|
||||
{
|
||||
@@ -145,7 +148,7 @@
|
||||
"defaultConfiguration": ""
|
||||
},
|
||||
"serve": {
|
||||
"builder": "@angular-builders/custom-webpack:dev-server",
|
||||
"builder": "@angular/build:dev-server",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build:en-US"
|
||||
},
|
||||
@@ -156,7 +159,7 @@
|
||||
}
|
||||
},
|
||||
"extract-i18n": {
|
||||
"builder": "@angular-builders/custom-webpack:extract-i18n",
|
||||
"builder": "@angular/build:extract-i18n",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build"
|
||||
}
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
import {
|
||||
CustomWebpackBrowserSchema,
|
||||
TargetOptions,
|
||||
} from '@angular-builders/custom-webpack'
|
||||
import * as webpack from 'webpack'
|
||||
const { codecovWebpackPlugin } = require('@codecov/webpack-plugin')
|
||||
|
||||
export default (
|
||||
config: webpack.Configuration,
|
||||
options: CustomWebpackBrowserSchema,
|
||||
targetOptions: TargetOptions
|
||||
) => {
|
||||
if (config.plugins) {
|
||||
config.plugins.push(
|
||||
codecovWebpackPlugin({
|
||||
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
|
||||
bundleName: 'paperless-ngx',
|
||||
uploadToken: process.env.CODECOV_TOKEN,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
+765
-766
File diff suppressed because it is too large
Load Diff
+27
-30
@@ -11,16 +11,16 @@
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/cdk": "^22.0.3",
|
||||
"@angular/common": "~22.0.5",
|
||||
"@angular/compiler": "~22.0.5",
|
||||
"@angular/core": "~22.0.5",
|
||||
"@angular/forms": "~22.0.5",
|
||||
"@angular/localize": "~22.0.5",
|
||||
"@angular/platform-browser": "~22.0.5",
|
||||
"@angular/router": "~22.0.5",
|
||||
"@angular/cdk": "^22.0.6",
|
||||
"@angular/common": "~22.1.0",
|
||||
"@angular/compiler": "~22.1.0",
|
||||
"@angular/core": "~22.1.0",
|
||||
"@angular/forms": "~22.1.0",
|
||||
"@angular/localize": "~22.1.0",
|
||||
"@angular/platform-browser": "~22.1.0",
|
||||
"@angular/router": "~22.1.0",
|
||||
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||
"@ng-select/ng-select": "^23.2.0",
|
||||
"@ng-select/ng-select": "^23.5.0",
|
||||
"@ngneat/dirty-check-forms": "^3.0.3",
|
||||
"@popperjs/core": "^2.11.8",
|
||||
"bootstrap": "^5.3.8",
|
||||
@@ -32,33 +32,31 @@
|
||||
"ngx-device-detector": "^12.0.0",
|
||||
"ngx-ui-tour-ng-bootstrap": "^19.0.0",
|
||||
"normalize-diacritics": "^5.0.0",
|
||||
"pdfjs-dist": "^6.0.227",
|
||||
"pdfjs-dist": "^6.2.108",
|
||||
"rxjs": "^7.8.2",
|
||||
"tslib": "^2.8.1",
|
||||
"utif": "^3.1.0",
|
||||
"uuid": "^14.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-builders/custom-webpack": "^22.0.1",
|
||||
"@angular-builders/jest": "^22.0.1",
|
||||
"@angular-devkit/core": "^22.0.5",
|
||||
"@angular-devkit/schematics": "^22.0.5",
|
||||
"@angular-eslint/builder": "22.0.0",
|
||||
"@angular-eslint/eslint-plugin": "22.0.0",
|
||||
"@angular-eslint/eslint-plugin-template": "22.0.0",
|
||||
"@angular-eslint/schematics": "22.0.0",
|
||||
"@angular-eslint/template-parser": "22.0.0",
|
||||
"@angular/build": "^22.0.5",
|
||||
"@angular/cli": "~22.0.5",
|
||||
"@angular/compiler-cli": "~22.0.5",
|
||||
"@codecov/webpack-plugin": "^2.0.1",
|
||||
"@playwright/test": "^1.61.1",
|
||||
"@angular-devkit/core": "^22.1.2",
|
||||
"@angular-devkit/schematics": "^22.1.2",
|
||||
"@angular-eslint/builder": "22.1.0",
|
||||
"@angular-eslint/eslint-plugin": "22.1.0",
|
||||
"@angular-eslint/eslint-plugin-template": "22.1.0",
|
||||
"@angular-eslint/schematics": "22.1.0",
|
||||
"@angular-eslint/template-parser": "22.1.0",
|
||||
"@angular/build": "22.1.2",
|
||||
"@angular/cli": "22.1.2",
|
||||
"@angular/compiler-cli": "~22.1.0",
|
||||
"@playwright/test": "^1.62.0",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^26.0.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.62.0",
|
||||
"@typescript-eslint/parser": "^8.62.0",
|
||||
"@typescript-eslint/utils": "^8.62.0",
|
||||
"eslint": "^10.5.0",
|
||||
"@types/node": "^26.1.1",
|
||||
"@typescript-eslint/eslint-plugin": "^8.65.0",
|
||||
"@typescript-eslint/parser": "^8.65.0",
|
||||
"@typescript-eslint/utils": "^8.65.0",
|
||||
"eslint": "^10.8.0",
|
||||
"jest": "30.4.2",
|
||||
"jest-environment-jsdom": "^30.4.1",
|
||||
"jest-junit": "^17.0.0",
|
||||
@@ -66,8 +64,7 @@
|
||||
"jest-websocket-mock": "^2.5.0",
|
||||
"prettier-plugin-organize-imports": "^4.3.0",
|
||||
"ts-node": "~10.9.1",
|
||||
"typescript": "^6.0.3",
|
||||
"webpack": "^5.107.2"
|
||||
"typescript": "^6.0.3"
|
||||
},
|
||||
"packageManager": "pnpm@10.26.0"
|
||||
}
|
||||
|
||||
Generated
+2177
-1981
File diff suppressed because it is too large
Load Diff
@@ -104,7 +104,7 @@
|
||||
</h6>
|
||||
<ul class="nav flex-column mb-2" cdkDropList (cdkDropListDropped)="onDrop($event)">
|
||||
@for (view of savedViewService.sidebarViews; track view.id) {
|
||||
<li class="nav-item w-100 app-link" cdkDrag [cdkDragDisabled]="!settingsService.organizingSidebarSavedViews()"
|
||||
<li class="nav-item w-100 app-link" cdkDrag [cdkDragDisabled]="!settingsService.organizingSidebarSavedViews() || !canSaveSettings"
|
||||
cdkDragPreviewContainer="parent" cdkDragPreviewClass="navItemDrag" (cdkDragStarted)="onDragStart($event)"
|
||||
(cdkDragEnded)="onDragEnd($event)">
|
||||
<a class="nav-link" routerLink="view/{{view.id}}"
|
||||
@@ -120,7 +120,7 @@
|
||||
<span class="badge bg-info text-dark position-absolute top-0 end-0 d-none d-md-block">{{ savedViewService.getDocumentCount(view) }}</span>
|
||||
}
|
||||
</a>
|
||||
@if (settingsService.organizingSidebarSavedViews()) {
|
||||
@if (settingsService.organizingSidebarSavedViews() && canSaveSettings) {
|
||||
<div class="position-absolute end-0 top-0 px-3 py-2" [class.me-n3]="slimSidebarEnabled" cdkDragHandle>
|
||||
<i-bs name="grip-vertical"></i-bs>
|
||||
</div>
|
||||
|
||||
+24
@@ -1019,4 +1019,28 @@ describe('WorkflowEditDialogComponent', () => {
|
||||
'pass3',
|
||||
])
|
||||
})
|
||||
|
||||
it('should parse passwords again when retrying a failed save', () => {
|
||||
component.object = {
|
||||
name: 'Workflow with blank Passwords',
|
||||
id: 1,
|
||||
order: null,
|
||||
enabled: true,
|
||||
triggers: [],
|
||||
actions: [
|
||||
{
|
||||
id: 1,
|
||||
type: WorkflowActionType.PasswordRemoval,
|
||||
passwords: [],
|
||||
},
|
||||
],
|
||||
}
|
||||
component.ngOnInit()
|
||||
|
||||
component.save()
|
||||
component.objectForm.get('order').setValue(1)
|
||||
|
||||
expect(() => component.save()).not.toThrow()
|
||||
expect(component.objectForm.get('actions').value[0].passwords).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
+2
-3
@@ -1207,9 +1207,8 @@ export class WorkflowEditDialogComponent
|
||||
return passwords.join('\n')
|
||||
}
|
||||
|
||||
private parsePasswords(value: string = ''): string[] {
|
||||
return value
|
||||
.split(/[\n,]+/)
|
||||
private parsePasswords(value: string | string[] = ''): string[] {
|
||||
return (Array.isArray(value) ? value : value.split(/[\n,]+/))
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0)
|
||||
}
|
||||
|
||||
@@ -151,6 +151,13 @@
|
||||
inset: 0;
|
||||
pointer-events: none;
|
||||
|
||||
& section {
|
||||
position: absolute;
|
||||
text-align: initial;
|
||||
box-sizing: border-box;
|
||||
transform-origin: 0 0;
|
||||
}
|
||||
|
||||
& .annotationTextContent {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
@@ -90,6 +90,7 @@ describe('PngxPdfViewerComponent', () => {
|
||||
}
|
||||
expect(viewer).toBeInstanceOf(PDFSinglePageViewer)
|
||||
expect(viewer.options.textLayerMode).toBe(0)
|
||||
expect(viewer.options.enableSelectionRendering).toBe(false)
|
||||
})
|
||||
|
||||
it('applies zoom, rotation, and page changes', async () => {
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
ViewChild,
|
||||
} from '@angular/core'
|
||||
import {
|
||||
AnnotationMode,
|
||||
getDocument,
|
||||
GlobalWorkerOptions,
|
||||
PDFDocumentLoadingTask,
|
||||
@@ -221,6 +222,8 @@ export class PngxPdfViewerComponent
|
||||
linkService: this.linkService,
|
||||
findController: this.findController,
|
||||
textLayerMode,
|
||||
annotationMode: AnnotationMode.ENABLE,
|
||||
enableSelectionRendering: false,
|
||||
removePageBorders: true,
|
||||
}
|
||||
|
||||
|
||||
@@ -2161,8 +2161,14 @@ describe('DocumentDetailComponent', () => {
|
||||
it('should support open share links and email modals', () => {
|
||||
const modalSpy = jest.spyOn(modalService, 'open')
|
||||
initNormally()
|
||||
component.selectedVersionId.set(10)
|
||||
component.openShareLinks()
|
||||
expect(modalSpy).toHaveBeenCalled()
|
||||
expect(
|
||||
(
|
||||
modalSpy.mock.results[0].value as NgbModalRef
|
||||
).componentInstance.documentId()
|
||||
).toBe(10)
|
||||
component.openEmailDocument()
|
||||
expect(modalSpy).toHaveBeenCalled()
|
||||
})
|
||||
@@ -2191,9 +2197,6 @@ describe('DocumentDetailComponent', () => {
|
||||
const appendChildSpy = jest
|
||||
.spyOn(document.body, 'appendChild')
|
||||
.mockImplementation((node: Node) => node)
|
||||
const removeChildSpy = jest
|
||||
.spyOn(document.body, 'removeChild')
|
||||
.mockImplementation((node: Node) => node)
|
||||
const createObjectURLSpy = jest
|
||||
.spyOn(URL, 'createObjectURL')
|
||||
.mockReturnValue('blob:mock-url')
|
||||
@@ -2212,6 +2215,7 @@ describe('DocumentDetailComponent', () => {
|
||||
src: '',
|
||||
onload: null,
|
||||
contentWindow: mockContentWindow,
|
||||
remove: jest.fn(),
|
||||
}
|
||||
|
||||
const createElementSpy = jest
|
||||
@@ -2255,12 +2259,11 @@ describe('DocumentDetailComponent', () => {
|
||||
mockContentWindow.onafterprint(new Event('afterprint'))
|
||||
}
|
||||
|
||||
expect(removeChildSpy).toHaveBeenCalledWith(mockIframe)
|
||||
expect(mockIframe.remove).toHaveBeenCalled()
|
||||
expect(revokeObjectURLSpy).toHaveBeenCalledWith('blob:mock-url')
|
||||
|
||||
createElementSpy.mockRestore()
|
||||
appendChildSpy.mockRestore()
|
||||
removeChildSpy.mockRestore()
|
||||
createObjectURLSpy.mockRestore()
|
||||
revokeObjectURLSpy.mockRestore()
|
||||
})
|
||||
@@ -2307,9 +2310,6 @@ describe('DocumentDetailComponent', () => {
|
||||
const appendChildSpy = jest
|
||||
.spyOn(document.body, 'appendChild')
|
||||
.mockImplementation((node: Node) => node)
|
||||
const removeChildSpy = jest
|
||||
.spyOn(document.body, 'removeChild')
|
||||
.mockImplementation((node: Node) => node)
|
||||
const createObjectURLSpy = jest
|
||||
.spyOn(URL, 'createObjectURL')
|
||||
.mockReturnValue('blob:mock-url')
|
||||
@@ -2332,6 +2332,7 @@ describe('DocumentDetailComponent', () => {
|
||||
src: '',
|
||||
onload: null,
|
||||
contentWindow: mockContentWindow,
|
||||
remove: jest.fn(),
|
||||
}
|
||||
|
||||
const createElementSpy = jest
|
||||
@@ -2354,15 +2355,21 @@ describe('DocumentDetailComponent', () => {
|
||||
|
||||
if (expectToast) {
|
||||
expect(toastSpy).toHaveBeenCalled()
|
||||
expect(mockIframe.remove).toHaveBeenCalled()
|
||||
expect(revokeObjectURLSpy).toHaveBeenCalledWith('blob:mock-url')
|
||||
} else {
|
||||
expect(toastSpy).not.toHaveBeenCalled()
|
||||
expect(mockIframe.remove).not.toHaveBeenCalled()
|
||||
expect(revokeObjectURLSpy).not.toHaveBeenCalled()
|
||||
|
||||
component.ngOnDestroy()
|
||||
|
||||
expect(mockIframe.remove).toHaveBeenCalled()
|
||||
expect(revokeObjectURLSpy).toHaveBeenCalledWith('blob:mock-url')
|
||||
}
|
||||
expect(removeChildSpy).toHaveBeenCalledWith(mockIframe)
|
||||
expect(revokeObjectURLSpy).toHaveBeenCalledWith('blob:mock-url')
|
||||
|
||||
createElementSpy.mockRestore()
|
||||
appendChildSpy.mockRestore()
|
||||
removeChildSpy.mockRestore()
|
||||
createObjectURLSpy.mockRestore()
|
||||
revokeObjectURLSpy.mockRestore()
|
||||
})
|
||||
|
||||
@@ -289,6 +289,8 @@ export class DocumentDetailComponent
|
||||
private incomingUpdateModal: NgbModalRef
|
||||
private pendingIncomingUpdate: IncomingDocumentUpdate
|
||||
private lastLocalSaveModified: string | null = null
|
||||
private printIframe: HTMLIFrameElement | null = null
|
||||
private printBlobUrl: string | null = null
|
||||
|
||||
requiresPassword: boolean = false
|
||||
password: string
|
||||
@@ -868,6 +870,7 @@ export class DocumentDetailComponent
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
this.cleanupPrintDocument()
|
||||
this.unsubscribeNotifier.next(this)
|
||||
this.unsubscribeNotifier.complete()
|
||||
}
|
||||
@@ -1889,6 +1892,7 @@ export class DocumentDetailComponent
|
||||
}
|
||||
|
||||
printDocument() {
|
||||
this.cleanupPrintDocument()
|
||||
const selectedVersionId = this.getSelectedNonLatestVersionId()
|
||||
const printUrl = this.documentsService.getDownloadUrl(
|
||||
this.document().id,
|
||||
@@ -1902,6 +1906,8 @@ export class DocumentDetailComponent
|
||||
next: (blob) => {
|
||||
const blobUrl = URL.createObjectURL(blob)
|
||||
const iframe = document.createElement('iframe')
|
||||
this.printIframe = iframe
|
||||
this.printBlobUrl = blobUrl
|
||||
iframe.style.position = 'fixed'
|
||||
iframe.style.right = '0'
|
||||
iframe.style.bottom = '0'
|
||||
@@ -1917,22 +1923,19 @@ export class DocumentDetailComponent
|
||||
iframe.contentWindow.focus()
|
||||
iframe.contentWindow.print()
|
||||
iframe.contentWindow.onafterprint = () => {
|
||||
document.body.removeChild(iframe)
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
this.cleanupPrintDocument()
|
||||
}
|
||||
} catch (err) {
|
||||
// FF throws cross-origin error on onafterprint
|
||||
const isCrossOriginAfterPrintError =
|
||||
err instanceof DOMException &&
|
||||
err.message.includes('onafterprint')
|
||||
// FF throws here while print preview is still reading the iframe
|
||||
// so keep it alive until the next print or teardown
|
||||
if (!isCrossOriginAfterPrintError) {
|
||||
this.toastService.showError($localize`Print failed.`, err)
|
||||
timer(100).subscribe(() => this.cleanupPrintDocument())
|
||||
}
|
||||
timer(100).subscribe(() => {
|
||||
// delay to avoid FF print failure
|
||||
document.body.removeChild(iframe)
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1945,9 +1948,20 @@ export class DocumentDetailComponent
|
||||
})
|
||||
}
|
||||
|
||||
private cleanupPrintDocument() {
|
||||
if (this.printIframe) this.printIframe.remove()
|
||||
this.printIframe = null
|
||||
if (this.printBlobUrl) {
|
||||
URL.revokeObjectURL(this.printBlobUrl)
|
||||
this.printBlobUrl = null
|
||||
}
|
||||
}
|
||||
|
||||
public openShareLinks() {
|
||||
const modal = this.modalService.open(ShareLinksDialogComponent)
|
||||
modal.componentInstance.documentId.set(this.document().id)
|
||||
modal.componentInstance.documentId.set(
|
||||
this.selectedVersionId() ?? this.document().id
|
||||
)
|
||||
modal.componentInstance.hasArchiveVersion.set(
|
||||
this.metadata()?.has_archive_version ??
|
||||
!!this.document()?.archived_file_name
|
||||
|
||||
@@ -1697,6 +1697,24 @@ describe('FilterEditorComponent', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('should carry over text filtering once with created and added relative dates', () => {
|
||||
component.textFilter = 'foo'
|
||||
const datesDropdown = fixture.debugElement.query(
|
||||
By.directive(DatesDropdownComponent)
|
||||
)
|
||||
component.dateCreatedRelativeDate = RelativeDate.WITHIN_1_WEEK
|
||||
component.dateAddedRelativeDate = RelativeDate.WITHIN_1_MONTH
|
||||
datesDropdown.triggerEventHandler('datesSet')
|
||||
fixture.detectChanges()
|
||||
tick(400)
|
||||
expect(component.filterRules).toEqual([
|
||||
{
|
||||
rule_type: FILTER_FULLTEXT_QUERY,
|
||||
value: 'foo,created:[-1 week to now],added:[-1 month to now]',
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('should convert legacy title filters into full text query when adding a created relative date', () => {
|
||||
component.filterRules = [
|
||||
{
|
||||
@@ -2195,6 +2213,20 @@ describe('FilterEditorComponent', () => {
|
||||
expect(blurSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should only dismiss open autocomplete suggestions on Escape, keeping the query', () => {
|
||||
component.textFilter = 'foo bar'
|
||||
component.textFilterInput.nativeElement.value = 'foo bar'
|
||||
jest.spyOn(component.searchTypeahead, 'isPopupOpen').mockReturnValue(true)
|
||||
const dismissSpy = jest
|
||||
.spyOn(component.searchTypeahead, 'dismissPopup')
|
||||
.mockImplementation(() => {})
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keydown', { key: 'Escape' })
|
||||
)
|
||||
expect(dismissSpy).toHaveBeenCalled()
|
||||
expect(component.textFilter).toEqual('foo bar')
|
||||
})
|
||||
|
||||
it('should adjust text filter targets if more like search', () => {
|
||||
const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike' // private const
|
||||
component.textFilterTarget = TEXT_FILTER_TARGET_FULLTEXT_MORELIKE
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
|
||||
import {
|
||||
NgbDropdownModule,
|
||||
NgbTypeahead,
|
||||
NgbTypeaheadModule,
|
||||
} from '@ng-bootstrap/ng-bootstrap'
|
||||
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
|
||||
@@ -351,6 +352,9 @@ export class FilterEditorComponent
|
||||
@ViewChild('textFilterInput')
|
||||
textFilterInput: ElementRef
|
||||
|
||||
@ViewChild(NgbTypeahead)
|
||||
searchTypeahead: NgbTypeahead
|
||||
|
||||
readonly customFields = signal<CustomField[]>([])
|
||||
|
||||
tagDocumentCounts: SelectionDataItem[]
|
||||
@@ -1016,7 +1020,6 @@ export class FilterEditorComponent
|
||||
this.dateAddedRelativeDate !== null ||
|
||||
this.dateCreatedRelativeDate !== null
|
||||
) {
|
||||
let queryArgs: Array<string> = []
|
||||
let existingRule = filterRules.find(
|
||||
(fr) => fr.rule_type == FILTER_FULLTEXT_QUERY
|
||||
)
|
||||
@@ -1038,32 +1041,28 @@ export class FilterEditorComponent
|
||||
existingRule.rule_type = FILTER_FULLTEXT_QUERY
|
||||
}
|
||||
|
||||
let existingRuleArgs = existingRule?.value.split(',')
|
||||
let queryArgs = existingRule?.value.split(',') ?? []
|
||||
if (this.dateCreatedRelativeDate !== null) {
|
||||
const rd = RELATIVE_DATE_QUERYSTRINGS.find(
|
||||
(qS) => qS.relativeDate == this.dateCreatedRelativeDate
|
||||
)
|
||||
queryArgs = queryArgs.filter(
|
||||
(arg) => !arg.match(RELATIVE_DATE_QUERY_REGEXP_CREATED)
|
||||
)
|
||||
queryArgs.push(
|
||||
`created:${rd.isRange ? `[${rd.dateQuery}]` : `"${rd.dateQuery}"`}`
|
||||
)
|
||||
if (existingRule) {
|
||||
queryArgs = existingRuleArgs
|
||||
.filter((arg) => !arg.match(RELATIVE_DATE_QUERY_REGEXP_CREATED))
|
||||
.concat(queryArgs)
|
||||
}
|
||||
}
|
||||
if (this.dateAddedRelativeDate !== null) {
|
||||
const rd = RELATIVE_DATE_QUERYSTRINGS.find(
|
||||
(qS) => qS.relativeDate == this.dateAddedRelativeDate
|
||||
)
|
||||
queryArgs = queryArgs.filter(
|
||||
(arg) => !arg.match(RELATIVE_DATE_QUERY_REGEXP_ADDED)
|
||||
)
|
||||
queryArgs.push(
|
||||
`added:${rd.isRange ? `[${rd.dateQuery}]` : `"${rd.dateQuery}"`}`
|
||||
)
|
||||
if (existingRule) {
|
||||
queryArgs = existingRuleArgs
|
||||
.filter((arg) => !arg.match(RELATIVE_DATE_QUERY_REGEXP_ADDED))
|
||||
.concat(queryArgs)
|
||||
}
|
||||
}
|
||||
|
||||
if (existingRule) {
|
||||
@@ -1155,6 +1154,7 @@ export class FilterEditorComponent
|
||||
}
|
||||
|
||||
set textFilter(value) {
|
||||
this._textFilter = value // set immediately to prevent loss of keystrokes
|
||||
this.textFilterDebounce.next(value)
|
||||
}
|
||||
|
||||
@@ -1247,9 +1247,9 @@ export class FilterEditorComponent
|
||||
distinctUntilChanged(),
|
||||
filter((query) => !query.length || query.length > 2)
|
||||
)
|
||||
.subscribe((text) =>
|
||||
.subscribe(() =>
|
||||
this.updateTextFilter(
|
||||
text,
|
||||
this._textFilter, // use the current value, not the debounced (possibly stale) one
|
||||
this.textFilterTarget !== TEXT_FILTER_TARGET_FULLTEXT_QUERY
|
||||
)
|
||||
)
|
||||
@@ -1325,6 +1325,11 @@ export class FilterEditorComponent
|
||||
this.updateTextFilter(filterString)
|
||||
}
|
||||
} else if (event.key === 'Escape') {
|
||||
if (this.searchTypeahead?.isPopupOpen()) {
|
||||
// only dismiss the suggestions, so longer query can use Enter
|
||||
this.searchTypeahead.dismissPopup()
|
||||
return
|
||||
}
|
||||
if (this._textFilter?.length) {
|
||||
this.resetTextField()
|
||||
} else {
|
||||
|
||||
+1
-1
@@ -88,7 +88,7 @@
|
||||
@if (depth > 0) {
|
||||
<div class="indicator"></div>
|
||||
}
|
||||
<button class="btn btn-link ms-0 ps-0 text-start" (click)="userCanEdit(object) ? openEditDialog(object) : null; $event.stopPropagation()">{{ object.name }}</button>
|
||||
<button class="btn btn-link ms-0 ps-0 text-start" style="user-select: text;" [disabled]="!userCanEdit(object)" (click)="userCanEdit(object) ? openEditDialog(object) : null; $event.stopPropagation()">{{ object.name }}</button>
|
||||
</td>
|
||||
<td class="d-none d-sm-table-cell">{{ getMatching(object) }}</td>
|
||||
<td>{{ getDocumentCount(object) }}</td>
|
||||
|
||||
@@ -14,16 +14,18 @@
|
||||
<div class="col">
|
||||
<pngx-input-text title="Name" formControlName="name"></pngx-input-text>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="form-check form-switch mt-3">
|
||||
<input type="checkbox" class="form-check-input" id="show_on_dashboard_{{view.id}}" formControlName="show_on_dashboard">
|
||||
<label class="form-check-label" for="show_on_dashboard_{{view.id}}" i18n>Show on dashboard</label>
|
||||
@if (canSaveSettings) {
|
||||
<div class="col">
|
||||
<div class="form-check form-switch mt-3">
|
||||
<input type="checkbox" class="form-check-input" id="show_on_dashboard_{{view.id}}" formControlName="show_on_dashboard">
|
||||
<label class="form-check-label" for="show_on_dashboard_{{view.id}}" i18n>Show on dashboard</label>
|
||||
</div>
|
||||
<div class="form-check form-switch">
|
||||
<input type="checkbox" class="form-check-input" id="show_in_sidebar_{{view.id}}" formControlName="show_in_sidebar">
|
||||
<label class="form-check-label" for="show_in_sidebar_{{view.id}}" i18n>Show in sidebar</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-check form-switch">
|
||||
<input type="checkbox" class="form-check-input" id="show_in_sidebar_{{view.id}}" formControlName="show_in_sidebar">
|
||||
<label class="form-check-label" for="show_in_sidebar_{{view.id}}" i18n>Show in sidebar</label>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
<div class="col-auto">
|
||||
@if (canDeleteSavedView(view)) {
|
||||
<label class="form-label" for="name_{{view.id}}" i18n>Actions</label>
|
||||
|
||||
@@ -17,6 +17,7 @@ import { IfPermissionsDirective } from 'src/app/directives/if-permissions.direct
|
||||
import {
|
||||
PermissionAction,
|
||||
PermissionsService,
|
||||
PermissionType,
|
||||
} from 'src/app/services/permissions.service'
|
||||
import { SavedViewService } from 'src/app/services/rest/saved-view.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
@@ -71,7 +72,9 @@ export class SavedViewsComponent
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.settings.organizingSidebarSavedViews.set(true)
|
||||
if (this.canSaveSettings) {
|
||||
this.settings.organizingSidebarSavedViews.set(true)
|
||||
}
|
||||
}
|
||||
|
||||
ngOnInit(): void {
|
||||
@@ -89,7 +92,9 @@ export class SavedViewsComponent
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
this.settings.organizingSidebarSavedViews.set(false)
|
||||
if (this.canSaveSettings) {
|
||||
this.settings.organizingSidebarSavedViews.set(false)
|
||||
}
|
||||
super.ngOnDestroy()
|
||||
}
|
||||
|
||||
@@ -216,7 +221,7 @@ export class SavedViewsComponent
|
||||
switchMap(() => this.savedViewService.patchMany(changed))
|
||||
)
|
||||
}
|
||||
if (visibilityChanged) {
|
||||
if (visibilityChanged && this.canSaveSettings) {
|
||||
saveOperation = saveOperation.pipe(
|
||||
switchMap(() =>
|
||||
this.settings.updateSavedViewsVisibility(
|
||||
@@ -250,6 +255,19 @@ export class SavedViewsComponent
|
||||
return this.permissionsService.currentUserOwnsObject(view)
|
||||
}
|
||||
|
||||
public get canSaveSettings(): boolean {
|
||||
return (
|
||||
this.permissionsService.currentUserCan(
|
||||
PermissionAction.Change,
|
||||
PermissionType.UISettings
|
||||
) &&
|
||||
this.permissionsService.currentUserCan(
|
||||
PermissionAction.Add,
|
||||
PermissionType.UISettings
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
public editPermissions(savedView: SavedView): void {
|
||||
const modal = this.modalService.open(PermissionsDialogComponent, {
|
||||
backdrop: 'static',
|
||||
|
||||
@@ -57,6 +57,19 @@ describe('LocalizedDateParserFormatter', () => {
|
||||
expect(val).toEqual({ day: 4, month: 5, year: 2023 })
|
||||
})
|
||||
|
||||
it('should parse yyyy-mm-dd input with unpadded month or day by locale', () => {
|
||||
let val = dateParserFormatter.parse('2023-5-4')
|
||||
expect(val).toEqual({ day: 4, month: 5, year: 2023 })
|
||||
|
||||
settingsService.setLanguage('de-de') // dd.mm.yyyy
|
||||
val = dateParserFormatter.parse('2023-5-4')
|
||||
expect(val).toEqual({ day: 4, month: 5, year: 2023 })
|
||||
|
||||
settingsService.setLanguage('tr-tr') // yyyy-mm-dd
|
||||
val = dateParserFormatter.parse('2023-5-4')
|
||||
expect(val).toEqual({ day: 4, month: 5, year: 2023 })
|
||||
})
|
||||
|
||||
it('should parse date struct to string by locale', () => {
|
||||
const dateStruct = {
|
||||
day: 4,
|
||||
|
||||
@@ -52,15 +52,11 @@ export class LocalizedDateParserFormatter extends NgbDateParserFormatter {
|
||||
let segments = value.split(this.separatorRegExp)
|
||||
|
||||
// always accept strict yyyy*mm*dd format even if that's not the input format since we can be certain its not yyyy*dd*mm
|
||||
if (
|
||||
value.length == 10 &&
|
||||
segments.length == 3 &&
|
||||
segments[0].length == 4
|
||||
) {
|
||||
if (segments.length == 3 && segments[0].length == 4) {
|
||||
return inputFormat
|
||||
.replace('yyyy', segments[0])
|
||||
.replace('mm', segments[1])
|
||||
.replace('dd', segments[2])
|
||||
.replace('mm', segments[1].padStart(2, '0'))
|
||||
.replace('dd', segments[2].padStart(2, '0'))
|
||||
} else {
|
||||
// otherwise pad & re-join without separator
|
||||
value = segments.map((segment) => segment.padStart(2, '0')).join('')
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+2023
-2025
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,13 @@ export const GlobalWorkerOptions = {
|
||||
workerSrc: '',
|
||||
}
|
||||
|
||||
export const AnnotationMode = {
|
||||
DISABLE: 0,
|
||||
ENABLE: 1,
|
||||
ENABLE_FORMS: 2,
|
||||
ENABLE_STORAGE: 3,
|
||||
}
|
||||
|
||||
export const getDocument = (_src: unknown): PDFDocumentLoadingTask => {
|
||||
return new PDFDocumentLoadingTask(Promise.resolve(new PDFDocumentProxy()))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from contextlib import AbstractContextManager
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from pathlib import PurePosixPath
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.serializers.json import DjangoJSONEncoder
|
||||
|
||||
from documents.file_handling import delete_empty_directories
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from typing import TextIO
|
||||
|
||||
|
||||
def _dumps(content: list | dict) -> str:
|
||||
"""Serialize export JSON consistently across all sinks."""
|
||||
return json.dumps(content, cls=DjangoJSONEncoder, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
class StreamingManifestWriter:
|
||||
"""Incrementally writes a JSON array to a text handle, one record at a time.
|
||||
|
||||
Knows nothing about folders or zips: it writes the array framing and records
|
||||
to whatever handle the sink's ``stream()`` yields. The sink owns the handle's
|
||||
lifecycle (atomic rename, compare, spooling).
|
||||
"""
|
||||
|
||||
def __init__(self, handle: TextIO) -> None:
|
||||
self._file = handle
|
||||
self._first = True
|
||||
self._file.write("[")
|
||||
|
||||
def write_record(self, record: dict) -> None:
|
||||
if not self._first:
|
||||
self._file.write(",\n")
|
||||
else:
|
||||
self._first = False
|
||||
self._file.write(_dumps(record))
|
||||
|
||||
def write_batch(self, records: list[dict]) -> None:
|
||||
for record in records:
|
||||
self.write_record(record)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Write the closing bracket. Does NOT close the handle (the sink owns it)."""
|
||||
self._file.write("\n]")
|
||||
|
||||
|
||||
class ExportSink(AbstractContextManager, abc.ABC):
|
||||
"""Destination for a document export.
|
||||
|
||||
The command declares export contents via three verbs; the sink decides how to
|
||||
persist each. ``arcname`` is always a relative POSIX path
|
||||
(e.g. ``"manifest.json"``, ``"originals/foo.pdf"``).
|
||||
|
||||
Contract:
|
||||
* At most one ``stream()`` open at a time (it is the manifest);
|
||||
``add_file``/``add_json`` may be called while it is open.
|
||||
* Context-manager: normal exit finalizes, an exception aborts. No partial or
|
||||
failed run leaves a complete-looking artifact.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def add_file(
|
||||
self,
|
||||
source: Path,
|
||||
arcname: str,
|
||||
*,
|
||||
checksum: str | None = None,
|
||||
) -> None: ...
|
||||
|
||||
@abc.abstractmethod
|
||||
def add_json(self, content: list | dict, arcname: str) -> None: ...
|
||||
|
||||
@abc.abstractmethod
|
||||
def stream(self, arcname: str) -> AbstractContextManager[TextIO]: ...
|
||||
|
||||
def _open(self) -> None:
|
||||
"""Hook called on context entry. Override as needed."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _finalize(self) -> None:
|
||||
"""Commit on clean exit."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _abort(self) -> None:
|
||||
"""Roll back on exception."""
|
||||
|
||||
def __enter__(self) -> ExportSink:
|
||||
self._open()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if exc_type is not None:
|
||||
self._abort()
|
||||
else:
|
||||
self._finalize()
|
||||
|
||||
|
||||
class DirectoryExportSink(ExportSink):
|
||||
"""Writes loose files into a target directory, with incremental sync.
|
||||
|
||||
Owns the snapshot/skip/compare/prune machinery that used to live in the
|
||||
command (``files_in_export_dir``, ``check_and_copy``, ``check_and_write_json``,
|
||||
and the ``--delete`` pass).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target: Path,
|
||||
*,
|
||||
compare_checksums: bool,
|
||||
compare_json: bool,
|
||||
delete: bool,
|
||||
) -> None:
|
||||
self._target = target.resolve()
|
||||
self._compare_checksums = compare_checksums
|
||||
self._compare_json = compare_json
|
||||
self._delete = delete
|
||||
self._snapshot: set[Path] = set()
|
||||
self._stream_open = False
|
||||
|
||||
def _open(self) -> None:
|
||||
for x in self._target.glob("**/*"):
|
||||
if x.is_file():
|
||||
self._snapshot.add(x.resolve())
|
||||
|
||||
def add_file(
|
||||
self,
|
||||
source: Path,
|
||||
arcname: str,
|
||||
*,
|
||||
checksum: str | None = None,
|
||||
) -> None:
|
||||
target = (self._target / arcname).resolve()
|
||||
self._snapshot.discard(target)
|
||||
perform_copy = False
|
||||
if target.exists():
|
||||
source_stat = source.stat()
|
||||
target_stat = target.stat()
|
||||
if self._compare_checksums and checksum:
|
||||
perform_copy = compute_checksum(target) != checksum
|
||||
elif (
|
||||
source_stat.st_mtime != target_stat.st_mtime
|
||||
or source_stat.st_size != target_stat.st_size
|
||||
):
|
||||
perform_copy = True
|
||||
else:
|
||||
perform_copy = True
|
||||
if perform_copy:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy_file_with_basic_stats(source, target)
|
||||
|
||||
@staticmethod
|
||||
def _content_unchanged(target: Path, new_bytes: bytes) -> bool:
|
||||
"""True if ``target`` already holds byte-identical content (BLAKE2b)."""
|
||||
return (
|
||||
hashlib.blake2b(target.read_bytes()).hexdigest()
|
||||
== hashlib.blake2b(new_bytes).hexdigest()
|
||||
)
|
||||
|
||||
def add_json(self, content: list | dict, arcname: str) -> None:
|
||||
target = (self._target / arcname).resolve()
|
||||
json_str = _dumps(content)
|
||||
perform_write = True
|
||||
if target in self._snapshot:
|
||||
self._snapshot.discard(target)
|
||||
if self._compare_json and self._content_unchanged(
|
||||
target,
|
||||
json_str.encode("utf-8"),
|
||||
):
|
||||
perform_write = False
|
||||
if perform_write:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json_str, encoding="utf-8")
|
||||
|
||||
@contextmanager
|
||||
def stream(self, arcname: str) -> Iterator[TextIO]:
|
||||
if self._stream_open:
|
||||
raise RuntimeError("A stream is already open on this sink")
|
||||
target = (self._target / arcname).resolve()
|
||||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = tmp.open("w", encoding="utf-8")
|
||||
self._stream_open = True
|
||||
try:
|
||||
yield handle
|
||||
except BaseException:
|
||||
handle.close()
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
else:
|
||||
handle.close()
|
||||
self._commit_streamed_file(target, tmp)
|
||||
finally:
|
||||
self._stream_open = False
|
||||
|
||||
def _commit_streamed_file(self, target: Path, tmp: Path) -> None:
|
||||
if target in self._snapshot:
|
||||
self._snapshot.discard(target)
|
||||
if self._compare_json and self._content_unchanged(
|
||||
target,
|
||||
tmp.read_bytes(),
|
||||
):
|
||||
tmp.unlink()
|
||||
return
|
||||
tmp.rename(target)
|
||||
|
||||
def _finalize(self) -> None:
|
||||
if self._delete:
|
||||
for f in self._snapshot:
|
||||
if not f.is_relative_to(self._target): # pragma: no cover
|
||||
# Defense in depth: a symlink inside the export dir can
|
||||
# resolve outside of it; never delete outside the target.
|
||||
continue
|
||||
f.unlink()
|
||||
delete_empty_directories(f.parent, self._target)
|
||||
|
||||
def _abort(self) -> None:
|
||||
# Folder mode is in-place/incremental: streamed .tmp files are already
|
||||
# cleaned in stream(); leave everything else intact and skip the prune.
|
||||
return None
|
||||
|
||||
|
||||
class ZipExportSink(ExportSink):
|
||||
"""Writes a single zip archive, produced atomically only on success.
|
||||
|
||||
Builds into ``<target>/<zip_name>.zip.tmp`` and renames to ``.zip`` on clean
|
||||
finalize. The manifest stream is spooled to a temp file in SCRATCH_DIR and
|
||||
added as an entry at finalize (a zip entry cannot be interleaved with others).
|
||||
"""
|
||||
|
||||
def __init__(self, target: Path, zip_name: str, *, delete: bool = False) -> None:
|
||||
self._target = target.resolve()
|
||||
self._zip_path = (self._target / zip_name).with_suffix(".zip")
|
||||
self._tmp_path = self._zip_path.with_name(self._zip_path.name + ".tmp")
|
||||
self._delete = delete
|
||||
self._zip: zipfile.ZipFile | None = None
|
||||
self._dirs: set[str] = set()
|
||||
self._pending_manifest: tuple[Path, str] | None = None
|
||||
self._stream_open = False
|
||||
|
||||
def _open(self) -> None:
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
self._zip = zipfile.ZipFile(
|
||||
self._tmp_path,
|
||||
"w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
allowZip64=True,
|
||||
)
|
||||
|
||||
def _ensure_dirs(self, arcname: str) -> None:
|
||||
assert self._zip is not None
|
||||
dir_arc = ""
|
||||
for part in PurePosixPath(arcname).parts[:-1]:
|
||||
dir_arc += f"{part}/"
|
||||
if dir_arc not in self._dirs:
|
||||
self._dirs.add(dir_arc)
|
||||
self._zip.mkdir(dir_arc)
|
||||
|
||||
def add_file(
|
||||
self,
|
||||
source: Path,
|
||||
arcname: str,
|
||||
*,
|
||||
checksum: str | None = None,
|
||||
) -> None:
|
||||
assert self._zip is not None
|
||||
self._ensure_dirs(arcname)
|
||||
self._zip.write(source, arcname=arcname)
|
||||
|
||||
def add_json(self, content: list | dict, arcname: str) -> None:
|
||||
assert self._zip is not None
|
||||
self._ensure_dirs(arcname)
|
||||
self._zip.writestr(arcname, _dumps(content))
|
||||
|
||||
@contextmanager
|
||||
def stream(self, arcname: str) -> Iterator[TextIO]:
|
||||
if self._stream_open:
|
||||
raise RuntimeError("A stream is already open on this sink")
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=settings.SCRATCH_DIR,
|
||||
prefix="export-manifest-",
|
||||
suffix=".json",
|
||||
)
|
||||
tmp = Path(tmp_name)
|
||||
handle = os.fdopen(fd, "w", encoding="utf-8")
|
||||
self._stream_open = True
|
||||
try:
|
||||
yield handle
|
||||
except BaseException:
|
||||
handle.close()
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
else:
|
||||
handle.close()
|
||||
self._pending_manifest = (tmp, arcname)
|
||||
finally:
|
||||
self._stream_open = False
|
||||
|
||||
def _finalize(self) -> None:
|
||||
assert self._zip is not None
|
||||
if self._pending_manifest is not None:
|
||||
tmp, arcname = self._pending_manifest
|
||||
self._ensure_dirs(arcname)
|
||||
self._zip.write(tmp, arcname=arcname)
|
||||
tmp.unlink(missing_ok=True)
|
||||
self._pending_manifest = None
|
||||
self._zip.close()
|
||||
self._zip = None
|
||||
if self._delete:
|
||||
self._wipe_destination()
|
||||
self._tmp_path.replace(self._zip_path)
|
||||
|
||||
def _wipe_destination(self) -> None:
|
||||
skip = {self._zip_path.resolve(), self._tmp_path.resolve()}
|
||||
for item in self._target.glob("*"):
|
||||
if item.resolve() in skip:
|
||||
continue
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
|
||||
def _abort(self) -> None:
|
||||
if self._zip is not None:
|
||||
self._zip.close()
|
||||
self._zip = None
|
||||
self._tmp_path.unlink(missing_ok=True)
|
||||
if self._pending_manifest is not None:
|
||||
self._pending_manifest[0].unlink(missing_ok=True)
|
||||
self._pending_manifest = None
|
||||
+29
-50
@@ -39,7 +39,6 @@ from guardian.utils import get_user_obj_perms_model
|
||||
from rest_framework import serializers
|
||||
from rest_framework.filters import BaseFilterBackend
|
||||
from rest_framework.filters import OrderingFilter
|
||||
from rest_framework_guardian.filters import ObjectPermissionsFilter
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
@@ -51,7 +50,7 @@ from documents.models import ShareLink
|
||||
from documents.models import ShareLinkBundle
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -725,9 +724,13 @@ class CustomFieldQueryParser:
|
||||
)
|
||||
|
||||
# First we look up reverse links from the requested documents.
|
||||
# Scoped to this specific field (not just any document link field) and
|
||||
# excluding unset instances, which have a null value_document_ids and
|
||||
# are equivalent to having no reverse link at all.
|
||||
links = CustomFieldInstance.objects.filter(
|
||||
document_id__in=value,
|
||||
field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
|
||||
field=custom_field,
|
||||
value_document_ids__isnull=False,
|
||||
)
|
||||
|
||||
# Check if any of the requested IDs are missing.
|
||||
@@ -1024,59 +1027,35 @@ class PaperlessTaskFilterSet(FilterSet):
|
||||
return queryset.exclude(status__in=PaperlessTask.COMPLETE_STATUSES)
|
||||
|
||||
|
||||
class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
|
||||
class PermittedObjectsFilter(BaseFilterBackend):
|
||||
"""
|
||||
A filter backend that limits results to those where the requesting user
|
||||
has read object level permissions, owns the objects, or objects without
|
||||
an owner (for backwards compat)
|
||||
Filters a queryset down to objects the requesting user owns, are
|
||||
unowned, or (when ``include_granted`` is True) has an explicit
|
||||
user/group guardian permission on. Backed by ``permitted_object_ids``
|
||||
-- a single ``id__in`` subquery, not a join -- so it can't produce
|
||||
duplicate rows even when the base queryset already carries independent
|
||||
joins (e.g. multi-value ``tags__id__all`` filtering), and stays
|
||||
index-friendly at scale instead of falling back to guardian's
|
||||
varchar-cast join.
|
||||
|
||||
Set ``include_granted = False`` on a subclass for endpoints that
|
||||
intentionally only show owned/unowned objects regardless of explicit
|
||||
shares (e.g. ``TrashView``).
|
||||
"""
|
||||
|
||||
include_granted: bool = True
|
||||
perm_codename: str | None = None
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
objects_with_perms = super().filter_queryset(request, queryset, view)
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_with_perms | objects_owned | objects_unowned
|
||||
|
||||
|
||||
class DocumentPermissionsFilter(BaseFilterBackend):
|
||||
"""
|
||||
A filter backend limiting Document results to those the requesting user
|
||||
owns, are unowned, or has explicit (user- or group-level) view
|
||||
permission on.
|
||||
|
||||
Unlike ``ObjectOwnedOrGrantedPermissionsFilter``, this does not build an
|
||||
``objects_with_perms | objects_owned | objects_unowned`` union of
|
||||
querysets derived from the same base queryset. When that base queryset
|
||||
already carries independent joins on a multi-valued relation (e.g. two
|
||||
separate joins from ``tags__id__all`` filtering on two tags), each
|
||||
OR-ed branch can end up pairing those joins' aliases differently,
|
||||
letting more than one row out of the join's cross product satisfy the
|
||||
combined WHERE -- returning the same document more than once. Filtering
|
||||
via a single ``id__in`` against ``permitted_document_ids`` (a plain
|
||||
subquery, not a join) sidesteps that entirely and is also cheaper than
|
||||
guardian's join-based permission check.
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
return queryset.filter(id__in=permitted_document_ids(request.user))
|
||||
|
||||
|
||||
class ObjectOwnedPermissionsFilter(ObjectPermissionsFilter):
|
||||
"""
|
||||
A filter backend that limits results to those where the requesting user
|
||||
owns the objects or objects without an owner (for backwards compat)
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_owned | objects_unowned
|
||||
if not self.include_granted:
|
||||
return queryset.filter(Q(owner=request.user) | Q(owner__isnull=True))
|
||||
model = queryset.model
|
||||
perm = self.perm_codename or f"view_{model._meta.model_name}"
|
||||
return queryset.filter(
|
||||
id__in=permitted_object_ids(request.user, model, perm),
|
||||
)
|
||||
|
||||
|
||||
class DocumentsOrderingFilter(OrderingFilter):
|
||||
|
||||
@@ -632,8 +632,20 @@ class Command(BaseCommand):
|
||||
# Process each change
|
||||
for change_type, path in changes:
|
||||
path = Path(path).resolve()
|
||||
if change_type == Change.deleted:
|
||||
# Consumed (or otherwise removed); a later file
|
||||
# reusing this name must not be skipped as
|
||||
# already-queued.
|
||||
queued.discard(path)
|
||||
if not path.is_file():
|
||||
continue
|
||||
if path in queued:
|
||||
# Already queued and awaiting consumption; a stray
|
||||
# event (NAS metadata touch, AV scan, etc.) while
|
||||
# the file sits on disk mid-consumption must not
|
||||
# cause it to be queued a second time (GH #13511).
|
||||
logger.debug(f"Ignoring event for queued file: {path}")
|
||||
continue
|
||||
logger.debug(f"Event: {change_type.name} for {path}")
|
||||
tracker.track(path, change_type)
|
||||
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from itertools import islice
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -19,7 +15,6 @@ from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core import serializers
|
||||
from django.core.management.base import CommandError
|
||||
from django.core.serializers.json import DjangoJSONEncoder
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
@@ -34,7 +29,10 @@ if TYPE_CHECKING:
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.models import LogEntry
|
||||
|
||||
from documents.file_handling import delete_empty_directories
|
||||
from documents.export.sinks import DirectoryExportSink
|
||||
from documents.export.sinks import ExportSink
|
||||
from documents.export.sinks import StreamingManifestWriter
|
||||
from documents.export.sinks import ZipExportSink
|
||||
from documents.file_handling import generate_filename
|
||||
from documents.management.commands.base import PaperlessCommand
|
||||
from documents.management.commands.mixins import CryptMixin
|
||||
@@ -60,8 +58,7 @@ from documents.settings import EXPORTER_ARCHIVE_NAME
|
||||
from documents.settings import EXPORTER_FILE_NAME
|
||||
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
|
||||
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import QuerySetStream
|
||||
from paperless import version
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_mail.models import MailAccount
|
||||
@@ -84,87 +81,6 @@ def serialize_queryset_batched(
|
||||
yield serializers.serialize("python", chunk)
|
||||
|
||||
|
||||
class StreamingManifestWriter:
|
||||
"""Incrementally writes a JSON array to a file, one record at a time.
|
||||
|
||||
Writes to <target>.tmp first; on close(), optionally BLAKE2b-compares
|
||||
with the existing file (--compare-json) and renames or discards accordingly.
|
||||
On exception, discard() deletes the tmp file and leaves the original intact.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Path,
|
||||
*,
|
||||
compare_json: bool = False,
|
||||
files_in_export_dir: "set[Path] | None" = None,
|
||||
) -> None:
|
||||
self._path = path.resolve()
|
||||
self._tmp_path = self._path.with_suffix(self._path.suffix + ".tmp")
|
||||
self._compare_json = compare_json
|
||||
self._files_in_export_dir: set[Path] = (
|
||||
files_in_export_dir if files_in_export_dir is not None else set()
|
||||
)
|
||||
self._file = None
|
||||
self._first = True
|
||||
|
||||
def open(self) -> None:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._file = self._tmp_path.open("w", encoding="utf-8")
|
||||
self._file.write("[")
|
||||
self._first = True
|
||||
|
||||
def write_record(self, record: dict) -> None:
|
||||
if not self._first:
|
||||
self._file.write(",\n")
|
||||
else:
|
||||
self._first = False
|
||||
self._file.write(
|
||||
json.dumps(record, cls=DjangoJSONEncoder, indent=2, ensure_ascii=False),
|
||||
)
|
||||
|
||||
def write_batch(self, records: list[dict]) -> None:
|
||||
for record in records:
|
||||
self.write_record(record)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._file is None:
|
||||
return
|
||||
self._file.write("\n]")
|
||||
self._file.close()
|
||||
self._file = None
|
||||
self._finalize()
|
||||
|
||||
def discard(self) -> None:
|
||||
if self._file is not None:
|
||||
self._file.close()
|
||||
self._file = None
|
||||
if self._tmp_path.exists():
|
||||
self._tmp_path.unlink()
|
||||
|
||||
def _finalize(self) -> None:
|
||||
"""Compare with existing file (if --compare-json) then rename or discard tmp."""
|
||||
if self._path in self._files_in_export_dir:
|
||||
self._files_in_export_dir.remove(self._path)
|
||||
if self._compare_json:
|
||||
existing_hash = hashlib.blake2b(self._path.read_bytes()).hexdigest()
|
||||
new_hash = hashlib.blake2b(self._tmp_path.read_bytes()).hexdigest()
|
||||
if existing_hash == new_hash:
|
||||
self._tmp_path.unlink()
|
||||
return
|
||||
self._tmp_path.rename(self._path)
|
||||
|
||||
def __enter__(self) -> "StreamingManifestWriter":
|
||||
self.open()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if exc_type is not None:
|
||||
self.discard()
|
||||
else:
|
||||
self.close()
|
||||
|
||||
|
||||
class Command(CryptMixin, PaperlessCommand):
|
||||
help = (
|
||||
"Decrypt and rename all files in our collection into a given target "
|
||||
@@ -314,20 +230,13 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
self.passphrase: str | None = options.get("passphrase")
|
||||
self.batch_size: int = options["batch_size"]
|
||||
|
||||
self.files_in_export_dir: set[Path] = set()
|
||||
self.exported_files: set[str] = set()
|
||||
|
||||
# If zipping, save the original target for later and
|
||||
# get a temporary directory for the target instead
|
||||
temp_dir = None
|
||||
self.original_target = self.target
|
||||
if self.zip_export:
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
temp_dir = tempfile.TemporaryDirectory(
|
||||
dir=settings.SCRATCH_DIR,
|
||||
prefix="paperless-export",
|
||||
if self.zip_export and (self.compare_checksums or self.compare_json):
|
||||
raise CommandError(
|
||||
"--compare-checksums and --compare-json have no effect when "
|
||||
"used with --zip",
|
||||
)
|
||||
self.target = Path(temp_dir.name).resolve()
|
||||
|
||||
if not self.target.exists():
|
||||
raise CommandError("That path doesn't exist")
|
||||
@@ -338,33 +247,28 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
if not os.access(self.target, os.W_OK):
|
||||
raise CommandError("That path doesn't appear to be writable")
|
||||
|
||||
try:
|
||||
# Prevent any ongoing changes in the documents
|
||||
with FileLock(settings.MEDIA_LOCK):
|
||||
self.dump()
|
||||
sink: ExportSink
|
||||
if self.zip_export:
|
||||
sink = ZipExportSink(
|
||||
self.target,
|
||||
options["zip_name"],
|
||||
delete=self.delete,
|
||||
)
|
||||
else:
|
||||
sink = DirectoryExportSink(
|
||||
self.target,
|
||||
compare_checksums=self.compare_checksums,
|
||||
compare_json=self.compare_json,
|
||||
delete=self.delete,
|
||||
)
|
||||
|
||||
# We've written everything to the temporary directory in this case,
|
||||
# now make an archive in the original target, with all files stored
|
||||
if self.zip_export and temp_dir is not None:
|
||||
shutil.make_archive(
|
||||
self.original_target / options["zip_name"],
|
||||
format="zip",
|
||||
root_dir=temp_dir.name,
|
||||
)
|
||||
# Prevent any ongoing changes in the documents while exporting
|
||||
with FileLock(settings.MEDIA_LOCK), sink:
|
||||
self.dump(sink)
|
||||
|
||||
finally:
|
||||
# Always cleanup the temporary directory, if one was created
|
||||
if self.zip_export and temp_dir is not None:
|
||||
temp_dir.cleanup()
|
||||
|
||||
def dump(self) -> None:
|
||||
# 1. Take a snapshot of what files exist in the current export folder
|
||||
for x in self.target.glob("**/*"):
|
||||
if x.is_file():
|
||||
self.files_in_export_dir.add(x.resolve())
|
||||
|
||||
# 2. Create manifest, containing all correspondents, types, tags, storage paths
|
||||
# note, documents and ui_settings
|
||||
def dump(self, sink: ExportSink) -> None:
|
||||
# 1. Create manifest, containing all correspondents, types, tags, storage
|
||||
# paths, note, documents and ui_settings
|
||||
_excluded_usernames = ["consumer", "AnonymousUser"]
|
||||
manifest_key_to_object_query: dict[str, QuerySet[Any]] = {
|
||||
"correspondents": Correspondent.objects.all(),
|
||||
@@ -427,13 +331,9 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
|
||||
document_manifest: list[dict] = []
|
||||
share_link_bundle_manifest: list[dict] = []
|
||||
manifest_path = (self.target / "manifest.json").resolve()
|
||||
|
||||
with StreamingManifestWriter(
|
||||
manifest_path,
|
||||
compare_json=self.compare_json,
|
||||
files_in_export_dir=self.files_in_export_dir,
|
||||
) as writer:
|
||||
with sink.stream("manifest.json") as handle:
|
||||
writer = StreamingManifestWriter(handle)
|
||||
with transaction.atomic():
|
||||
for key, qs in manifest_key_to_object_query.items():
|
||||
if key == "documents":
|
||||
@@ -469,9 +369,6 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
self._encrypt_record_inline(record)
|
||||
writer.write_batch(batch)
|
||||
|
||||
document_map: dict[int, Document] = {
|
||||
d.pk: d for d in Document.global_objects.order_by("id")
|
||||
}
|
||||
share_link_bundle_map: dict[int, ShareLinkBundle] = {
|
||||
b.pk: b
|
||||
for b in ShareLinkBundle.objects.order_by("id").prefetch_related(
|
||||
@@ -479,84 +376,72 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
)
|
||||
}
|
||||
|
||||
# 3. Export files from each document
|
||||
for index, document_dict in enumerate(
|
||||
self.track(
|
||||
document_manifest,
|
||||
description="Exporting documents...",
|
||||
total=len(document_manifest),
|
||||
),
|
||||
# 2. Export files from each document
|
||||
# document_manifest and this stream are both ordered by id from the
|
||||
# same underlying rows, so zip them in lockstep instead of building
|
||||
# a dict of every Document instance up front (QuerySetStream keeps
|
||||
# only one batch of documents resident at a time).
|
||||
documents_stream = QuerySetStream(
|
||||
Document.global_objects.order_by("id"),
|
||||
chunk_size=self.batch_size,
|
||||
)
|
||||
for document_dict, document in self.track(
|
||||
zip(document_manifest, documents_stream, strict=True),
|
||||
description="Exporting documents...",
|
||||
total=len(document_manifest),
|
||||
):
|
||||
document = document_map[document_dict["pk"]]
|
||||
# Both document_manifest and documents_stream come from the same
|
||||
# Document.global_objects.order_by("id") query, taken while
|
||||
# MEDIA_LOCK is held, so this should be unreachable -- it guards
|
||||
# against silent data corruption if that invariant ever breaks.
|
||||
if document.pk != document_dict["pk"]: # pragma: no cover
|
||||
raise CommandError(
|
||||
"Document export ordering mismatch: expected "
|
||||
f"pk={document_dict['pk']}, got pk={document.pk}. "
|
||||
"Documents may have changed during export.",
|
||||
)
|
||||
|
||||
# 3.1. generate a unique filename
|
||||
# generate a unique filename, then the arcnames for its files
|
||||
base_name = self.generate_base_name(document)
|
||||
|
||||
# 3.2. write filenames into manifest
|
||||
original_target, thumbnail_target, archive_target = (
|
||||
original_arc, thumbnail_arc, archive_arc = (
|
||||
self.generate_document_targets(document, base_name, document_dict)
|
||||
)
|
||||
|
||||
# 3.3. write files to target folder
|
||||
if not self.data_only:
|
||||
self.copy_document_files(
|
||||
document,
|
||||
original_target,
|
||||
thumbnail_target,
|
||||
archive_target,
|
||||
sink,
|
||||
original_arc,
|
||||
thumbnail_arc,
|
||||
archive_arc,
|
||||
)
|
||||
|
||||
if self.split_manifest:
|
||||
self._write_split_manifest(document_dict, document, base_name)
|
||||
self._write_split_manifest(sink, document_dict, document, base_name)
|
||||
else:
|
||||
writer.write_record(document_dict)
|
||||
|
||||
for bundle_dict in share_link_bundle_manifest:
|
||||
bundle = share_link_bundle_map[bundle_dict["pk"]]
|
||||
|
||||
bundle_target = self.generate_share_link_bundle_target(
|
||||
bundle_arc = self.generate_share_link_bundle_target(
|
||||
bundle,
|
||||
bundle_dict,
|
||||
)
|
||||
|
||||
if not self.data_only and bundle_target is not None:
|
||||
self.copy_share_link_bundle_file(bundle, bundle_target)
|
||||
|
||||
if not self.data_only and bundle_arc is not None:
|
||||
self.copy_share_link_bundle_file(bundle, sink, bundle_arc)
|
||||
writer.write_record(bundle_dict)
|
||||
|
||||
# 4.2 write version information to target folder
|
||||
extra_metadata_path = (self.target / "metadata.json").resolve()
|
||||
writer.close()
|
||||
|
||||
# 3. Write version (and crypto params) to metadata.json
|
||||
# Django stores most crypto values in the field itself; we store
|
||||
# them once here for the whole export
|
||||
metadata: dict[str, str | int | dict[str, str | int]] = {
|
||||
"version": version.__full_version_str__,
|
||||
}
|
||||
|
||||
# 4.2.1 If needed, write the crypto values into the metadata
|
||||
# Django stores most of these in the field itself, we store them once here
|
||||
if self.passphrase:
|
||||
metadata.update(self.get_crypt_params())
|
||||
|
||||
self.check_and_write_json(
|
||||
metadata,
|
||||
extra_metadata_path,
|
||||
)
|
||||
|
||||
if self.delete:
|
||||
# 5. Remove files which we did not explicitly export in this run
|
||||
if not self.zip_export:
|
||||
for f in self.files_in_export_dir:
|
||||
f.unlink()
|
||||
|
||||
delete_empty_directories(
|
||||
f.parent,
|
||||
self.target,
|
||||
)
|
||||
else:
|
||||
# 5. Remove anything in the original location (before moving the zip)
|
||||
for item in self.original_target.glob("*"):
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
sink.add_json(metadata, "metadata.json")
|
||||
|
||||
def generate_base_name(self, document: Document) -> Path:
|
||||
"""
|
||||
@@ -584,73 +469,69 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
document: Document,
|
||||
base_name: Path,
|
||||
document_dict: dict,
|
||||
) -> tuple[Path, Path | None, Path | None]:
|
||||
) -> tuple[str, str | None, str | None]:
|
||||
"""
|
||||
Generates the targets for a given document, including the original file, archive file and thumbnail (depending on settings).
|
||||
Generates the relative POSIX arcnames for a document's original, thumbnail
|
||||
and archive files (depending on settings), and records them in the manifest.
|
||||
"""
|
||||
original_name = base_name
|
||||
if self.use_folder_prefix:
|
||||
original_name = Path("originals") / original_name
|
||||
original_target = (self.target / original_name).resolve()
|
||||
document_dict[EXPORTER_FILE_NAME] = str(original_name)
|
||||
original_arc = original_name.as_posix()
|
||||
document_dict[EXPORTER_FILE_NAME] = original_arc
|
||||
|
||||
if not self.no_thumbnail:
|
||||
thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp")
|
||||
if self.use_folder_prefix:
|
||||
thumbnail_name = Path("thumbnails") / thumbnail_name
|
||||
thumbnail_target = (self.target / thumbnail_name).resolve()
|
||||
document_dict[EXPORTER_THUMBNAIL_NAME] = str(thumbnail_name)
|
||||
thumbnail_arc = thumbnail_name.as_posix()
|
||||
document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_arc
|
||||
else:
|
||||
thumbnail_target = None
|
||||
thumbnail_arc = None
|
||||
|
||||
if not self.no_archive and document.has_archive_version:
|
||||
archive_name = base_name.parent / (base_name.stem + "-archive.pdf")
|
||||
if self.use_folder_prefix:
|
||||
archive_name = Path("archive") / archive_name
|
||||
archive_target = (self.target / archive_name).resolve()
|
||||
document_dict[EXPORTER_ARCHIVE_NAME] = str(archive_name)
|
||||
archive_arc = archive_name.as_posix()
|
||||
document_dict[EXPORTER_ARCHIVE_NAME] = archive_arc
|
||||
else:
|
||||
archive_target = None
|
||||
archive_arc = None
|
||||
|
||||
return original_target, thumbnail_target, archive_target
|
||||
return original_arc, thumbnail_arc, archive_arc
|
||||
|
||||
def copy_document_files(
|
||||
self,
|
||||
document: Document,
|
||||
original_target: Path,
|
||||
thumbnail_target: Path | None,
|
||||
archive_target: Path | None,
|
||||
sink: ExportSink,
|
||||
original_arc: str,
|
||||
thumbnail_arc: str | None,
|
||||
archive_arc: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
Copies files from the document storage location to the specified target location.
|
||||
|
||||
If the document is encrypted, the files are decrypted before copying them to the target location.
|
||||
Hands the document's files to the sink (original, thumbnail, archive).
|
||||
"""
|
||||
self.check_and_copy(
|
||||
document.source_path,
|
||||
document.checksum,
|
||||
original_target,
|
||||
)
|
||||
sink.add_file(document.source_path, original_arc, checksum=document.checksum)
|
||||
|
||||
if thumbnail_target:
|
||||
self.check_and_copy(document.thumbnail_path, None, thumbnail_target)
|
||||
if thumbnail_arc:
|
||||
sink.add_file(document.thumbnail_path, thumbnail_arc)
|
||||
|
||||
if archive_target:
|
||||
if archive_arc:
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(document.archive_path, Path)
|
||||
self.check_and_copy(
|
||||
sink.add_file(
|
||||
document.archive_path,
|
||||
document.archive_checksum,
|
||||
archive_target,
|
||||
archive_arc,
|
||||
checksum=document.archive_checksum,
|
||||
)
|
||||
|
||||
def generate_share_link_bundle_target(
|
||||
self,
|
||||
bundle: ShareLinkBundle,
|
||||
bundle_dict: dict,
|
||||
) -> Path | None:
|
||||
) -> str | None:
|
||||
"""
|
||||
Generates the export target for a share link bundle file, when present.
|
||||
Generates the relative POSIX arcname for a share link bundle file, if any.
|
||||
"""
|
||||
if not bundle.file_path:
|
||||
return None
|
||||
@@ -666,25 +547,22 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
bundle_dict["fields"]["file_path"] = portable_bundle_path.as_posix()
|
||||
bundle_dict[EXPORTER_SHARE_LINK_BUNDLE_NAME] = export_bundle_path.as_posix()
|
||||
|
||||
return (self.target / export_bundle_path).resolve()
|
||||
return export_bundle_path.as_posix()
|
||||
|
||||
def copy_share_link_bundle_file(
|
||||
self,
|
||||
bundle: ShareLinkBundle,
|
||||
bundle_target: Path,
|
||||
sink: ExportSink,
|
||||
bundle_arc: str,
|
||||
) -> None:
|
||||
"""
|
||||
Copies a share link bundle ZIP into the export directory.
|
||||
Hands a share link bundle ZIP to the sink.
|
||||
"""
|
||||
bundle_source_path = bundle.absolute_file_path
|
||||
if bundle_source_path is None:
|
||||
raise FileNotFoundError(f"Share link bundle {bundle.pk} has no file path")
|
||||
|
||||
self.check_and_copy(
|
||||
bundle_source_path,
|
||||
None,
|
||||
bundle_target,
|
||||
)
|
||||
sink.add_file(bundle_source_path, bundle_arc)
|
||||
|
||||
def _encrypt_record_inline(self, record: dict) -> None:
|
||||
"""Encrypt sensitive fields in a single record, if passphrase is set."""
|
||||
@@ -700,6 +578,7 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
|
||||
def _write_split_manifest(
|
||||
self,
|
||||
sink: ExportSink,
|
||||
document_dict: dict,
|
||||
document: Document,
|
||||
base_name: Path,
|
||||
@@ -721,81 +600,4 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
manifest_name = base_name.with_name(f"{base_name.stem}-manifest.json")
|
||||
if self.use_folder_prefix:
|
||||
manifest_name = Path("json") / manifest_name
|
||||
manifest_name = (self.target / manifest_name).resolve()
|
||||
manifest_name.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.check_and_write_json(content, manifest_name)
|
||||
|
||||
def check_and_write_json(
|
||||
self,
|
||||
content: list[dict] | dict,
|
||||
target: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Writes the source content to the target json file.
|
||||
If --compare-json arg was used, don't write to target file if
|
||||
the file exists and checksum is identical to content checksum.
|
||||
This preserves the file timestamps when no changes are made.
|
||||
"""
|
||||
|
||||
target = target.resolve()
|
||||
perform_write = True
|
||||
if target in self.files_in_export_dir:
|
||||
self.files_in_export_dir.remove(target)
|
||||
if self.compare_json:
|
||||
target_checksum = hashlib.blake2b(target.read_bytes()).hexdigest()
|
||||
src_str = json.dumps(
|
||||
content,
|
||||
cls=DjangoJSONEncoder,
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
src_checksum = hashlib.blake2b(src_str.encode("utf-8")).hexdigest()
|
||||
if src_checksum == target_checksum:
|
||||
perform_write = False
|
||||
|
||||
if perform_write:
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
content,
|
||||
cls=DjangoJSONEncoder,
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def check_and_copy(
|
||||
self,
|
||||
source: Path,
|
||||
source_checksum: str | None,
|
||||
target: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Copies the source to the target, if target doesn't exist or the target doesn't seem to match
|
||||
the source attributes
|
||||
"""
|
||||
|
||||
target = target.resolve()
|
||||
if target in self.files_in_export_dir:
|
||||
self.files_in_export_dir.remove(target)
|
||||
|
||||
perform_copy = False
|
||||
|
||||
if target.exists():
|
||||
source_stat = source.stat()
|
||||
target_stat = target.stat()
|
||||
if self.compare_checksums and source_checksum:
|
||||
target_checksum = compute_checksum(target)
|
||||
perform_copy = target_checksum != source_checksum
|
||||
elif (
|
||||
source_stat.st_mtime != target_stat.st_mtime
|
||||
or source_stat.st_size != target_stat.st_size
|
||||
):
|
||||
perform_copy = True
|
||||
else:
|
||||
# Copy if it does not exist
|
||||
perform_copy = True
|
||||
|
||||
if perform_copy:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy_file_with_basic_stats(source, target)
|
||||
sink.add_json(content, manifest_name.as_posix())
|
||||
|
||||
+14
-19
@@ -19,7 +19,7 @@ from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.regex import safe_regex_search
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -55,10 +55,8 @@ def match_correspondents(document: Document, classifier: DocumentClassifier, use
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
correspondents = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_correspondent",
|
||||
Correspondent,
|
||||
correspondents = Correspondent.objects.filter(
|
||||
id__in=permitted_object_ids(user, Correspondent, "view_correspondent"),
|
||||
)
|
||||
else:
|
||||
correspondents = Correspondent.objects.all()
|
||||
@@ -86,10 +84,8 @@ def match_document_types(document: Document, classifier: DocumentClassifier, use
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
document_types = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_documenttype",
|
||||
DocumentType,
|
||||
document_types = DocumentType.objects.filter(
|
||||
id__in=permitted_object_ids(user, DocumentType, "view_documenttype"),
|
||||
)
|
||||
else:
|
||||
document_types = DocumentType.objects.all()
|
||||
@@ -116,7 +112,9 @@ def match_tags(document: Document, classifier: DocumentClassifier, user=None):
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
tags = get_objects_for_user_owner_aware(user, "documents.view_tag", Tag)
|
||||
tags = Tag.objects.filter(
|
||||
id__in=permitted_object_ids(user, Tag, "view_tag"),
|
||||
)
|
||||
else:
|
||||
tags = Tag.objects.all()
|
||||
|
||||
@@ -145,10 +143,8 @@ def match_storage_paths(document: Document, classifier: DocumentClassifier, user
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
storage_paths = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_storagepath",
|
||||
StoragePath,
|
||||
storage_paths = StoragePath.objects.filter(
|
||||
id__in=permitted_object_ids(user, StoragePath, "view_storagepath"),
|
||||
)
|
||||
else:
|
||||
storage_paths = StoragePath.objects.all()
|
||||
@@ -300,7 +296,7 @@ def consumable_document_matches_workflow(
|
||||
]:
|
||||
reason = (
|
||||
f"Document source {document.source.name} not in"
|
||||
f" {[DocumentSource(int(x)).name for x in trigger.sources]}",
|
||||
f" {[DocumentSource(int(x)).name for x in trigger.sources]}"
|
||||
)
|
||||
trigger_matched = False
|
||||
|
||||
@@ -310,8 +306,7 @@ def consumable_document_matches_workflow(
|
||||
and document.mailrule_id != trigger.filter_mailrule.pk
|
||||
):
|
||||
reason = (
|
||||
f"Document mail rule {document.mailrule_id}"
|
||||
f" != {trigger.filter_mailrule.pk}",
|
||||
f"Document mail rule {document.mailrule_id} != {trigger.filter_mailrule.pk}"
|
||||
)
|
||||
trigger_matched = False
|
||||
|
||||
@@ -326,7 +321,7 @@ def consumable_document_matches_workflow(
|
||||
):
|
||||
reason = (
|
||||
f"Document filename {document.original_file.name} does not match"
|
||||
f" {trigger.filter_filename.lower()}",
|
||||
f" {trigger.filter_filename.lower()}"
|
||||
)
|
||||
trigger_matched = False
|
||||
|
||||
@@ -349,7 +344,7 @@ def consumable_document_matches_workflow(
|
||||
):
|
||||
reason = (
|
||||
f"Document path {document.original_file}"
|
||||
f" does not match {trigger.filter_path}",
|
||||
f" does not match {trigger.filter_path}"
|
||||
)
|
||||
trigger_matched = False
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from django.contrib.contenttypes.models import ContentType
|
||||
from django.db.models import Case
|
||||
from django.db.models import Count
|
||||
from django.db.models import IntegerField
|
||||
from django.db.models import Model
|
||||
from django.db.models import Q
|
||||
from django.db.models import QuerySet
|
||||
from django.db.models import Value
|
||||
@@ -163,47 +164,79 @@ def set_permissions_for_object(
|
||||
)
|
||||
|
||||
|
||||
def permitted_document_ids(user):
|
||||
def permitted_object_ids(
|
||||
user: User | None,
|
||||
model: type[Model],
|
||||
perm: str,
|
||||
*,
|
||||
include_deleted: bool = False,
|
||||
) -> QuerySet[int]:
|
||||
"""
|
||||
Return a queryset of document IDs the user may view, limited to non-deleted
|
||||
documents. This intentionally avoids ``get_objects_for_user`` to keep the
|
||||
subquery small and index-friendly.
|
||||
Generic version of ``permitted_document_ids`` for any model with an
|
||||
``owner`` field and guardian object-level permissions. ``include_deleted``
|
||||
only has an effect for models exposing a ``global_objects``/``deleted_at``
|
||||
soft-delete pattern (currently only ``Document``); for every other model
|
||||
it is accepted but has no effect, since those models have no soft-delete
|
||||
concept.
|
||||
"""
|
||||
|
||||
base_docs = Document.objects.filter(deleted_at__isnull=True).only("id", "owner")
|
||||
has_soft_delete = hasattr(model, "global_objects")
|
||||
manager = (
|
||||
model.global_objects if include_deleted and has_soft_delete else model.objects
|
||||
)
|
||||
base_qs = manager.all().only("id", "owner")
|
||||
|
||||
if user is None or not getattr(user, "is_authenticated", False):
|
||||
# Just Anonymous user e.g. for drf-spectacular
|
||||
return base_docs.filter(owner__isnull=True).values_list("id", flat=True)
|
||||
return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
|
||||
|
||||
if getattr(user, "is_superuser", False):
|
||||
return base_docs.values_list("id", flat=True)
|
||||
return base_qs.values_list("id", flat=True)
|
||||
|
||||
document_ct = ContentType.objects.get_for_model(Document)
|
||||
# Guardian's UserObjectPermission/GroupObjectPermission always store a bare
|
||||
# codename, but has_perm()-style callers commonly pass the qualified
|
||||
# "app_label.codename" form. content_type already disambiguates the
|
||||
# codename, so just drop any prefix rather than silently under-permitting.
|
||||
perm = perm.rsplit(".", 1)[-1]
|
||||
|
||||
content_type = ContentType.objects.get_for_model(model)
|
||||
perm_filter = {
|
||||
"permission__codename": "view_document",
|
||||
"permission__content_type": document_ct,
|
||||
"permission__codename": perm,
|
||||
"permission__content_type": content_type,
|
||||
}
|
||||
|
||||
user_perm_docs = (
|
||||
user_perm_ids = (
|
||||
UserObjectPermission.objects.filter(user=user, **perm_filter)
|
||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||
.values_list("object_pk_int", flat=True)
|
||||
)
|
||||
|
||||
group_perm_docs = (
|
||||
group_perm_ids = (
|
||||
GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
|
||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||
.values_list("object_pk_int", flat=True)
|
||||
)
|
||||
permitted_ids = user_perm_ids.union(group_perm_ids)
|
||||
|
||||
permitted_documents = user_perm_docs.union(group_perm_docs)
|
||||
|
||||
return base_docs.filter(
|
||||
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_documents),
|
||||
return base_qs.filter(
|
||||
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
|
||||
).values_list("id", flat=True)
|
||||
|
||||
|
||||
def permitted_document_ids(
|
||||
user: User | None,
|
||||
*,
|
||||
perm: str = "view_document",
|
||||
include_deleted: bool = False,
|
||||
) -> QuerySet[int]:
|
||||
"""
|
||||
Document-specific convenience wrapper around ``permitted_object_ids``.
|
||||
Return a queryset of document IDs the user has ``perm`` on (default
|
||||
``"view_document"``). By default limited to non-deleted documents; pass
|
||||
``include_deleted=True`` for callers that need to check permission on
|
||||
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
||||
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
||||
"""
|
||||
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
|
||||
|
||||
|
||||
def get_document_count_filter_for_user(user, related_name: str = "documents"):
|
||||
"""
|
||||
Return the Q object used to filter document counts for the given user.
|
||||
@@ -326,6 +359,13 @@ def get_objects_for_user_owner_aware(
|
||||
"""
|
||||
Returns objects the user owns, are unowned, or has explicit perms.
|
||||
When include_deleted is True, soft-deleted items are also included.
|
||||
|
||||
Legacy slow path (guardian-backed, O(n) style permission resolution).
|
||||
Most queryset-filtering call sites have migrated onto
|
||||
``PermittedObjectsFilter``/``permitted_object_ids()``, but this function
|
||||
is kept because production callers still remain. Several callers remain
|
||||
across ``documents/``, ``paperless_mail/``, and ``paperless_ai/`` --
|
||||
grep for this function name before removing it.
|
||||
"""
|
||||
manager = (
|
||||
Model.global_objects
|
||||
@@ -345,6 +385,15 @@ def get_objects_for_user_owner_aware(
|
||||
|
||||
|
||||
def has_perms_owner_aware(user, perms, obj):
|
||||
"""
|
||||
Legacy slow path (guardian-backed) single-object permission check.
|
||||
|
||||
The queryset-filtering side of this migrated onto
|
||||
``PermittedObjectsFilter``/``permitted_object_ids()``, but this
|
||||
single-object check still has many production callers. Several callers
|
||||
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
||||
-- grep for this function name before removing it.
|
||||
"""
|
||||
checker = ObjectPermissionChecker(user)
|
||||
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
||||
|
||||
|
||||
@@ -139,31 +139,38 @@ def _simple_query_tokens(raw_query: str) -> list[str]:
|
||||
return simple_search_tokens(raw_query)
|
||||
|
||||
|
||||
def _build_simple_field_query(
|
||||
def _build_simple_token_query(
|
||||
index: tantivy.Index,
|
||||
field: str,
|
||||
tokens: list[str],
|
||||
fields: list[str],
|
||||
token: str,
|
||||
*,
|
||||
allow_infix: bool,
|
||||
) -> tantivy.Query:
|
||||
patterns = []
|
||||
for idx, token in enumerate(tokens):
|
||||
escaped = regex.escape(token)
|
||||
# For multi-token substring search, only the first token can begin mid-word.
|
||||
# Later tokens follow a whitespace boundary in the original query, so anchor
|
||||
# them to the start of the next indexed token to reduce false positives like
|
||||
# matching "Z-Berichte 16" for the query "Z-Berichte 6".
|
||||
if idx == 0:
|
||||
patterns.append(f".*{escaped}.*")
|
||||
else:
|
||||
patterns.append(f"{escaped}.*")
|
||||
if len(patterns) == 1:
|
||||
query = tantivy.Query.regex_query(index.schema, field, patterns[0])
|
||||
else:
|
||||
query = tantivy.Query.regex_phrase_query(index.schema, field, patterns)
|
||||
escaped = regex.escape(token)
|
||||
# The simple analyzer keeps punctuation inside whitespace-delimited terms.
|
||||
# Boundary-constrained query tokens may therefore begin either at the indexed
|
||||
# term boundary or after punctuation within a term (for example,
|
||||
# ``medical-history``). This avoids matching a numeric token such as ``6``
|
||||
# in the middle of ``16``.
|
||||
pattern = (
|
||||
f".*{escaped}.*"
|
||||
if allow_infix
|
||||
else (
|
||||
f"({escaped}.*|"
|
||||
rf".*[\x20-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]{escaped}.*)"
|
||||
)
|
||||
)
|
||||
field_queries: list[tuple[tantivy.Occur, tantivy.Query]] = []
|
||||
for field in fields:
|
||||
query = tantivy.Query.regex_query(index.schema, field, pattern)
|
||||
boost = _SIMPLE_FIELD_BOOSTS.get(field, 1.0)
|
||||
if boost > 1.0:
|
||||
query = tantivy.Query.boost_query(query, boost)
|
||||
field_queries.append((tantivy.Occur.Should, query))
|
||||
|
||||
boost = _SIMPLE_FIELD_BOOSTS.get(field, 1.0)
|
||||
if boost > 1.0:
|
||||
return tantivy.Query.boost_query(query, boost)
|
||||
return query
|
||||
if len(field_queries) == 1:
|
||||
return field_queries[0][1]
|
||||
return tantivy.Query.boolean_query(field_queries)
|
||||
|
||||
|
||||
def parse_user_query(
|
||||
@@ -265,10 +272,31 @@ def parse_simple_query(
|
||||
|
||||
clauses: list[tuple[tantivy.Occur, tantivy.Query]] = []
|
||||
if tokens:
|
||||
clauses = [
|
||||
(tantivy.Occur.Should, _build_simple_field_query(index, field, tokens))
|
||||
for field in fields
|
||||
# Match every query token, regardless of its position in the document.
|
||||
# Each token may occur in any of the requested fields, so text mode also
|
||||
# finds documents whose matches are split between title and content.
|
||||
token_queries = [
|
||||
(
|
||||
tantivy.Occur.Must,
|
||||
_build_simple_token_query(
|
||||
index,
|
||||
fields,
|
||||
token,
|
||||
# Preserve historical infix matching for single-token
|
||||
# searches. In multi-token searches, constrain numeric
|
||||
# tokens to boundaries to avoid partial-number overlap.
|
||||
# This depends on token content, not query order.
|
||||
allow_infix=len(tokens) == 1 or not token.isdecimal(),
|
||||
),
|
||||
)
|
||||
for token in tokens
|
||||
]
|
||||
simple_query = (
|
||||
token_queries[0][1]
|
||||
if len(token_queries) == 1
|
||||
else tantivy.Query.boolean_query(token_queries)
|
||||
)
|
||||
clauses.append((tantivy.Occur.Should, simple_query))
|
||||
|
||||
if cjk_fields and _has_cjk(raw_query):
|
||||
cjk_q = _build_cjk_query(index, raw_query, cjk_fields)
|
||||
|
||||
@@ -39,7 +39,6 @@ from drf_spectacular.utils import extend_schema_field
|
||||
from drf_spectacular.utils import extend_schema_serializer
|
||||
from drf_writable_nested.serializers import NestedUpdateMixin
|
||||
from guardian.core import ObjectPermissionChecker
|
||||
from guardian.shortcuts import get_objects_for_user
|
||||
from guardian.shortcuts import get_users_with_perms
|
||||
from guardian.utils import get_group_obj_perms_model
|
||||
from guardian.utils import get_user_obj_perms_model
|
||||
@@ -80,8 +79,8 @@ from documents.models import WorkflowTrigger
|
||||
from documents.parsers import is_mime_type_supported
|
||||
from documents.permissions import get_document_count_filter_for_user
|
||||
from documents.permissions import get_groups_with_only_permission
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.permissions import has_perms_owner_aware
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.regex import validate_regex_pattern
|
||||
from documents.templating.filepath import validate_filepath_template_and_render
|
||||
@@ -865,10 +864,10 @@ def validate_documentlink_targets(user, doc_ids):
|
||||
if user is None:
|
||||
return
|
||||
|
||||
target_documents = Document.objects.filter(id__in=doc_ids).select_related("owner")
|
||||
if not all(
|
||||
has_perms_owner_aware(user, "change_document", document)
|
||||
for document in target_documents
|
||||
if (
|
||||
Document.objects.filter(id__in=doc_ids)
|
||||
.exclude(id__in=permitted_document_ids(user, perm="change_document"))
|
||||
.exists()
|
||||
):
|
||||
raise PermissionDenied(
|
||||
_("Insufficient permissions."),
|
||||
@@ -1011,13 +1010,8 @@ def _get_viewable_duplicates(
|
||||
).exclude(pk=document.pk)
|
||||
duplicates = duplicates.filter(root_document__isnull=True)
|
||||
duplicates = duplicates.order_by("-created")
|
||||
allowed = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_document",
|
||||
Document,
|
||||
include_deleted=True,
|
||||
)
|
||||
return duplicates.filter(id__in=allowed)
|
||||
allowed_ids = permitted_document_ids(user, include_deleted=True)
|
||||
return duplicates.filter(id__in=allowed_ids)
|
||||
|
||||
|
||||
class DuplicateDocumentSummarySerializer(serializers.Serializer[dict[str, Any]]):
|
||||
@@ -1975,6 +1969,8 @@ class BulkEditSerializer(
|
||||
return ownerUser
|
||||
|
||||
def _validate_parameters_set_permissions(self, parameters) -> None:
|
||||
if "set_permissions" not in parameters:
|
||||
raise serializers.ValidationError("set_permissions not specified")
|
||||
parameters["set_permissions"] = self.validate_set_permissions(
|
||||
parameters["set_permissions"],
|
||||
)
|
||||
@@ -2672,13 +2668,8 @@ class TaskSerializerV9(serializers.ModelSerializer[PaperlessTask]):
|
||||
user = request.user
|
||||
qs = Document.global_objects.filter(pk=dup_of)
|
||||
if not user.is_staff:
|
||||
with_perms = get_objects_for_user(
|
||||
user,
|
||||
"documents.view_document",
|
||||
qs,
|
||||
accept_global_perms=False,
|
||||
)
|
||||
qs = with_perms | qs.filter(owner=user) | qs.filter(owner__isnull=True)
|
||||
allowed_ids = permitted_document_ids(user, include_deleted=True)
|
||||
qs = qs.filter(pk__in=allowed_ids)
|
||||
return list(qs.values("id", "title", "deleted_at"))
|
||||
|
||||
|
||||
@@ -3528,8 +3519,6 @@ class StoragePathTestSerializer(SerializerWithPerms):
|
||||
document_field = self.fields.get("document")
|
||||
if not isinstance(document_field, serializers.PrimaryKeyRelatedField):
|
||||
return
|
||||
document_field.queryset = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_document",
|
||||
Document,
|
||||
document_field.queryset = Document.objects.filter(
|
||||
id__in=permitted_document_ids(user),
|
||||
)
|
||||
|
||||
@@ -70,8 +70,7 @@
|
||||
]
|
||||
</script>
|
||||
</pngx-root>
|
||||
<script src="{% static runtime_js %}" defer></script>
|
||||
<script src="{% static polyfills_js %}" defer></script>
|
||||
<script src="{% static main_js %}" defer></script>
|
||||
<script src="{% static polyfills_js %}" type="module"></script>
|
||||
<script src="{% static main_js %}" type="module"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest_django.fixtures import SettingsWrapper
|
||||
|
||||
from documents.export.sinks import DirectoryExportSink
|
||||
from documents.export.sinks import ExportSink
|
||||
from documents.export.sinks import StreamingManifestWriter
|
||||
from documents.export.sinks import ZipExportSink
|
||||
from documents.export.sinks import _dumps
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def source_file(tmp_path: Path) -> Path:
|
||||
src: Path = tmp_path / "src" / "doc.pdf"
|
||||
src.parent.mkdir(parents=True)
|
||||
src.write_bytes(b"PDF-CONTENT")
|
||||
return src
|
||||
|
||||
|
||||
class TestDumps:
|
||||
def test_dumps_is_indented_unicode_json(self) -> None:
|
||||
result: str = _dumps({"a": "é", "b": 1})
|
||||
assert '"é"' in result # ensure_ascii=False keeps unicode literal
|
||||
assert "\n" in result # indent=2 produces newlines
|
||||
assert json.loads(result) == {"a": "é", "b": 1}
|
||||
|
||||
|
||||
class TestStreamingManifestWriter:
|
||||
def test_writes_json_array_of_records(self) -> None:
|
||||
handle: io.StringIO = io.StringIO()
|
||||
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||
writer.write_batch([{"pk": 1}, {"pk": 2}])
|
||||
writer.write_record({"pk": 3})
|
||||
writer.close()
|
||||
assert json.loads(handle.getvalue()) == [{"pk": 1}, {"pk": 2}, {"pk": 3}]
|
||||
|
||||
def test_empty_manifest_is_valid_empty_array(self) -> None:
|
||||
handle: io.StringIO = io.StringIO()
|
||||
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||
writer.close()
|
||||
assert json.loads(handle.getvalue()) == []
|
||||
|
||||
|
||||
class TestDirectoryExportSink:
|
||||
def test_add_file_copies_to_relative_arcname(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
assert (target / "originals" / "doc.pdf").read_bytes() == b"PDF-CONTENT"
|
||||
|
||||
def test_add_json_writes_file(self, tmp_path: Path) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_json({"version": "x"}, "metadata.json")
|
||||
assert json.loads((target / "metadata.json").read_text()) == {"version": "x"}
|
||||
|
||||
def test_stream_writes_manifest(self, tmp_path: Path) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
with sink.stream("manifest.json") as handle:
|
||||
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||
writer.write_record({"pk": 1})
|
||||
writer.close()
|
||||
assert json.loads((target / "manifest.json").read_text()) == [{"pk": 1}]
|
||||
|
||||
def test_add_file_skips_when_size_and_mtime_match(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
# Pre-existing target with identical size+mtime but DIFFERENT content:
|
||||
# if add_file skips (no compare-checksums), the old content survives.
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
existing: Path = target / "originals" / "doc.pdf"
|
||||
existing.parent.mkdir(parents=True)
|
||||
# Same byte length as the source but different content + matching mtime,
|
||||
# so a size/mtime comparison treats it as unchanged and skips the copy.
|
||||
existing.write_bytes(b"X" * len(b"PDF-CONTENT"))
|
||||
stat = source_file.stat()
|
||||
os.utime(existing, (stat.st_atime, stat.st_mtime))
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf", checksum="abc")
|
||||
assert existing.read_bytes() == b"X" * len(b"PDF-CONTENT") # skipped
|
||||
|
||||
def test_add_file_recopies_when_compare_checksums_differ(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
existing: Path = target / "originals" / "doc.pdf"
|
||||
existing.parent.mkdir(parents=True)
|
||||
existing.write_bytes(b"X" * len(b"PDF-CONTENT"))
|
||||
stat = source_file.stat()
|
||||
os.utime(existing, (stat.st_atime, stat.st_mtime))
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=True,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
# wrong checksum forces recopy despite matching size/mtime
|
||||
sink.add_file(source_file, "originals/doc.pdf", checksum="not-the-real-sum")
|
||||
assert existing.read_bytes() == b"PDF-CONTENT" # recopied
|
||||
|
||||
def test_delete_prunes_unwritten_snapshot_files(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
stale: Path = target / "stale.pdf"
|
||||
stale.write_bytes(b"STALE")
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=True,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
assert not stale.exists()
|
||||
assert (target / "originals" / "doc.pdf").exists()
|
||||
|
||||
def test_no_delete_keeps_unwritten_files(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
stale: Path = target / "stale.pdf"
|
||||
stale.write_bytes(b"STALE")
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
assert stale.exists()
|
||||
|
||||
|
||||
class TestZipExportSink:
|
||||
def test_round_trip_files_json_and_stream(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
sink.add_json({"version": "x"}, "metadata.json")
|
||||
with sink.stream("manifest.json") as handle:
|
||||
writer = StreamingManifestWriter(handle)
|
||||
writer.write_record({"pk": 1})
|
||||
writer.close()
|
||||
zip_path: Path = target / "export.zip"
|
||||
assert zip_path.exists()
|
||||
assert not (target / "export.zip.tmp").exists()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
names = set(zf.namelist())
|
||||
assert {"originals/doc.pdf", "metadata.json", "manifest.json"} <= names
|
||||
assert zf.read("originals/doc.pdf") == b"PDF-CONTENT"
|
||||
assert json.loads(zf.read("manifest.json")) == [{"pk": 1}]
|
||||
|
||||
def test_nested_arcname_emits_directory_marker(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||
assert "originals/" in zf.namelist()
|
||||
|
||||
def test_flat_arcname_has_no_directory_markers(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||
assert all(not n.endswith("/") for n in zf.namelist())
|
||||
|
||||
def test_exception_leaves_no_zip_and_no_tmp(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
raise RuntimeError("boom")
|
||||
assert not (target / "export.zip").exists()
|
||||
assert not (target / "export.zip.tmp").exists()
|
||||
|
||||
def test_exception_inside_stream_cleans_up_manifest_tmp(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
scratch_dir = tmp_path / "scratch"
|
||||
settings.SCRATCH_DIR = scratch_dir
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
with sink.stream("manifest.json") as handle:
|
||||
handle.write("[")
|
||||
raise RuntimeError("boom")
|
||||
assert list(scratch_dir.glob("export-manifest-*")) == []
|
||||
assert not (target / "export.zip").exists()
|
||||
assert not (target / "export.zip.tmp").exists()
|
||||
|
||||
def test_abort_after_manifest_written_cleans_up_pending_tmp(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
scratch_dir = tmp_path / "scratch"
|
||||
settings.SCRATCH_DIR = scratch_dir
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
with sink.stream("manifest.json") as handle:
|
||||
handle.write("[]")
|
||||
raise RuntimeError("boom")
|
||||
assert list(scratch_dir.glob("export-manifest-*")) == []
|
||||
assert not (target / "export.zip").exists()
|
||||
|
||||
def test_delete_wipes_destination_on_success(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
(target / "preexisting.txt").write_text("old")
|
||||
(target / "olddir").mkdir()
|
||||
with ZipExportSink(target, "export", delete=True) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
assert (target / "export.zip").exists()
|
||||
assert not (target / "preexisting.txt").exists()
|
||||
assert not (target / "olddir").exists()
|
||||
|
||||
def test_abort_with_delete_does_not_wipe_destination(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
(target / "preexisting.txt").write_text("old")
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=True) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
raise RuntimeError("boom")
|
||||
assert (target / "preexisting.txt").exists()
|
||||
assert not (target / "export.zip").exists()
|
||||
|
||||
|
||||
class TestStreamContract:
|
||||
@pytest.fixture(params=["dir", "zip"])
|
||||
def sink(self, request: pytest.FixtureRequest, tmp_path: Path) -> ExportSink:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
if request.param == "dir":
|
||||
return DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
)
|
||||
return ZipExportSink(target, "export", delete=False)
|
||||
|
||||
def test_second_concurrent_stream_is_rejected(self, sink: ExportSink) -> None:
|
||||
with sink:
|
||||
with sink.stream("manifest.json"):
|
||||
with pytest.raises(RuntimeError, match="already open"):
|
||||
with sink.stream("other.json"):
|
||||
pass
|
||||
@@ -163,10 +163,55 @@ class TestSearch:
|
||||
assert (
|
||||
len(backend.search_ids("sswo", user=None, search_mode=SearchMode.TEXT)) == 1
|
||||
)
|
||||
assert (
|
||||
len(backend.search_ids("sswo re", user=None, search_mode=SearchMode.TEXT))
|
||||
== 1
|
||||
for query in ["sswo re", "re sswo"]:
|
||||
assert (
|
||||
len(backend.search_ids(query, user=None, search_mode=SearchMode.TEXT))
|
||||
== 1
|
||||
), query
|
||||
|
||||
def test_text_mode_matches_all_terms_without_requiring_adjacency(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""Simple text mode should match all terms in any order or field."""
|
||||
doc = Document.objects.create(
|
||||
title="complete-medical-history",
|
||||
content="Samsung Odyssey curved monitor",
|
||||
checksum="TXT13",
|
||||
pk=19,
|
||||
)
|
||||
backend.add_or_update(doc)
|
||||
|
||||
for query in [
|
||||
"complete history",
|
||||
"history complete",
|
||||
"Samsung curved",
|
||||
"curved Samsung",
|
||||
]:
|
||||
assert backend.search_ids(
|
||||
query,
|
||||
user=None,
|
||||
search_mode=SearchMode.TEXT,
|
||||
) == [doc.pk], query
|
||||
|
||||
def test_text_mode_matches_terms_across_title_and_content(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""Each simple-search term may match either title or content."""
|
||||
doc = Document.objects.create(
|
||||
title="Complete record",
|
||||
content="Patient history",
|
||||
checksum="TXT14",
|
||||
pk=20,
|
||||
)
|
||||
backend.add_or_update(doc)
|
||||
|
||||
assert backend.search_ids(
|
||||
"complete history",
|
||||
user=None,
|
||||
search_mode=SearchMode.TEXT,
|
||||
) == [doc.pk]
|
||||
|
||||
def test_text_mode_does_not_match_on_partial_term_overlap(
|
||||
self,
|
||||
@@ -186,11 +231,11 @@ class TestSearch:
|
||||
== 0
|
||||
)
|
||||
|
||||
def test_text_mode_anchors_later_query_tokens_to_token_starts(
|
||||
def test_text_mode_anchors_numeric_tokens_regardless_of_query_order(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""Multi-token simple search should not match later tokens in the middle of a word."""
|
||||
"""Numeric tokens must not match in the middle of a larger number."""
|
||||
exact_doc = Document.objects.create(
|
||||
title="Z-Berichte 6",
|
||||
content="monthly report",
|
||||
@@ -213,13 +258,14 @@ class TestSearch:
|
||||
backend.add_or_update(prefix_doc)
|
||||
backend.add_or_update(false_positive)
|
||||
|
||||
result_ids = set(
|
||||
backend.search_ids("Z-Berichte 6", user=None, search_mode=SearchMode.TEXT),
|
||||
)
|
||||
for query in ["Z-Berichte 6", "6 Z-Berichte"]:
|
||||
result_ids = set(
|
||||
backend.search_ids(query, user=None, search_mode=SearchMode.TEXT),
|
||||
)
|
||||
|
||||
assert exact_doc.id in result_ids
|
||||
assert prefix_doc.id in result_ids
|
||||
assert false_positive.id not in result_ids
|
||||
assert exact_doc.id in result_ids, query
|
||||
assert prefix_doc.id in result_ids, query
|
||||
assert false_positive.id not in result_ids, query
|
||||
|
||||
def test_text_mode_ignores_queries_without_searchable_tokens(
|
||||
self,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user