[Web] add mTLS Authentication

initialize db before validating credentials

.drone.yml

@@ -1,114 +0,0 @@
----
-kind: pipeline
-name: integration-testing
-
-platform:
-  os: linux
-  arch: amd64
-
-clone:
-  disable: true
-
-steps:
-- name: prepare-tests
-  pull: default
-  image: timovibritannia/ansible
-  commands:
-  - git clone https://github.com/mailcow/mailcow-integration-tests.git --branch $(curl -sL https://api.github.com/repos/mailcow/mailcow-integration-tests/releases/latest | jq -r '.tag_name') --single-branch .
-  - chmod +x ci.sh
-  - chmod +x ci-ssh.sh
-  - chmod +x ci-piprequierments.sh
-  - ./ci.sh
-  - wget -O group_vars/all/secrets.yml $SECRETS_DOWNLOAD_URL --quiet
-  environment:
-    SECRETS_DOWNLOAD_URL:
-      from_secret: SECRETS_DOWNLOAD_URL
-    VAULT_PW:
-      from_secret: VAULT_PW
-  when:
-    branch:
-    - master
-    event:
-    - push
-
-- name: lint
-  pull: default
-  image: timovibritannia/ansible
-  commands:
-  - ansible-lint ./
-  when:
-    branch:
-    - master
-    event:
-    - push
-
-- name: create-server
-  pull: default
-  image: timovibritannia/ansible
-  commands:
-  - ./ci-piprequierments.sh
-  - ansible-playbook mailcow-start-server.yml --diff
-  - ./ci-ssh.sh
-  environment:
-    ANSIBLE_HOST_KEY_CHECKING: false
-    ANSIBLE_FORCE_COLOR: true
-  when:
-    branch:
-    - master
-    event:
-    - push
-
-- name: setup-server
-  pull: default
-  image: timovibritannia/ansible
-  commands:
-  - sleep 120
-  - ./ci-piprequierments.sh
-  - ansible-playbook mailcow-setup-server.yml --private-key /drone/src/id_ssh_rsa --diff
-  environment:
-    ANSIBLE_HOST_KEY_CHECKING: false
-    ANSIBLE_FORCE_COLOR: true
-  when:
-    branch:
-    - master
-    event:
-    - push
-
-- name: run-tests
-  pull: default
-  image: timovibritannia/ansible
-  commands:
-  - ./ci-piprequierments.sh
-  - ansible-playbook mailcow-integration-tests.yml --private-key /drone/src/id_ssh_rsa --diff
-  environment:
-    ANSIBLE_HOST_KEY_CHECKING: false
-    ANSIBLE_FORCE_COLOR: true
-  when:
-    branch:
-    - master
-    event:
-    - push
-
-- name: delete-server
-  pull: default
-  image: timovibritannia/ansible
-  commands:
-  - ./ci-piprequierments.sh
-  - ansible-playbook mailcow-delete-server.yml --diff
-  environment:
-    ANSIBLE_HOST_KEY_CHECKING: false
-    ANSIBLE_FORCE_COLOR: true
-  when:
-    branch:
-    - master
-    event:
-    - push
-    status:
-    - failure
-    - success
-
----
-kind: signature
-hmac: 78793b0d115828cf74f6dbac3341205c6d48ef2e59661529f3412f2fac3831a9
-
-...

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ["https://www.servercow.de/mailcow?lang=en#sal"]

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,57 +0,0 @@
----
-name: Bug report
-about: Report a bug for this project. NOT to be used for support questions.
-
----
-
-<!--
-  Please DO NOT delete this template or use it for support questions.
-  You are welcome to visit us on our community channels listed at https://mailcow.github.io/mailcow-dockerized-docs/#community-support
-  For official support, please check https://mailcow.github.io/mailcow-dockerized-docs/#commercial-support
--->
-
-**Prior to placing the issue, please check following:** *(fill out each checkbox with an `X` once done)*
-- [ ] I understand, that not following or deleting the below instructions, will result in immediate closing and deletion of my issue.
-- [ ] I have understood that answers are voluntary and community-driven, and not commercial support.
-- [ ] I have verified that my issue has not been already answered in the past. I also checked previous [issues](https://github.com/mailcow/mailcow-dockerized/issues).
-
-**Description of the bug**:
-<!--
-  This should be a clear and concise description of what the bug is. What EXACTLY does happen?
-  If applicable, add screenshots to help explain your problem. Very useful for bugs in mailcow UI.
-  Write your detailed description below.
--->
-
-**Docker container logs of affected containers**:
-<!--
-  Please take a look at the [official documentation](https://mailcow.github.io/mailcow-dockerized-docs/debug-logs/) and post the last     few lines of logs, when the error occurs.
--->
-
-**Reproduction of said bug**:
-<!--
-  It is really helpful to know how exactly you are able to reproduce the reported issue.
-  Have you tried to fix the issue? What did you try?
-  What are the exact steps to get the above described behavior?
-  Screenshots can be added, if helpful. Add the text below.
--->
-
-**System information**:
-<!--
-  In this stage we would kindly ask you to attach general system information about your setup.
-  Please carefully read the questions and instructions below.
--->
-
-| Question | Answer |
-| --- | --- |
-| My operating system | I_DO_REPLY_HERE |
-| Is Apparmor, SELinux or similar active? | I_DO_REPLY_HERE |
-| Virtualization technlogy (KVM, VMware, Xen, etc - **LXC and OpenVZ are not supported** | I_DO_REPLY_HERE |
-| Server/VM specifications (Memory, CPU Cores) | I_DO_REPLY_HERE |
-| Docker Version (`docker version`) | I_DO_REPLY_HERE |
-| Docker-Compose Version (`docker-compose version`) | I_DO_REPLY_HERE |
-| Reverse proxy (custom solution) | I_DO_REPLY_HERE |
-
-- Output of `git diff origin/master`, any other changes to the code? If so, **please post them**.
-- All third-party firewalls and custom iptables rules are unsupported. *Please check the Docker docs about how to use Docker with your own ruleset*. Nevertheless, iptabels output can help us to help you: `iptables -L -vn`, `ip6tables -L -vn`, `iptables -L -vn -t nat` and `ip6tables -L -vn -t nat`.
-- DNS problems? Please run `docker exec -it $(docker ps -qf name=acme-mailcow) dig +short stackoverflow.com @172.22.1.254` (set the IP accordingly, if you changed the internal mailcow network) and post the output.
-

.github/ISSUE_TEMPLATE/Bug_report.yml

@@ -0,0 +1,167 @@
+name: 🐞 Bug Report
+description: Report a reproducible bug for mailcow. (NOT to be used for support questions.)
+labels: ["bug"]
+body:
+  - type: checkboxes
+    attributes:
+      label: Contribution guidelines
+      description: Please read the contribution guidelines before proceeding.
+      options:
+      - label: I've read the [contribution guidelines](https://github.com/mailcow/mailcow-dockerized/blob/master/CONTRIBUTING.md) and wholeheartedly agree
+        required: true
+  - type: checkboxes
+    attributes:
+      label: I've found a bug and checked that ...
+      description: Prior to placing the issue, please check following:** *(fill out each checkbox with an `X` once done)*
+      options:
+      - label: ... I understand that not following the below instructions will result in immediate closure and/or deletion of my issue.
+        required: true
+      - label: ... I have understood that this bug report is dedicated for bugs, and not for support-related inquiries.
+        required: true
+      - label: ... I have understood that answers are voluntary and community-driven, and not commercial support.
+        required: true
+      - label: ... I have verified that my issue has not been already answered in the past. I also checked previous [issues](https://github.com/mailcow/mailcow-dockerized/issues).
+        required: true
+  - type: textarea
+    attributes:
+      label: Description
+      description: Please provide a brief description of the bug in 1-2 sentences. If applicable, add screenshots to help explain your problem. Very useful for bugs in mailcow UI.
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Logs:"
+      description: "Please take a look at the [official documentation](https://docs.mailcow.email/troubleshooting/debug-logs/) and post the last few lines of logs, when the error occurs. For example, docker container logs of affected containers. This will be automatically formatted into code, so no need for backticks."
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Steps to reproduce:"
+      description: "Please describe the steps to reproduce the bug. Screenshots can be added, if helpful."
+      render: plain text
+      placeholder: |-
+        1. ...
+        2. ...
+        3. ...
+    validations:
+      required: true
+  - type: markdown
+    attributes:
+      value: |
+        ## System information
+        ### In this stage we would kindly ask you to attach general system information about your setup.
+  - type: dropdown
+    attributes:
+      label: "Which branch are you using?"
+      description: "#### `git rev-parse --abbrev-ref HEAD`"
+      multiple: false
+      options:
+        - master
+        - nightly
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: "Which architecture are you using?"
+      description: "#### `uname -m`"
+      multiple: false
+      options:
+        - x86
+        - ARM64 (aarch64)
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "Operating System:"
+      placeholder: "e.g. Ubuntu 22.04 LTS"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "Server/VM specifications:"
+      placeholder: "Memory, CPU Cores"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "Is Apparmor, SELinux or similar active?"
+      placeholder: "yes/no"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "Virtualization technology:"
+      placeholder: "KVM, VMware, Xen, etc - **LXC and OpenVZ are not supported**"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "Docker version:"
+      description: "#### `docker version`"
+      placeholder: "20.10.21"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "docker-compose version or docker compose version:"
+      description: "#### `docker-compose version` or `docker compose version`"
+      placeholder: "v2.12.2"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "mailcow version:"
+      description: "#### ```git describe --tags `git rev-list --tags --max-count=1` ```"
+      placeholder: "2022-08"
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: "Reverse proxy:"
+      placeholder: "e.g. Nginx/Traefik"
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Logs of git diff:"
+      description: "#### Output of `git diff origin/master`, any other changes to the code? If so, **please post them**:"
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Logs of iptables -L -vn:"
+      description: "#### Output of `iptables -L -vn`"
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Logs of ip6tables -L -vn:"
+      description: "#### Output of `ip6tables -L -vn`"
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Logs of iptables -L -vn -t nat:"
+      description: "#### Output of `iptables -L -vn -t nat`"
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Logs of ip6tables -L -vn -t nat:"
+      description: "#### Output of `ip6tables -L -vn -t nat`"
+      render: plain text
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "DNS check:"
+      description: "#### Output of `docker exec -it $(docker ps -qf name=acme-mailcow) dig +short stackoverflow.com @172.22.1.254` (set the IP accordingly, if you changed the internal mailcow network)"
+      render: plain text
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -1,14 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Additional context**
-Add any other context or screenshots about the feature request here or remove this section

.github/ISSUE_TEMPLATE/Feature_request.yml

@@ -0,0 +1,20 @@
+name: 💡 Feature Request
+description: Suggest an idea for mailcow.
+labels: ["enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: Summary
+      description: Please describe your idea in a reasonable amount of detail.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Motivation
+      description: Please describe how your idea would benefit you and other users.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: ❓ Community-driven support (Free)
+    url: https://docs.mailcow.email/#get-support
+    about: Please use the community forum for questions or assistance
+  - name: 🔥 Premium Support (Paid)
+    url: https://www.servercow.de/mailcow?lang=en#support
+    about: Buy a support subscription for any critical issues and get assisted by the mailcow Team. See conditions!
+  - name: 🚨 Report a security vulnerability
+    url: "mailto:info@servercow.de?subject=mailcow: dockerized Security Vulnerability"
+    about: Please give us appropriate time to verify, respond and fix before disclosure.

.github/ISSUE_TEMPLATE/pr_to_nighty_template.yml

@@ -0,0 +1,13 @@
+## :memo:  Brief description
+<!-- Diff summary - START -->
+<!-- Diff summary - END -->
+
+
+## :computer:  Commits
+<!-- Diff commits - START -->
+<!-- Diff commits - END -->
+
+
+## :file_folder:  Modified files
+<!-- Diff files - START -->
+<!-- Diff files - END -->

.github/renovate.json

@@ -0,0 +1,31 @@
+{
+  "enabled": true,
+  "timezone": "Europe/Berlin",
+  "dependencyDashboard": true,
+  "dependencyDashboardTitle": "Renovate Dashboard",
+  "commitBody": "Signed-off-by: milkmaker <milkmaker@mailcow.de>",
+  "rebaseWhen": "auto",
+  "labels": ["renovate"],
+  "assignees": [
+    "@magiccc"
+  ],
+  "baseBranches": ["staging"],
+  "enabledManagers": ["github-actions", "regex", "docker-compose"],
+  "ignorePaths": [
+    "data\/web\/inc\/lib\/vendor\/**"
+  ],
+  "regexManagers": [
+    {
+      "fileMatch": ["^helper-scripts\/nextcloud.sh$"],
+      "matchStrings": [
+        "#\\srenovate:\\sdatasource=(?<datasource>.*?) depName=(?<depName>.*?)( versioning=(?<versioning>.*?))?( extractVersion=(?<extractVersion>.*?))?\\s.*?_VERSION=(?<currentValue>.*)"
+       ]
+    },
+    {
+      "fileMatch": ["(^|/)Dockerfile[^/]*$"],
+      "matchStrings": [
+        "#\\srenovate:\\sdatasource=(?<datasource>.*?) depName=(?<depName>.*?)( versioning=(?<versioning>.*?))?( extractVersion=(?<extractVersion>.*?))?\\s(ENV|ARG) .*?_VERSION=(?<currentValue>.*)\\s"
+       ]
+    }
+  ]
+}

.github/stale.yml

@@ -1,18 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-# Issues with these labels will never be considered stale
-exemptLabels:
-  - pinned
-  - security
-  - enhancement
-# Label to use when marking an issue as stale
-staleLabel: dunno
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false

.github/workflows/check_prs_if_on_staging.yml

@@ -0,0 +1,33 @@
+name: Check PRs if on staging
+on:
+    pull_request_target:
+      types: [opened, edited]
+permissions: {}
+
+jobs:
+  is_not_staging:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.base.ref != 'staging' #check if the target branch is not staging
+    steps:
+      - name: Send message
+        uses: thollander/actions-comment-pull-request@v2.4.3
+        with:
+          GITHUB_TOKEN: ${{ secrets.CHECKIFPRISSTAGING_ACTION_PAT }}
+          message: |
+                   Thanks for contributing!
+
+                   I noticed that you didn't select `staging` as your base branch. Please change the base branch to `staging`.
+                   See the attached picture on how to change the base branch to `staging`:
+
+                   ![check_prs_if_on_staging.png](https://raw.githubusercontent.com/mailcow/mailcow-dockerized/master/.github/workflows/assets/check_prs_if_on_staging.png)
+
+      - name: Fail #we want to see failed checks in the PR
+        if: ${{ success() }} #set exit code to 1 even if commenting somehow failed
+        run: exit 1
+
+  is_staging:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.base.ref == 'staging' #check if the target branch is staging
+    steps:
+      - name: Success
+        run: exit 0

.github/workflows/close_old_issues_and_prs.yml

@@ -0,0 +1,37 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    # Once every day at midnight UTC
+    - cron: "0 0 * * *"
+  workflow_dispatch: # Allow to run workflow manually
+  issue_comment: # Run workflow on comments
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Mark/Close Stale Issues and Pull Requests 🗑️
+        uses: actions/stale@v9.0.0
+        with:
+          repo-token: ${{ secrets.STALE_ACTION_PAT }}
+          days-before-stale: 60
+          days-before-close: 7
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not had
+            recent activity. It will be closed if no further activity occurs.
+          stale-pr-message: >
+            This pull request has been automatically marked as stale because it has not had
+            recent activity. It will be closed if no further activity occurs.
+          exempt-issue-labels: "pinned,security,enhancement,investigating,neverstale"
+          exempt-pr-labels: "pinned,security,enhancement,investigating,neverstale"
+          stale-issue-label: "stale"
+          stale-pr-label: "stale"
+          exempt-draft-pr: "true"
+          close-issue-reason: "not_planned"
+          operations-per-run: "250"
+          ascending: "true"
+          #DRY-RUN
+          debug-only: "false"

.github/workflows/image_builds.yml

@@ -0,0 +1,43 @@
+name: Build mailcow Docker Images
+
+on:
+  push:
+    branches: [ "master", "staging" ]
+  workflow_dispatch:
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  docker_image_builds:
+    strategy:
+      matrix:
+        images:
+          - "acme-mailcow"
+          - "clamd-mailcow"
+          - "dockerapi-mailcow"
+          - "dovecot-mailcow"
+          - "netfilter-mailcow"
+          - "olefy-mailcow"
+          - "php-fpm-mailcow"
+          - "postfix-mailcow"
+          - "rspamd-mailcow"
+          - "sogo-mailcow"
+          - "solr-mailcow"
+          - "unbound-mailcow"
+          - "watchdog-mailcow"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Docker
+        run: |
+          curl -sSL https://get.docker.com/ | CHANNEL=stable sudo sh
+          sudo service docker start
+      - name: Prepair Image Builds
+        run: |
+          cp helper-scripts/docker-compose.override.yml.d/BUILD_FLAGS/docker-compose.override.yml docker-compose.override.yml
+      - name: Build Docker Images
+        run: |
+          docker compose build ${image}
+        env:
+          image: ${{ matrix.images }}

.github/workflows/pr_to_nightly.yml

@@ -0,0 +1,25 @@
+name: Create PR to merge to nightly from staging
+on:
+  push:
+    branches:
+      - staging
+jobs:
+  action-pull-request:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Run the Action
+        uses: devops-infra/action-pull-request@v0.5.5
+        with:
+          github_token: ${{ secrets.PRTONIGHTLY_ACTION_PAT }}
+          title: Automatic PR to nightly from ${{ github.event.repository.updated_at}}
+          assignee: DerLinkman
+          source_branch: staging
+          target_branch: nightly
+          reviewer: DerLinkman
+          label: upstream
+          template: .github/ISSUE_TEMPLATE/pr_to_nighty_template.yml
+          get_diff: true

.github/workflows/rebuild_backup_image.yml

@@ -0,0 +1,35 @@
+name: Build mailcow backup image
+
+on:
+  schedule:
+    # At 00:00 on Sunday
+    - cron: "0 0 * * 0"
+  workflow_dispatch: # Allow to run workflow manually
+
+jobs:
+  docker_image_build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.BACKUPIMAGEBUILD_ACTION_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.BACKUPIMAGEBUILD_ACTION_DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: data/Dockerfiles/backup/Dockerfile
+          push: true
+          tags: mailcow/backup:latest

.github/workflows/update_postscreen_access_list.yml

@@ -0,0 +1,39 @@
+name: Update postscreen_access.cidr
+
+on:
+  schedule:
+    # Monthly
+    - cron: "0 0 1 * *"
+  workflow_dispatch: # Allow to run workflow manually
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+  
+  
+jobs:
+  Update-postscreen_access_cidr:
+   runs-on: ubuntu-latest
+   steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Generate postscreen_access.cidr
+      run: |
+          bash helper-scripts/update_postscreen_whitelist.sh
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v6
+      with:
+        token: ${{ secrets.mailcow_action_Update_postscreen_access_cidr_pat }}
+        commit-message: update postscreen_access.cidr
+        committer: milkmaker <milkmaker@mailcow.de>
+        author: milkmaker <milkmaker@mailcow.de>
+        signoff: false
+        branch: update/postscreen_access.cidr
+        base: staging
+        delete-branch: true
+        add-paths: |
+          data/conf/postfix/postscreen_access.cidr
+        title: '[Postfix] update postscreen_access.cidr'
+        body: |
+          This PR updates the postscreen_access.cidr using GitHub Actions and [helper-scripts/update_postscreen_whitelist.sh](https://github.com/mailcow/mailcow-dockerized/blob/master/helper-scripts/update_postscreen_whitelist.sh)

.gitignore

@@ -1,55 +1,72 @@
-rebuild-images.sh
-data/conf/sogo/sieve.creds
+!data/conf/nginx/dynmaps.conf
+!data/conf/nginx/meta_exporter.conf
+!data/conf/nginx/site.conf
+!/**/.gitkeep
+*.iml
+.idea
+.vscode/*
+data/assets/ssl-example/*
+data/assets/ssl/*
+data/conf/borgmatic/
 data/conf/clamav/whitelist.ign2
-data/conf/phpfpm/sogo-sso/sogo-sso.pass
+data/conf/dovecot/acl_anyone
 data/conf/dovecot/dovecot-master.passwd
 data/conf/dovecot/dovecot-master.userdb
-mailcow.conf
-mailcow.conf_backup
-data/conf/nginx/*.active
-data/conf/postfix/sni.map
-data/conf/postfix/sni.map.db
-data/conf/postfix/extra.cf
-data/conf/postfix/sql
-data/conf/postfix/custom_transport.pcre
-data/conf/postfix/custom_postscreen_whitelist.cidr
-data/conf/postfix/allow_mailcow_local.regexp
-data/conf/dovecot/sql
-data/conf/dovecot/lua
+data/conf/dovecot/extra.conf
+data/conf/dovecot/mail_replica.conf
 data/conf/dovecot/global_sieve_*
-data/conf/dovecot/sogo_trusted_ip.conf
-data/conf/nextcloud-*.bak
-data/web/inc/vars.local.inc.php
-data/web/css/build/0081-custom-mailcow.css
-data/assets/ssl/*
-.vscode/*
-.idea
-*.iml
-data/web/.well-known/acme-challenge
-data/web/nextcloud*/
-data/web/rc*/
-data/conf/rspamd/local.d/*
-data/conf/rspamd/override.d/*
-!data/conf/nginx/dynmaps.conf
-!data/conf/nginx/site.conf
-!data/conf/nginx/meta_exporter.conf
-data/conf/nginx/*.conf
-data/conf/nginx/*.custom
-data/conf/nginx/*.bak
-data/conf/dovecot/acl_anyone
-data/conf/dovecot/mail_plugins*
 data/conf/dovecot/last_login
+data/conf/dovecot/lua
+data/conf/dovecot/mail_plugins*
+data/conf/dovecot/shared_namespace.conf
 data/conf/dovecot/sni.conf
 data/conf/dovecot/sogo-sso.conf
-data/conf/dovecot/extra.conf
-data/conf/dovecot/shared_namespace.conf
-data/conf/rspamd/custom/*
+data/conf/dovecot/sogo_trusted_ip.conf
+data/conf/dovecot/sql
+data/conf/nextcloud-*.bak
+data/conf/nginx/*.active
+data/conf/nginx/*.bak
+data/conf/nginx/*.conf
+data/conf/nginx/*.custom
+data/conf/phpfpm/sogo-sso/sogo-sso.pass
 data/conf/portainer/
-data/hooks/
+data/conf/postfix/allow_mailcow_local.regexp
+data/conf/postfix/custom_postscreen_whitelist.cidr
+data/conf/postfix/custom_transport.pcre
+data/conf/postfix/extra.cf
+data/conf/postfix/sni.map
+data/conf/postfix/sni.map.db
+data/conf/postfix/sql
+data/conf/postfix/dns_blocklists.cf
+data/conf/postfix/dnsbl_reply.map
+data/conf/rspamd/custom/*
+data/conf/rspamd/local.d/*
+data/conf/rspamd/override.d/*
+data/conf/sogo/custom-theme.js
+data/conf/sogo/plist_ldap
+data/conf/sogo/sieve.creds
+data/conf/sogo/sogo-full.svg
 data/gitea/
 data/gogs/
-data/conf/sogo/plist_ldap
-update_diffs/
-.github/
+data/hooks/dovecot/*
+data/hooks/phpfpm/*
+data/hooks/postfix/*
+data/hooks/rspamd/*
+data/hooks/sogo/*
+data/hooks/unbound/*
+data/web/templates/cache/*
+!data/web/templates/cache/.gitkeep
+data/web/.well-known/acme-challenge
+data/web/css/build/0081-custom-mailcow.css
+data/web/inc/vars.local.inc.php
+data/web/inc/app_info.inc.php
+data/web/nextcloud*/
+data/web/rc*/
 docker-compose.override.yml
+mailcow.conf
+mailcow.conf_backup
+rebuild-images.sh
 refresh_images.sh
+update_diffs/
+create_cold_standby.sh
+!data/conf/nginx/mailcow_auth.conf

.travis.yml

@@ -1,16 +0,0 @@
-sudo: required
-services:
-- docker
-script:
-- echo 'Europe/Berlin' | MAILCOW_HOSTNAME=build.mailcow ./generate_config.sh
-- docker-compose pull --ignore-pull-failures --parallel
-- docker-compose build
-- docker login --username=$DOCKER_HUB_USERNAME --password=$DOCKER_HUB_PASSWORD
-- docker-compose push
-branches:
-  only:
-  - master_disabled
-env:
-  global:
-  - secure: MpxpTwD7f0CNEVLitSpVmocK7O9r+BwFE1deEHK4AlQo/oc9cOlhGe1EL3mx9zbglPmjlDg/8kMUGv6vSirIabfBo9Szjps76bHckFr9lr2Ykkg0e29oC8pgPpSXD1eY/1ZIN/FvIkxpUFLETo1okS/j9q/A0DCGFmti0n3EoMORsgRz9CpNAiEh0zpSd6+euPAGHuczuCrDuO84my9bIOCjA/+aPunHNeXiuM8yIM2SxCSyGtIKT0+jvquIvLF58VxivysXBlRfhDn8fhB09nXA2Ru/derYQACfcmNSn9Pd4bDpebPJW5B9H/XA8xjb58uKinUlncbAMB/QnxoT75j9YRWJZRSQ+34XNYP6ZgK9soZ2TC6djQyEKTUu45Kp/1s+poSn42m9jytJJTmmK0KxsZTRcC8JD5nrjIMZWPUNNTwC5L4+I7ZRWg2WooK3LNyq1Ng8Hn6W77wSgsvAJw2HD3Lx58AprGUhHuBeaIZRuSN9aKwZrl9vKQJLqPnOp/nF2EC6kot5HYYtcotGtETXPUDih21gWD5ZM2BqVqYfQQnJnNMgeYmMdj6QQuTFqhuNJf7hXRIRkTnD3j1gDOLKQZazW0+N2JE8XWDFwi6fKScDsxT85lJti9HmzHa7+k4RVHmUYuDgRoPuzUgjWHvPsiz3/Z8WQ9JYpH84S8w=
-  - secure: fWzZisT6nGDNL4lf6tXB07eFG2drgBakHxzdF/NFVvzuP861RFR6omuL+ED0PgXrEHDJBxaBLv52je8irmUXrAH1CNr7T8DWiZo/h5h609Uzr+38T1NnIu4krL0Wo6/CDwlLKnzqTq9yBIZLQSHVJmo8AOpo1JPIi2ajodqj9ZfmAxDQTQl+G6zvQjtqIkYHsHY7A44Rto0f14ykn7w2S82Jn6Ry89VNI5V1WEO3sMpM/XekNP/HokNcRIuntL/0+kuLvTJ5akGoTjBQxSnSW95opzPeGky74HRU2obExJYqKvF0VfVJRNAqejwjIiFIbbjqV0Sk5391kFuhuBErQQDM1bOHGdxZ41HsJH29qNWIl7C33Yl10qERoqecgsJ1N/bS2ZEmWqm/zQh5GClCXPvYmzEqMYsMGM3vjbKdjDlc1Wh2w/eFclsXN9LSXh1mc35rtj46frcT6e5Kof87AIfC9hTgDvk9kAsyjaHMkSHSZthbZXCIcsD8qriNm5UqfFBYD79mPIP1S2YMQ2jscCsjHOZgYVrcm0kzDF21J1w6H0Lo7d1jw37LYlegBdtLQ9gYgqY2D5m+nxWuVoD5FZmpR+5JGtK+ootyLFF8aiFoHXd4op1JCxRLjgkmnZKXzw3kTQSpE7oa7CgzchtQmK2nqcqla1b5Qk7ilVcjooo=

CONTRIBUTING.md

@@ -1,9 +1,39 @@
-When a problem occurs, then always for a reason! What you want to do in such a case is:
+# Contribution Guidelines (Last modified on 18th December 2023)
 
+First of all, thank you for wanting to provide a bugfix or a new feature for the mailcow community, it's because of your help that the project can continue to grow!
+
+## Pull Requests (Last modified on 18th December 2023)
+
+However, please note the following regarding pull requests:
+
+1. **ALWAYS** create your PR using the staging branch of your locally cloned mailcow instance, as the pull request will end up in said staging branch of mailcow once approved. Ideally, you should simply create a new branch for your pull request that is named after the type of your PR (e.g. `feat/` for function updates or `fix/` for bug fixes) and the actual content (e.g. `sogo-6.0.0` for an update from SOGo to version 6 or `html-escape` for a fix that includes escaping HTML in mailcow).
+2. Please **keep** this pull request branch **clean** and free of commits that have nothing to do with the changes you have made (e.g. commits from other users from other branches). *If you make changes to the `update.sh` script or other scripts that trigger a commit, there is usually a developer mode for clean working in this case.
+3. **Test your changes before you commit them as a pull request.** <ins>If possible</ins>, write a small **test log** or demonstrate the functionality with a **screenshot or GIF**. *We will of course also test your pull request ourselves, but proof from you will save us the question of whether you have tested your own changes yourself.*
+4. Please **ALWAYS** create the actual pull request against the staging branch and **NEVER** directly against the master branch. *If you forget to do this, our moobot will remind you to switch the branch to staging.*
+5. Wait for a merge commit: It may happen that we do not accept your pull request immediately or sometimes not at all for various reasons. Please do not be disappointed if this is the case. We always endeavor to incorporate any meaningful changes from the community into the mailcow project.
+6. If you are planning larger and therefore more complex pull requests, it would be advisable to first announce this in a separate issue and then start implementing it after the idea has been accepted in order to avoid unnecessary frustration and effort!
+
+---
+
+## Issue Reporting (Last modified on 18th December 2023)
+
+If you plan to report a issue within mailcow please read and understand the following rules:
+
+1. **ONLY** use the issue tracker for bug reports or improvement requests and NOT for support questions. For support questions you can either contact the [mailcow community on Telegram](https://docs.mailcow.email/#community-support-and-chat) or the mailcow team directly in exchange for a [support fee](https://docs.mailcow.email/#commercial-support).
+2. **ONLY** report an error if you have the **necessary know-how (at least the basics)** for the administration of an e-mail server and the usage of Docker. mailcow is a complex and fully-fledged e-mail server including groupware components on a Docker basement and it requires a bit of technical know-how for debugging and operating.
+3. **ONLY** report bugs that are contained in the latest mailcow release series. *The definition of the latest release series includes the last major patch (e.g. 2023-12) and all minor patches (revisions) below it (e.g. 2023-12a, b, c etc.).* New issue reports published starting from January 1, 2024 must meet this criterion, as versions below the latest releases are no longer supported by us.
+4. When reporting a problem, please be as detailed as possible and include even the smallest changes to your mailcow installation. Simply fill out the corresponding bug report form in detail and accurately to minimize possible questions.
+5. **Before you open an issue/feature request**, please first check whether a similar request already exists in the mailcow tracker on GitHub. If so, please include yourself in this request.
+6. When you create a issue/feature request: Please note that the creation does <ins>**not guarantee an instant implementation or fix by the mailcow team or the community**</ins>.
+7. Please **ALWAYS** anonymize any sensitive information in your bug report or feature request before submitting it.
+
+### Quick guide to reporting problems:
 1. Read your logs; follow them to see what the reason for your problem is.
 2. Follow the leads given to you in your logfiles and start investigating.
 3. Restarting the troubled service or the whole stack to see if the problem persists.
-4. Read the [documentation](https://mailcow.github.io/mailcow-dockerized-docs/) of the troubled service and search it's bugtracker for your problem.
+4. Read the [documentation](https://docs.mailcow.email/) of the troubled service and search its bugtracker for your problem.
 5. Search our [issues](https://github.com/mailcow/mailcow-dockerized/issues) for your problem.
-6. [Create an issue](https://github.com/mailcow/mailcow-dockerized/issues) over at our GitHub repository if you think your problem might be a bug or a missing feature you badly need. But please make sure, that you include **all the logs** and a full description to your problem.
-8. Connect to IRC ([chat.freenode.net](https://webchat.freenode.net/)) and join our IRC channel #mailcow.
+6. [Create an issue](https://github.com/mailcow/mailcow-dockerized/issues/new/choose) over at our GitHub repository if you think your problem might be a bug or a missing feature you badly need. But please make sure, that you include **all the logs** and a full description to your problem.
+7. Ask your questions in our community-driven [support channels](https://docs.mailcow.email/#community-support-and-chat).
+
+## When creating an issue/feature request or a pull request, you will be asked to confirm these guidelines.

README.md

@@ -1,5 +1,10 @@
 # mailcow: dockerized - 🐮 + 🐋 = 💕
 
+[![Translation status](https://translate.mailcow.email/widgets/mailcow-dockerized/-/translation/svg-badge.svg)](https://translate.mailcow.email/engage/mailcow-dockerized/)
+[![Twitter URL](https://img.shields.io/twitter/url/https/twitter.com/mailcow_email.svg?style=social&label=Follow%20%40mailcow_email)](https://twitter.com/mailcow_email)
+![Mastodon Follow](https://img.shields.io/mastodon/follow/109388212176073348?domain=https%3A%2F%2Fmailcow.social&label=Follow%20%40doncow%40mailcow.social&link=https%3A%2F%2Fmailcow.social%2F%40doncow)
+
+
 ## Want to support mailcow?
 
 Please [consider a support contract with Servercow](https://www.servercow.de/mailcow?lang=en#support) to support further development. _We_ support _you_ while _you_ support _us_. :)
@@ -10,7 +15,7 @@ Or just spread the word: moo.
 
 ## Info, documentation and support
 
-Please see [the official documentation](https://mailcow.github.io/mailcow-dockerized-docs/) for installation and support instructions. 🐄
+Please see [the official documentation](https://docs.mailcow.email/) for installation and support instructions. 🐄
 
 🐛 **If you found a critical security issue, please mail us to [info at servercow.de](mailto:info@servercow.de).**
 
@@ -22,9 +27,19 @@ Please see [the official documentation](https://mailcow.github.io/mailcow-docker
 
 [Telegram mailcow Off-Topic channel](https://t.me/mailcowOfftopic)
 
+[Official 𝕏 (Twitter) Account](https://twitter.com/mailcow_email)
+
+[Official Mastodon Account](https://mailcow.social/@doncow)
+
 Telegram desktop clients are available for [multiple platforms](https://desktop.telegram.org). You can search the groups history for keywords.
 
 ## Misc
 
 **Important**: mailcow makes use of various open-source software. Please assure you agree with their license before using mailcow.
 Any part of mailcow itself is released under **GNU General Public License, Version 3**.
+
+mailcow is a registered word mark of The Infrastructure Company GmbH, Parkstr. 42, 47877 Willich, Germany.
+
+The project is managed and maintained by The Infrastructure Company GmbH.
+
+Originated from @andryyy (André)

SECURITY.md

@@ -0,0 +1,42 @@
+# Security Policies and Procedures
+
+This document outlines security procedures and general policies for the _mailcow: dockerized_ project as found on [mailcow-dockerized](https://github.com/mailcow/mailcow-dockerized).
+
+  * [Reporting a Vulnerability](#reporting-a-vulnerability)
+  * [Disclosure Policy](#disclosure-policy)
+  * [Comments on this Policy](#comments-on-this-policy)
+
+## Reporting a Vulnerability 
+
+The mailcow team and community take all security vulnerabilities
+seriously. Thank you for improving the security of our open source 
+software. We appreciate your efforts and responsible disclosure and will
+make every effort to acknowledge your contributions.
+
+Report security vulnerabilities by emailing the mailcow team at:
+    
+    info at servercow.de
+
+mailcow team will acknowledge your email as soon as possible, and will
+send a more detailed response afterwards indicating the next steps in 
+handling your report. After the initial reply to your report, the mailcow
+team will endeavor to keep you informed of the progress towards a fix and
+full announcement, and may ask for additional information or guidance.
+
+Report security vulnerabilities in third-party modules to the person or 
+team maintaining the module.
+
+## Disclosure Policy
+
+When the mailcow team receives a security bug report, they will assign it
+to a primary handler. This person will coordinate the fix and release
+process, involving the following steps:
+
+  * Confirm the problem and determine the affected versions.
+  * Audit code to find any potential similar problems.
+  * Prepare fixes for all releases still under maintenance.
+
+## Comments on this Policy
+
+If you have suggestions on how this process could be improved please submit a
+pull request.

create_cold_standby.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export REMOTE_SSH_KEY=/root/.ssh/id_rsa
+export REMOTE_SSH_PORT=22
+export REMOTE_SSH_HOST=my.remote.host
+
+/opt/mailcow-dockerized/helper-scripts/_cold-standby.sh

data/Dockerfiles/acme/Dockerfile

@@ -1,7 +1,8 @@
-FROM alpine:3.11
+FROM alpine:3.18
 
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
 
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
 RUN apk upgrade --no-cache \
   && apk add --update --no-cache \
   bash \
@@ -14,8 +15,9 @@ RUN apk upgrade --no-cache \
   tini \
   tzdata \
   python3 \
-  && python3 -m pip install --upgrade pip \
-  && python3 -m pip install acme-tiny
+  py3-pip \
+  && pip3 install --upgrade pip \
+  && pip3 install acme-tiny
 
 COPY acme.sh /srv/acme.sh
 COPY functions.sh /srv/functions.sh

data/Dockerfiles/acme/acme.sh

@@ -155,6 +155,18 @@ while true; do
   fi
   if [[ ! -f ${ACME_BASE}/acme/account.pem ]]; then
     log_f "Generating missing Lets Encrypt account key..."
+    if [[ ! -z ${ACME_CONTACT} ]]; then
+      if ! verify_email "${ACME_CONTACT}"; then
+        log_f "Invalid email address, will not start registration!"
+        sleep 365d
+        exec $(readlink -f "$0")
+      else
+        ACME_CONTACT_PARAMETER="--contact mailto:${ACME_CONTACT}"
+        log_f "Valid email address, using ${ACME_CONTACT} for registration"
+      fi
+    else
+      ACME_CONTACT_PARAMETER=""
+    fi
     openssl genrsa 4096 > ${ACME_BASE}/acme/account.pem
   else
     log_f "Using existing Lets Encrypt account key ${ACME_BASE}/acme/account.pem"
@@ -201,28 +213,17 @@ while true; do
   done
   ADDITIONAL_WC_ARR+=('autodiscover' 'autoconfig')
 
+  if [[ ${SKIP_IP_CHECK} != "y" ]]; then
   # Start IP detection
   log_f "Detecting IP addresses..."
   IPV4=$(get_ipv4)
   IPV6=$(get_ipv6)
   log_f "OK: ${IPV4}, ${IPV6:-"0000:0000:0000:0000:0000:0000:0000:0000"}"
-
-  # Hard-fail on CAA errors for MAILCOW_HOSTNAME
-  MH_PARENT_DOMAIN=$(echo ${MAILCOW_HOSTNAME} | cut -d. -f2-)
-  MH_CAAS=( $(dig CAA ${MH_PARENT_DOMAIN} +short | sed -n 's/\d issue "\(.*\)"/\1/p') )
-  if [[ ! -z ${MH_CAAS} ]]; then
-    if [[ ${MH_CAAS[@]} =~ "letsencrypt.org" ]]; then
-      log_f "Validated CAA for parent domain ${MH_PARENT_DOMAIN}"
-    else
-      log_f "Skipping ACME validation: Lets Encrypt disallowed for ${MAILCOW_HOSTNAME} by CAA record, retrying in 1h..."
-      sleep 1h
-      exec $(readlink -f "$0")
-    fi
   fi
 
   #########################################
   # IP and webroot challenge verification #
-  SQL_DOMAINS=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain WHERE backupmx=0" -Bs)
+  SQL_DOMAINS=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain WHERE backupmx=0 and active=1" -Bs)
   if [[ ! $? -eq 0 ]]; then
     log_f "Failed to read SQL domains, retrying in 1 minute..."
     sleep 1m
@@ -290,7 +291,7 @@ while true; do
     VALIDATED_CERTIFICATES+=("${CERT_NAME}")
 
     # obtain server certificate if required
-    DOMAINS=${SERVER_SAN_VALIDATED[@]} /srv/obtain-certificate.sh rsa
+    ACME_CONTACT_PARAMETER=${ACME_CONTACT_PARAMETER} DOMAINS=${SERVER_SAN_VALIDATED[@]} /srv/obtain-certificate.sh rsa
     RETURN="$?"
     if [[ "$RETURN" == "0" ]]; then # 0 = cert created successfully
       CERT_AMOUNT_CHANGED=1

data/Dockerfiles/acme/functions.sh

@@ -16,6 +16,15 @@ log_f() {
   fi
 }
 
+verify_email(){
+  regex="^(([A-Za-z0-9]+((\.|\-|\_|\+)?[A-Za-z0-9]?)*[A-Za-z0-9]+)|[A-Za-z0-9]+)@(([A-Za-z0-9]+)+((\.|\-|\_)?([A-Za-z0-9]+)+)*)+\.([A-Za-z]{2,})+$"
+  if [[ $1 =~ ${regex} ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
 verify_hash_match(){
   CERT_HASH=$(openssl x509 -in "${1}" -noout -pubkey | openssl md5)
   KEY_HASH=$(openssl pkey -in "${2}" -pubout | openssl md5)
@@ -33,7 +42,7 @@ get_ipv4(){
   local IPV4_SRCS=
   local TRY=
   IPV4_SRCS[0]="ip4.mailcow.email"
-  IPV4_SRCS[1]="ip4.korves.net"
+  IPV4_SRCS[1]="ip4.nevondo.com"
   until [[ ! -z ${IPV4} ]] || [[ ${TRY} -ge 10 ]]; do
     IPV4=$(curl --connect-timeout 3 -m 10 -L4s ${IPV4_SRCS[$RANDOM % ${#IPV4_SRCS[@]} ]} | grep -E "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$")
     [[ ! -z ${TRY} ]] && sleep 1
@@ -46,8 +55,8 @@ get_ipv6(){
   local IPV6=
   local IPV6_SRCS=
   local TRY=
-  IPV6_SRCS[0]="ip6.korves.net"
-  IPV6_SRCS[1]="ip6.mailcow.email"
+  IPV6_SRCS[0]="ip6.mailcow.email"
+  IPV6_SRCS[1]="ip6.nevondo.com"
   until [[ ! -z ${IPV6} ]] || [[ ${TRY} -ge 10 ]]; do
     IPV6=$(curl --connect-timeout 3 -m 10 -L6s ${IPV6_SRCS[$RANDOM % ${#IPV6_SRCS[@]} ]} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$")
     [[ ! -z ${TRY} ]] && sleep 1
@@ -60,6 +69,17 @@ check_domain(){
     DOMAIN=$1
     A_DOMAIN=$(dig A ${DOMAIN} +short | tail -n 1)
     AAAA_DOMAIN=$(dig AAAA ${DOMAIN} +short | tail -n 1)
+    # Hard-fail on CAA errors for MAILCOW_HOSTNAME
+    PARENT_DOMAIN=$(echo ${DOMAIN} | cut -d. -f2-)
+    CAAS=( $(dig CAA ${PARENT_DOMAIN} +short | sed -n 's/\d issue "\(.*\)"/\1/p') )
+    if [[ ! -z ${CAAS} ]]; then
+      if [[ ${CAAS[@]} =~ "letsencrypt.org" ]]; then
+        log_f "Validated CAA for parent domain ${PARENT_DOMAIN}"
+      else
+        log_f "Lets Encrypt disallowed for ${PARENT_DOMAIN} by CAA record"
+        return 1
+      fi
+    fi
     # Check if CNAME without v6 enabled target
     if [[ ! -z ${AAAA_DOMAIN} ]] && [[ -z $(echo ${AAAA_DOMAIN} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$") ]]; then
       AAAA_DOMAIN=

data/Dockerfiles/acme/obtain-certificate.sh

@@ -80,7 +80,7 @@ fi
 printf "[SAN]\nsubjectAltName=" > /tmp/_SAN
 printf "DNS:%s," "${CERT_DOMAINS[@]}" >> /tmp/_SAN
 sed -i '$s/,$//' /tmp/_SAN
-openssl req -new -sha256 -key ${KEY} -subj "/" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf /tmp/_SAN) > ${CSR}
+openssl req -new -sha256 -key ${KEY} -subj "/" -reqexts SAN -config <(cat "$(openssl version -d | sed 's/.*"\(.*\)"/\1/g')/openssl.cnf" /tmp/_SAN) > ${CSR}
 
 # acme-tiny writes info to stderr and ceritifcate to stdout
 # The redirects will do the following:
@@ -93,8 +93,8 @@ until dig letsencrypt.org +time=3 +tries=1 @unbound > /dev/null; do
   sleep 2
 done
 log_f "Resolver OK"
-
-ACME_RESPONSE=$(acme-tiny ${DIRECTORY_URL} \
+log_f "Using command acme-tiny ${DIRECTORY_URL} ${ACME_CONTACT_PARAMETER} --account-key ${ACME_BASE}/acme/account.pem --disable-check --csr ${CSR} --acme-dir /var/www/acme/"
+ACME_RESPONSE=$(acme-tiny ${DIRECTORY_URL} ${ACME_CONTACT_PARAMETER} \
   --account-key ${ACME_BASE}/acme/account.pem \
   --disable-check \
   --csr ${CSR} \

data/Dockerfiles/backup/Dockerfile

@@ -0,0 +1,3 @@
+FROM debian:bullseye-slim
+
+RUN apt update && apt install pigz

data/Dockerfiles/clamd/Dockerfile

@@ -1,61 +1,25 @@
-FROM debian:buster-slim
+FROM alpine:3.19
 
-LABEL maintainer "André Peters <andre.peters@servercow.de>"
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
 
-ARG CLAMAV=0.103.0
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  ca-certificates \
-  zlib1g-dev \
-  libcurl4-openssl-dev \
-  libncurses5-dev \
-  libzip-dev \
-  libpcre2-dev \
-  libxml2-dev \
-  libssl-dev \
-  build-essential \
-  libjson-c-dev \
-  curl \
-  bash \
-  wget \
-  tzdata \
-  dnsutils \
+RUN apk upgrade --no-cache \
+  && apk add --update --no-cache \
   rsync \
-  dos2unix \
-  netcat \
-  && rm -rf /var/lib/apt/lists/* \
-  && wget -O - https://www.clamav.net/downloads/production/clamav-${CLAMAV}.tar.gz | tar xfvz - \
-  && cd clamav-${CLAMAV} \
-  && ./configure \
-  --prefix=/usr \
-  --libdir=/usr/lib \
-  --sysconfdir=/etc/clamav \
-  --mandir=/usr/share/man \
-  --infodir=/usr/share/info \
-  --disable-llvm \
-  --with-user=clamav \
-  --with-group=clamav \
-  --with-dbdir=/var/lib/clamav \
-  --enable-clamdtop \
-  --enable-bigstack \
-  --with-pcre \
-  && make -j4 \
-  && make install \
-  && make clean \
-  && cd .. && rm -rf clamav-${CLAMAV} \
-  && apt-get -y --auto-remove purge build-essential \
-  && apt-get -y purge zlib1g-dev \
-  libncurses5-dev \
-  libzip-dev \
-  libpcre2-dev \
-  libxml2-dev \
-  libssl-dev \
-  libjson-c-dev \
-  && addgroup --system --gid 700 clamav \
-  && adduser --system --no-create-home --home /var/lib/clamav --uid 700 --gid 700 --disabled-login clamav \
-  && rm -rf /tmp/* /var/tmp/*
+  clamav \
+  bind-tools \
+  bash \
+  tini
 
-COPY clamd.sh ./
-COPY tini /sbin/tini
+# init
+COPY clamd.sh /clamd.sh
+RUN chmod +x /sbin/tini
 
-CMD ["/sbin/tini", "-g", "--", "/clamd.sh"]
+# healthcheck
+COPY healthcheck.sh /healthcheck.sh
+COPY clamdcheck.sh /usr/local/bin
+RUN chmod +x /healthcheck.sh
+RUN chmod +x /usr/local/bin/clamdcheck.sh
+HEALTHCHECK --start-period=6m CMD "/healthcheck.sh"
+
+ENTRYPOINT []
+CMD ["/sbin/tini", "-g", "--", "/clamd.sh"]

data/Dockerfiles/clamd/clamd.sh

@@ -26,6 +26,7 @@ if [[ ! -f /var/lib/clamav/whitelist.ign2 ]]; then
 Example-Signature.Ignore-1
 PUA.Win.Trojan.EmbeddedPDF-1
 PUA.Pdf.Trojan.EmbeddedJavaScript-1
+PUA.Pdf.Trojan.OpenActionObjectwithJavascript-1
 EOF
 fi
 

data/Dockerfiles/clamd/clamdcheck.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${CLAMAV_NO_CLAMD:-}" != "false" ]; then
+	if [ "$(echo "PING" | nc localhost 3310)" != "PONG" ]; then
+		echo "ERROR: Unable to contact server"
+		exit 1
+	fi
+
+	echo "Clamd is up"
+fi
+
+exit 0

data/Dockerfiles/clamd/healthcheck.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [[ "${SKIP_CLAMD}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  echo "SKIP_CLAMD=y, skipping ClamAV..."
+  exit 0
+fi
+
+# run clamd healthcheck
+/usr/local/bin/clamdcheck.sh

data/Dockerfiles/dockerapi/Dockerfile

@@ -1,12 +1,27 @@
-FROM alpine:3.11
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM alpine:3.19
 
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
+
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
 WORKDIR /app
 
-RUN apk add --update --no-cache python3 openssl tzdata \
- && pip3 install --upgrade pip \
- && pip3 install --upgrade docker flask flask-restful
+RUN apk add --update --no-cache python3 \
+  py3-pip \
+  openssl \
+  tzdata \
+  py3-psutil \
+  py3-redis \
+  py3-async-timeout \
+&& pip3 install --upgrade pip \
+  fastapi \
+  uvicorn \
+  aiodocker \
+  docker
+RUN mkdir /app/modules
 
-COPY dockerapi.py /app/
+COPY docker-entrypoint.sh /app/
+COPY main.py /app/main.py
+COPY modules/ /app/modules/
 
-CMD ["python3", "-u", "/app/dockerapi.py"]
+ENTRYPOINT ["/bin/sh", "/app/docker-entrypoint.sh"]
+CMD exec python main.py

data/Dockerfiles/dockerapi/docker-entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+`openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
+  -keyout /app/dockerapi_key.pem \
+  -out /app/dockerapi_cert.pem \
+  -subj /CN=dockerapi/O=mailcow \
+  -addext subjectAltName=DNS:dockerapi`
+
+exec "$@"

data/Dockerfiles/dockerapi/dockerapi.py

@@ -1,419 +0,0 @@
-#!/usr/bin/env python3
-
-from flask import Flask
-from flask_restful import Resource, Api
-from flask import jsonify
-from flask import Response
-from flask import request
-from threading import Thread
-import docker
-import uuid
-import signal
-import time
-import os
-import re
-import sys
-import ssl
-import socket
-import subprocess
-import traceback
-
-docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
-app = Flask(__name__)
-api = Api(app)
-
-class containers_get(Resource):
-  def get(self):
-    containers = {}
-    try:
-      for container in docker_client.containers.list(all=True):
-        containers.update({container.attrs['Id']: container.attrs})
-      return containers
-    except Exception as e:
-      return jsonify(type='danger', msg=str(e))
-
-class container_get(Resource):
-  def get(self, container_id):
-    if container_id and container_id.isalnum():
-      try:
-        for container in docker_client.containers.list(all=True, filters={"id": container_id}):
-          return container.attrs
-      except Exception as e:
-          return jsonify(type='danger', msg=str(e))
-    else:
-      return jsonify(type='danger', msg='no or invalid id defined')
-
-class container_post(Resource):
-  def post(self, container_id, post_action):
-    if container_id and container_id.isalnum() and post_action:
-      try:
-        """Dispatch container_post api call"""
-        if post_action == 'exec':
-          if not request.json or not 'cmd' in request.json:
-            return jsonify(type='danger', msg='cmd is missing')
-          if not request.json or not 'task' in request.json:
-            return jsonify(type='danger', msg='task is missing')
-
-          api_call_method_name = '__'.join(['container_post', str(post_action), str(request.json['cmd']), str(request.json['task']) ])
-        else:
-          api_call_method_name = '__'.join(['container_post', str(post_action) ])
-
-        api_call_method = getattr(self, api_call_method_name, lambda container_id: jsonify(type='danger', msg='container_post - unknown api call'))
-
-
-        print("api call: %s, container_id: %s" % (api_call_method_name, container_id))
-        return api_call_method(container_id)
-      except Exception as e:
-        print("error - container_post: %s" % str(e))
-        return jsonify(type='danger', msg=str(e))
-
-    else:
-      return jsonify(type='danger', msg='invalid container id or missing action')
-
-
-  # api call: container_post - post_action: stop
-  def container_post__stop(self, container_id):
-    for container in docker_client.containers.list(all=True, filters={"id": container_id}):
-      container.stop()
-    return jsonify(type='success', msg='command completed successfully')
-
-
-  # api call: container_post - post_action: start
-  def container_post__start(self, container_id):
-    for container in docker_client.containers.list(all=True, filters={"id": container_id}):
-      container.start()
-    return jsonify(type='success', msg='command completed successfully')
-
-
-  # api call: container_post - post_action: restart
-  def container_post__restart(self, container_id):
-    for container in docker_client.containers.list(all=True, filters={"id": container_id}):
-      container.restart()
-    return jsonify(type='success', msg='command completed successfully')
-
-
-  # api call: container_post - post_action: top
-  def container_post__top(self, container_id):
-    for container in docker_client.containers.list(all=True, filters={"id": container_id}):
-      return jsonify(type='success', msg=container.top())
-
-
-  # api call: container_post - post_action: stats
-  def container_post__stats(self, container_id):
-    for container in docker_client.containers.list(all=True, filters={"id": container_id}):
-      for stat in container.stats(decode=True, stream=True):
-        return jsonify(type='success', msg=stat )
-
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: delete
-  def container_post__exec__mailq__delete(self, container_id):
-    if 'items' in request.json:
-      r = re.compile("^[0-9a-fA-F]+$")
-      filtered_qids = filter(r.match, request.json['items'])
-      if filtered_qids:
-        flagged_qids = ['-d %s' % i for i in filtered_qids]
-        sanitized_string = str(' '.join(flagged_qids));
-
-        for container in docker_client.containers.list(filters={"id": container_id}):
-          postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
-          return exec_run_handler('generic', postsuper_r)
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: hold
-  def container_post__exec__mailq__hold(self, container_id):
-    if 'items' in request.json:
-      r = re.compile("^[0-9a-fA-F]+$")
-      filtered_qids = filter(r.match, request.json['items'])
-      if filtered_qids:
-        flagged_qids = ['-h %s' % i for i in filtered_qids]
-        sanitized_string = str(' '.join(flagged_qids));
-
-        for container in docker_client.containers.list(filters={"id": container_id}):
-          postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
-          return exec_run_handler('generic', postsuper_r)
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: cat
-  def container_post__exec__mailq__cat(self, container_id):
-    if 'items' in request.json:
-      r = re.compile("^[0-9a-fA-F]+$")
-      filtered_qids = filter(r.match, request.json['items'])
-      if filtered_qids:
-        sanitized_string = str(' '.join(filtered_qids));
-
-        for container in docker_client.containers.list(filters={"id": container_id}):
-          postcat_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postcat -q " + sanitized_string], user='postfix')
-        if not postcat_return:
-          postcat_return = 'err: invalid'
-        return exec_run_handler('utf8_text_only', postcat_return)
-
-   # api call: container_post - post_action: exec - cmd: mailq - task: unhold
-  def container_post__exec__mailq__unhold(self, container_id):
-    if 'items' in request.json:
-      r = re.compile("^[0-9a-fA-F]+$")
-      filtered_qids = filter(r.match, request.json['items'])
-      if filtered_qids:
-        flagged_qids = ['-H %s' % i for i in filtered_qids]
-        sanitized_string = str(' '.join(flagged_qids));
-
-        for container in docker_client.containers.list(filters={"id": container_id}):
-          postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
-          return exec_run_handler('generic', postsuper_r)
-
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: deliver
-  def container_post__exec__mailq__deliver(self, container_id):
-    if 'items' in request.json:
-      r = re.compile("^[0-9a-fA-F]+$")
-      filtered_qids = filter(r.match, request.json['items'])
-      if filtered_qids:
-        flagged_qids = ['-i %s' % i for i in filtered_qids]
-
-        for container in docker_client.containers.list(filters={"id": container_id}):
-          for i in flagged_qids:
-            postqueue_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postqueue " + i], user='postfix')
-            # todo: check each exit code
-          return jsonify(type='success', msg=str("Scheduled immediate delivery"))
-
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: list
-  def container_post__exec__mailq__list(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      mailq_return = container.exec_run(["/usr/sbin/postqueue", "-j"], user='postfix')
-      return exec_run_handler('utf8_text_only', mailq_return)
-
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: flush
-  def container_post__exec__mailq__flush(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      postqueue_r = container.exec_run(["/usr/sbin/postqueue", "-f"], user='postfix')
-      return exec_run_handler('generic', postqueue_r)
-
-
-  # api call: container_post - post_action: exec - cmd: mailq - task: super_delete
-  def container_post__exec__mailq__super_delete(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      postsuper_r = container.exec_run(["/usr/sbin/postsuper", "-d", "ALL"])
-      return exec_run_handler('generic', postsuper_r)
-
-
-  # api call: container_post - post_action: exec - cmd: system - task: fts_rescan
-  def container_post__exec__system__fts_rescan(self, container_id):
-    if 'username' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        rescan_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/doveadm fts rescan -u '" + request.json['username'].replace("'", "'\\''") + "'"], user='vmail')
-        if rescan_return.exit_code == 0:
-          return jsonify(type='success', msg='fts_rescan: rescan triggered')
-        else:
-          return jsonify(type='warning', msg='fts_rescan error')
-
-    if 'all' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        rescan_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/doveadm fts rescan -A"], user='vmail')
-        if rescan_return.exit_code == 0:
-          return jsonify(type='success', msg='fts_rescan: rescan triggered')
-        else:
-          return jsonify(type='warning', msg='fts_rescan error')
-
-
-  # api call: container_post - post_action: exec - cmd: system - task: df
-  def container_post__exec__system__df(self, container_id):
-    if 'dir' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        df_return = container.exec_run(["/bin/bash", "-c", "/bin/df -H '" + request.json['dir'].replace("'", "'\\''") + "' | /usr/bin/tail -n1 | /usr/bin/tr -s [:blank:] | /usr/bin/tr ' ' ','"], user='nobody')
-        if df_return.exit_code == 0:
-          return df_return.output.decode('utf-8').rstrip()
-        else:
-          return "0,0,0,0,0,0"
-
-
-  # api call: container_post - post_action: exec - cmd: system - task: mysql_upgrade
-  def container_post__exec__system__mysql_upgrade(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      sql_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n"], user='mysql')
-      if sql_return.exit_code == 0:
-        matched = False
-        for line in sql_return.output.decode('utf-8').split("\n"):
-          if 'is already upgraded to' in line:
-            matched = True
-        if matched:
-          return jsonify(type='success', msg='mysql_upgrade: already upgraded', text=sql_return.output.decode('utf-8'))
-        else:
-          container.restart()
-          return jsonify(type='warning', msg='mysql_upgrade: upgrade was applied', text=sql_return.output.decode('utf-8'))
-      else:
-        return jsonify(type='error', msg='mysql_upgrade: error running command', text=sql_return.output.decode('utf-8'))
-
-  # api call: container_post - post_action: exec - cmd: system - task: mysql_tzinfo_to_sql
-  def container_post__exec__system__mysql_tzinfo_to_sql(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      sql_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/mysql_tzinfo_to_sql /usr/share/zoneinfo | /bin/sed 's/Local time zone must be set--see zic manual page/FCTY/' | /usr/bin/mysql -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "' mysql \n"], user='mysql')
-      if sql_return.exit_code == 0:
-        return jsonify(type='info', msg='mysql_tzinfo_to_sql: command completed successfully', text=sql_return.output.decode('utf-8'))
-      else:
-        return jsonify(type='error', msg='mysql_tzinfo_to_sql: error running command', text=sql_return.output.decode('utf-8'))
-
-  # api call: container_post - post_action: exec - cmd: reload - task: dovecot
-  def container_post__exec__reload__dovecot(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      reload_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/dovecot reload"])
-      return exec_run_handler('generic', reload_return)
-
-
-  # api call: container_post - post_action: exec - cmd: reload - task: postfix
-  def container_post__exec__reload__postfix(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      reload_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postfix reload"])
-      return exec_run_handler('generic', reload_return)
-
-
-  # api call: container_post - post_action: exec - cmd: reload - task: nginx
-  def container_post__exec__reload__nginx(self, container_id):
-    for container in docker_client.containers.list(filters={"id": container_id}):
-      reload_return = container.exec_run(["/bin/sh", "-c", "/usr/sbin/nginx -s reload"])
-      return exec_run_handler('generic', reload_return)
-
-
-  # api call: container_post - post_action: exec - cmd: sieve - task: list
-  def container_post__exec__sieve__list(self, container_id):
-    if 'username' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        sieve_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/doveadm sieve list -u '" + request.json['username'].replace("'", "'\\''") + "'"])
-        return exec_run_handler('utf8_text_only', sieve_return)
-
-
-  # api call: container_post - post_action: exec - cmd: sieve - task: print
-  def container_post__exec__sieve__print(self, container_id):
-    if 'username' in request.json and 'script_name' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request.json['username'].replace("'", "'\\''") + "' '" + request.json['script_name'].replace("'", "'\\''") + "'"]  
-        sieve_return = container.exec_run(cmd)
-        return exec_run_handler('utf8_text_only', sieve_return)
-
-
-  # api call: container_post - post_action: exec - cmd: maildir - task: cleanup
-  def container_post__exec__maildir__cleanup(self, container_id):
-    if 'maildir' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        sane_name = re.sub(r'\W+', '', request.json['maildir'])
-        cmd = ["/bin/bash", "-c", "if [[ -d '/var/vmail/" + request.json['maildir'].replace("'", "'\\''") + "' ]]; then /bin/mv '/var/vmail/" + request.json['maildir'].replace("'", "'\\''") + "' '/var/vmail/_garbage/" + str(int(time.time())) + "_" + sane_name + "'; fi"]
-        maildir_cleanup = container.exec_run(cmd, user='vmail')
-        return exec_run_handler('generic', maildir_cleanup)
-
-
-
-  # api call: container_post - post_action: exec - cmd: rspamd - task: worker_password
-  def container_post__exec__rspamd__worker_password(self, container_id):
-    if 'raw' in request.json:
-      for container in docker_client.containers.list(filters={"id": container_id}):
-        cmd = "/usr/bin/rspamadm pw -e -p '" + request.json['raw'].replace("'", "'\\''") + "' 2> /dev/null"
-        cmd_response = exec_cmd_container(container, cmd, user="_rspamd")
-        matched = False
-        for line in cmd_response.split("\n"):
-          if '$2$' in line:
-            hash = line.strip()
-            hash_out = re.search('\$2\$.+$', hash).group(0)
-            rspamd_passphrase_hash = re.sub('[^0-9a-zA-Z\$]+', '', hash_out.rstrip())
-
-            rspamd_password_filename = "/etc/rspamd/override.d/worker-controller-password.inc"
-            cmd = '''/bin/echo 'enable_password = "%s";' > %s && cat %s''' % (rspamd_passphrase_hash, rspamd_password_filename, rspamd_password_filename)
-            cmd_response = exec_cmd_container(container, cmd, user="_rspamd")
-
-            if rspamd_passphrase_hash.startswith("$2$") and rspamd_passphrase_hash in cmd_response:
-              container.restart()
-              matched = True
-
-        if matched:
-            return jsonify(type='success', msg='command completed successfully')
-        else:
-            return jsonify(type='danger', msg='command did not complete')
-
-def exec_cmd_container(container, cmd, user, timeout=2, shell_cmd="/bin/bash"):
-
-  def recv_socket_data(c_socket, timeout):
-    c_socket.setblocking(0)
-    total_data=[];
-    data='';
-    begin=time.time()
-    while True:
-      if total_data and time.time()-begin > timeout:
-        break
-      elif time.time()-begin > timeout*2:
-        break
-      try:
-        data = c_socket.recv(8192)
-        if data:
-          total_data.append(data.decode('utf-8'))
-          #change the beginning time for measurement
-          begin=time.time()
-        else:
-          #sleep for sometime to indicate a gap
-          time.sleep(0.1)
-          break
-      except:
-        pass
-    return ''.join(total_data)
-
-  try :
-    socket = container.exec_run([shell_cmd], stdin=True, socket=True, user=user).output._sock
-    if not cmd.endswith("\n"):
-      cmd = cmd + "\n"
-    socket.send(cmd.encode('utf-8'))
-    data = recv_socket_data(socket, timeout)
-    socket.close()
-    return data
-
-  except Exception as e:
-    print("error - exec_cmd_container: %s" % str(e))
-    traceback.print_exc(file=sys.stdout)
-
-def exec_run_handler(type, output):
-  if type == 'generic':
-    if output.exit_code == 0:
-      return jsonify(type='success', msg='command completed successfully')
-    else:
-      return jsonify(type='danger', msg='command failed: ' + output.output.decode('utf-8'))
-  if type == 'utf8_text_only':
-    r = Response(response=output.output.decode('utf-8'), status=200, mimetype="text/plain")
-    r.headers["Content-Type"] = "text/plain; charset=utf-8"
-    return r
-
-class GracefulKiller:
-  kill_now = False
-  def __init__(self):
-    signal.signal(signal.SIGINT, self.exit_gracefully)
-    signal.signal(signal.SIGTERM, self.exit_gracefully)
-
-  def exit_gracefully(self, signum, frame):
-    self.kill_now = True
-
-def create_self_signed_cert():
-    process = subprocess.Popen(
-      "openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout /app/dockerapi_key.pem -out /app/dockerapi_cert.pem -subj /CN=dockerapi/O=mailcow -addext subjectAltName=DNS:dockerapi".split(),
-      stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=False
-    )
-    process.wait()
-
-def startFlaskAPI():
-  create_self_signed_cert()
-  try:
-    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-    ctx.check_hostname = False
-    ctx.load_cert_chain(certfile='/app/dockerapi_cert.pem', keyfile='/app/dockerapi_key.pem')
-  except:
-    print ("Cannot initialize TLS, retrying in 5s...")
-    time.sleep(5)
-  app.run(debug=False, host='0.0.0.0', port=443, threaded=True, ssl_context=ctx)
-
-api.add_resource(containers_get, '/containers/json')
-api.add_resource(container_get,  '/containers/<string:container_id>/json')
-api.add_resource(container_post, '/containers/<string:container_id>/<string:post_action>')
-
-if __name__ == '__main__':
-  api_thread = Thread(target=startFlaskAPI)
-  api_thread.daemon = True
-  api_thread.start()
-  killer = GracefulKiller()
-  while True:
-    time.sleep(1)
-    if killer.kill_now:
-      break
-  print ("Stopping dockerapi-mailcow")

data/Dockerfiles/dockerapi/main.py

@@ -0,0 +1,261 @@
+import os
+import sys
+import uvicorn
+import json
+import uuid
+import async_timeout
+import asyncio
+import aiodocker
+import docker
+import logging
+from logging.config import dictConfig
+from fastapi import FastAPI, Response, Request
+from modules.DockerApi import DockerApi
+from redis import asyncio as aioredis
+from contextlib import asynccontextmanager
+
+dockerapi = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+  global dockerapi
+
+  # Initialize a custom logger
+  logger = logging.getLogger("dockerapi")
+  logger.setLevel(logging.INFO)
+  # Configure the logger to output logs to the terminal
+  handler = logging.StreamHandler()
+  handler.setLevel(logging.INFO)
+  formatter = logging.Formatter("%(levelname)s:     %(message)s")
+  handler.setFormatter(formatter)
+  logger.addHandler(handler)
+
+  logger.info("Init APP")
+
+  # Init redis client
+  if os.environ['REDIS_SLAVEOF_IP'] != "":
+    redis_client = redis = await aioredis.from_url(f"redis://{os.environ['REDIS_SLAVEOF_IP']}:{os.environ['REDIS_SLAVEOF_PORT']}/0")
+  else:
+    redis_client = redis = await aioredis.from_url("redis://redis-mailcow:6379/0")
+
+  # Init docker clients
+  sync_docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
+  async_docker_client = aiodocker.Docker(url='unix:///var/run/docker.sock')
+
+  dockerapi = DockerApi(redis_client, sync_docker_client, async_docker_client, logger)
+
+  logger.info("Subscribe to redis channel")
+  # Subscribe to redis channel
+  dockerapi.pubsub = redis.pubsub()
+  await dockerapi.pubsub.subscribe("MC_CHANNEL")
+  asyncio.create_task(handle_pubsub_messages(dockerapi.pubsub))
+
+
+  yield
+
+  # Close docker connections
+  dockerapi.sync_docker_client.close()
+  await dockerapi.async_docker_client.close()
+
+  # Close redis
+  await dockerapi.pubsub.unsubscribe("MC_CHANNEL")
+  await dockerapi.redis_client.close()
+
+app = FastAPI(lifespan=lifespan)
+
+# Define Routes
+@app.get("/host/stats")
+async def get_host_update_stats():
+  global dockerapi
+
+  if dockerapi.host_stats_isUpdating == False:
+    asyncio.create_task(dockerapi.get_host_stats())
+    dockerapi.host_stats_isUpdating = True
+
+  while True:
+    if await dockerapi.redis_client.exists('host_stats'):
+      break
+    await asyncio.sleep(1.5)
+
+  stats = json.loads(await dockerapi.redis_client.get('host_stats'))
+  return Response(content=json.dumps(stats, indent=4), media_type="application/json")
+
+@app.get("/containers/{container_id}/json")
+async def get_container(container_id : str):
+  global dockerapi
+
+  if container_id and container_id.isalnum():
+    try:
+      for container in (await dockerapi.async_docker_client.containers.list()):
+        if container._id == container_id:
+          container_info = await container.show()
+          return Response(content=json.dumps(container_info, indent=4), media_type="application/json")
+     
+      res = {
+        "type": "danger",
+        "msg": "no container found"
+      }
+      return Response(content=json.dumps(res, indent=4), media_type="application/json")
+    except Exception as e:
+      res = {
+        "type": "danger",
+        "msg": str(e)
+      }
+      return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  else:
+    res = {
+      "type": "danger",
+      "msg": "no or invalid id defined"
+    }
+    return Response(content=json.dumps(res, indent=4), media_type="application/json")
+
+@app.get("/containers/json")
+async def get_containers():
+  global dockerapi
+
+  containers = {}
+  try:
+    for container in (await dockerapi.async_docker_client.containers.list()):
+      container_info = await container.show()
+      containers.update({container_info['Id']: container_info})
+    return Response(content=json.dumps(containers, indent=4), media_type="application/json")
+  except Exception as e:
+    res = {
+      "type": "danger",
+      "msg": str(e)
+    }
+    return Response(content=json.dumps(res, indent=4), media_type="application/json")
+
+@app.post("/containers/{container_id}/{post_action}")
+async def post_containers(container_id : str, post_action : str, request: Request):
+  global dockerapi
+
+  try : 
+    request_json = await request.json()
+  except Exception as err:
+    request_json = {}
+
+  if container_id and container_id.isalnum() and post_action:
+    try:
+      """Dispatch container_post api call"""
+      if post_action == 'exec':
+        if not request_json or not 'cmd' in request_json:
+          res = {
+            "type": "danger",
+            "msg": "cmd is missing"
+          }
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+        if not request_json or not 'task' in request_json:
+          res = {
+            "type": "danger",
+            "msg": "task is missing"
+          }
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+
+        api_call_method_name = '__'.join(['container_post', str(post_action), str(request_json['cmd']), str(request_json['task']) ])
+      else:
+        api_call_method_name = '__'.join(['container_post', str(post_action) ])
+
+      api_call_method = getattr(dockerapi, api_call_method_name, lambda container_id: Response(content=json.dumps({'type': 'danger', 'msg':'container_post - unknown api call' }, indent=4), media_type="application/json"))
+
+      dockerapi.logger.info("api call: %s, container_id: %s" % (api_call_method_name, container_id))
+      return api_call_method(request_json, container_id=container_id)
+    except Exception as e:
+      dockerapi.logger.error("error - container_post: %s" % str(e))
+      res = {
+        "type": "danger",
+        "msg": str(e)
+      }
+      return Response(content=json.dumps(res, indent=4), media_type="application/json")
+
+  else:
+    res = {
+      "type": "danger",
+      "msg": "invalid container id or missing action"
+    }
+    return Response(content=json.dumps(res, indent=4), media_type="application/json")
+
+@app.post("/container/{container_id}/stats/update")
+async def post_container_update_stats(container_id : str):
+  global dockerapi
+
+  # start update task for container if no task is running
+  if container_id not in dockerapi.containerIds_to_update:
+    asyncio.create_task(dockerapi.get_container_stats(container_id))
+    dockerapi.containerIds_to_update.append(container_id)
+
+  while True:
+    if await dockerapi.redis_client.exists(container_id + '_stats'):
+      break
+    await asyncio.sleep(1.5)
+
+  stats = json.loads(await dockerapi.redis_client.get(container_id + '_stats'))
+  return Response(content=json.dumps(stats, indent=4), media_type="application/json")
+  
+
+# PubSub Handler
+async def handle_pubsub_messages(channel: aioredis.client.PubSub):
+  global dockerapi
+
+  while True:
+    try:
+      async with async_timeout.timeout(60):
+        message = await channel.get_message(ignore_subscribe_messages=True, timeout=30)
+        if message is not None:
+          # Parse message
+          data_json = json.loads(message['data'].decode('utf-8'))
+          dockerapi.logger.info(f"PubSub Received - {json.dumps(data_json)}")
+
+          # Handle api_call
+          if 'api_call' in data_json:
+            # api_call: container_post
+            if data_json['api_call'] == "container_post":
+              if 'post_action' in data_json and 'container_name' in data_json:
+                try:
+                  """Dispatch container_post api call"""
+                  request_json = {}
+                  if data_json['post_action'] == 'exec':
+                    if 'request' in data_json:
+                      request_json = data_json['request']
+                      if 'cmd' in request_json:
+                        if 'task' in request_json:
+                          api_call_method_name = '__'.join(['container_post', str(data_json['post_action']), str(request_json['cmd']), str(request_json['task']) ])
+                        else:
+                          dockerapi.logger.error("api call: task missing")
+                      else:
+                        dockerapi.logger.error("api call: cmd missing")
+                    else:
+                      dockerapi.logger.error("api call: request missing")
+                  else:
+                    api_call_method_name = '__'.join(['container_post', str(data_json['post_action'])])
+
+                  if api_call_method_name:
+                    api_call_method = getattr(dockerapi, api_call_method_name)
+                    if api_call_method:
+                      dockerapi.logger.info("api call: %s, container_name: %s" % (api_call_method_name, data_json['container_name']))
+                      api_call_method(request_json, container_name=data_json['container_name'])
+                    else:
+                      dockerapi.logger.error("api call not found: %s, container_name: %s" % (api_call_method_name, data_json['container_name']))
+                except Exception as e:
+                  dockerapi.logger.error("container_post: %s" % str(e))
+              else:
+                dockerapi.logger.error("api call: missing container_name, post_action or request")
+            else:
+              dockerapi.logger.error("Unknwon PubSub recieved - %s" % json.dumps(data_json))
+          else:
+            dockerapi.logger.error("Unknwon PubSub recieved - %s" % json.dumps(data_json))
+              
+        await asyncio.sleep(0.0)
+    except asyncio.TimeoutError:
+      pass
+
+if __name__ == '__main__':
+  uvicorn.run(
+    app,
+    host="0.0.0.0",
+    port=443,
+    ssl_certfile="/app/dockerapi_cert.pem",
+    ssl_keyfile="/app/dockerapi_key.pem",
+    log_level="info",
+    loop="none"
+  )

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -0,0 +1,487 @@
+import psutil
+import sys
+import os
+import re
+import time
+import json
+import asyncio
+import platform
+from datetime import datetime
+from fastapi import FastAPI, Response, Request
+
+class DockerApi:
+  def __init__(self, redis_client, sync_docker_client, async_docker_client, logger):
+    self.redis_client = redis_client
+    self.sync_docker_client = sync_docker_client
+    self.async_docker_client = async_docker_client
+    self.logger = logger
+
+    self.host_stats_isUpdating = False
+    self.containerIds_to_update = []
+
+  # api call: container_post - post_action: stop
+  def container_post__stop(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(all=True, filters=filters):
+      container.stop()
+
+    res = { 'type': 'success', 'msg': 'command completed successfully'}
+    return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: start
+  def container_post__start(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(all=True, filters=filters):
+      container.start()
+
+    res = { 'type': 'success', 'msg': 'command completed successfully'}
+    return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: restart
+  def container_post__restart(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(all=True, filters=filters):
+      container.restart()
+
+    res = { 'type': 'success', 'msg': 'command completed successfully'}
+    return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: top
+  def container_post__top(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(all=True, filters=filters):
+      res = { 'type': 'success', 'msg': container.top()}
+      return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: stats
+  def container_post__stats(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(all=True, filters=filters):
+      for stat in container.stats(decode=True, stream=True):
+        res = { 'type': 'success', 'msg': stat}
+        return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: exec - cmd: mailq - task: delete
+  def container_post__exec__mailq__delete(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'items' in request_json:
+      r = re.compile("^[0-9a-fA-F]+$")
+      filtered_qids = filter(r.match, request_json['items'])
+      if filtered_qids:
+        flagged_qids = ['-d %s' % i for i in filtered_qids]
+        sanitized_string = str(' '.join(flagged_qids))
+        for container in self.sync_docker_client.containers.list(filters=filters):
+          postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
+          return self.exec_run_handler('generic', postsuper_r)
+  # api call: container_post - post_action: exec - cmd: mailq - task: hold
+  def container_post__exec__mailq__hold(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'items' in request_json:
+      r = re.compile("^[0-9a-fA-F]+$")
+      filtered_qids = filter(r.match, request_json['items'])
+      if filtered_qids:
+        flagged_qids = ['-h %s' % i for i in filtered_qids]
+        sanitized_string = str(' '.join(flagged_qids))
+        for container in self.sync_docker_client.containers.list(filters=filters):
+          postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
+          return self.exec_run_handler('generic', postsuper_r)
+  # api call: container_post - post_action: exec - cmd: mailq - task: cat
+  def container_post__exec__mailq__cat(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'items' in request_json:
+      r = re.compile("^[0-9a-fA-F]+$")
+      filtered_qids = filter(r.match, request_json['items'])
+      if filtered_qids:
+        sanitized_string = str(' '.join(filtered_qids))
+
+        for container in self.sync_docker_client.containers.list(filters=filters):
+          postcat_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postcat -q " + sanitized_string], user='postfix')
+        if not postcat_return:
+          postcat_return = 'err: invalid'
+        return self.exec_run_handler('utf8_text_only', postcat_return)
+  # api call: container_post - post_action: exec - cmd: mailq - task: unhold
+  def container_post__exec__mailq__unhold(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'items' in request_json:
+      r = re.compile("^[0-9a-fA-F]+$")
+      filtered_qids = filter(r.match, request_json['items'])
+      if filtered_qids:
+        flagged_qids = ['-H %s' % i for i in filtered_qids]
+        sanitized_string = str(' '.join(flagged_qids))
+        for container in self.sync_docker_client.containers.list(filters=filters):
+          postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
+          return self.exec_run_handler('generic', postsuper_r)
+  # api call: container_post - post_action: exec - cmd: mailq - task: deliver
+  def container_post__exec__mailq__deliver(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'items' in request_json:
+      r = re.compile("^[0-9a-fA-F]+$")
+      filtered_qids = filter(r.match, request_json['items'])
+      if filtered_qids:
+        flagged_qids = ['-i %s' % i for i in filtered_qids]
+        for container in self.sync_docker_client.containers.list(filters=filters):
+          for i in flagged_qids:
+            postqueue_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postqueue " + i], user='postfix')
+            # todo: check each exit code
+          res = { 'type': 'success', 'msg': 'Scheduled immediate delivery'}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")        
+  # api call: container_post - post_action: exec - cmd: mailq - task: list
+  def container_post__exec__mailq__list(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      mailq_return = container.exec_run(["/usr/sbin/postqueue", "-j"], user='postfix')
+      return self.exec_run_handler('utf8_text_only', mailq_return)
+  # api call: container_post - post_action: exec - cmd: mailq - task: flush
+  def container_post__exec__mailq__flush(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      postqueue_r = container.exec_run(["/usr/sbin/postqueue", "-f"], user='postfix')
+      return self.exec_run_handler('generic', postqueue_r)
+  # api call: container_post - post_action: exec - cmd: mailq - task: super_delete
+  def container_post__exec__mailq__super_delete(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      postsuper_r = container.exec_run(["/usr/sbin/postsuper", "-d", "ALL"])
+      return self.exec_run_handler('generic', postsuper_r)
+  # api call: container_post - post_action: exec - cmd: system - task: fts_rescan
+  def container_post__exec__system__fts_rescan(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'username' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        rescan_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/doveadm fts rescan -u '" + request_json['username'].replace("'", "'\\''") + "'"], user='vmail')
+        if rescan_return.exit_code == 0:
+          res = { 'type': 'success', 'msg': 'fts_rescan: rescan triggered'}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+        else:
+          res = { 'type': 'warning', 'msg': 'fts_rescan error'}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+    if 'all' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        rescan_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/doveadm fts rescan -A"], user='vmail')
+        if rescan_return.exit_code == 0:
+          res = { 'type': 'success', 'msg': 'fts_rescan: rescan triggered'}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+        else:
+          res = { 'type': 'warning', 'msg': 'fts_rescan error'}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: exec - cmd: system - task: df
+  def container_post__exec__system__df(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'dir' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        df_return = container.exec_run(["/bin/bash", "-c", "/bin/df -H '" + request_json['dir'].replace("'", "'\\''") + "' | /usr/bin/tail -n1 | /usr/bin/tr -s [:blank:] | /usr/bin/tr ' ' ','"], user='nobody')
+        if df_return.exit_code == 0:
+          return df_return.output.decode('utf-8').rstrip()
+        else:
+          return "0,0,0,0,0,0"
+  # api call: container_post - post_action: exec - cmd: system - task: mysql_upgrade
+  def container_post__exec__system__mysql_upgrade(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      sql_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n"], user='mysql')
+      if sql_return.exit_code == 0:
+        matched = False
+        for line in sql_return.output.decode('utf-8').split("\n"):
+          if 'is already upgraded to' in line:
+            matched = True
+        if matched:
+          res = { 'type': 'success', 'msg':'mysql_upgrade: already upgraded', 'text': sql_return.output.decode('utf-8')}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+        else:
+          container.restart()
+          res = { 'type': 'warning', 'msg':'mysql_upgrade: upgrade was applied', 'text': sql_return.output.decode('utf-8')}
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+      else:
+        res = { 'type': 'error', 'msg': 'mysql_upgrade: error running command', 'text': sql_return.output.decode('utf-8')}
+        return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: exec - cmd: system - task: mysql_tzinfo_to_sql
+  def container_post__exec__system__mysql_tzinfo_to_sql(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      sql_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/mysql_tzinfo_to_sql /usr/share/zoneinfo | /bin/sed 's/Local time zone must be set--see zic manual page/FCTY/' | /usr/bin/mysql -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "' mysql \n"], user='mysql')
+      if sql_return.exit_code == 0:
+        res = { 'type': 'info', 'msg': 'mysql_tzinfo_to_sql: command completed successfully', 'text': sql_return.output.decode('utf-8')}
+        return Response(content=json.dumps(res, indent=4), media_type="application/json")
+      else:
+        res = { 'type': 'error', 'msg': 'mysql_tzinfo_to_sql: error running command', 'text': sql_return.output.decode('utf-8')}
+        return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: exec - cmd: reload - task: dovecot
+  def container_post__exec__reload__dovecot(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      reload_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/dovecot reload"])
+      return self.exec_run_handler('generic', reload_return)
+  # api call: container_post - post_action: exec - cmd: reload - task: postfix
+  def container_post__exec__reload__postfix(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      reload_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postfix reload"])
+      return self.exec_run_handler('generic', reload_return)
+  # api call: container_post - post_action: exec - cmd: reload - task: nginx
+  def container_post__exec__reload__nginx(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      reload_return = container.exec_run(["/bin/sh", "-c", "/usr/sbin/nginx -s reload"])
+      return self.exec_run_handler('generic', reload_return)
+  # api call: container_post - post_action: exec - cmd: sieve - task: list
+  def container_post__exec__sieve__list(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'username' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        sieve_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/doveadm sieve list -u '" + request_json['username'].replace("'", "'\\''") + "'"])
+        return self.exec_run_handler('utf8_text_only', sieve_return)
+  # api call: container_post - post_action: exec - cmd: sieve - task: print
+  def container_post__exec__sieve__print(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'username' in request_json and 'script_name' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' '" + request_json['script_name'].replace("'", "'\\''") + "'"]  
+        sieve_return = container.exec_run(cmd)
+        return self.exec_run_handler('utf8_text_only', sieve_return)
+  # api call: container_post - post_action: exec - cmd: maildir - task: cleanup
+  def container_post__exec__maildir__cleanup(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'maildir' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        sane_name = re.sub(r'\W+', '', request_json['maildir'])
+        vmail_name = request_json['maildir'].replace("'", "'\\''")
+        cmd_vmail = "if [[ -d '/var/vmail/" + vmail_name + "' ]]; then /bin/mv '/var/vmail/" + vmail_name + "' '/var/vmail/_garbage/" + str(int(time.time())) + "_" + sane_name + "'; fi"
+        index_name = request_json['maildir'].split("/")
+        if len(index_name) > 1:
+          index_name = index_name[1].replace("'", "'\\''") + "@" + index_name[0].replace("'", "'\\''")
+          cmd_vmail_index = "if [[ -d '/var/vmail_index/" + index_name + "' ]]; then /bin/mv '/var/vmail_index/" + index_name + "' '/var/vmail/_garbage/" + str(int(time.time())) + "_" + sane_name + "_index'; fi"
+          cmd = ["/bin/bash", "-c", cmd_vmail + " && " + cmd_vmail_index]
+        else:
+          cmd = ["/bin/bash", "-c", cmd_vmail]
+        maildir_cleanup = container.exec_run(cmd, user='vmail')
+        return self.exec_run_handler('generic', maildir_cleanup)
+  # api call: container_post - post_action: exec - cmd: rspamd - task: worker_password
+  def container_post__exec__rspamd__worker_password(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'raw' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        cmd = "/usr/bin/rspamadm pw -e -p '" + request_json['raw'].replace("'", "'\\''") + "' 2> /dev/null"
+        cmd_response = self.exec_cmd_container(container, cmd, user="_rspamd")
+
+        matched = False
+        for line in cmd_response.split("\n"):
+          if '$2$' in line:
+            hash = line.strip()
+            hash_out = re.search('\$2\$.+$', hash).group(0)
+            rspamd_passphrase_hash = re.sub('[^0-9a-zA-Z\$]+', '', hash_out.rstrip())
+            rspamd_password_filename = "/etc/rspamd/override.d/worker-controller-password.inc"
+            cmd = '''/bin/echo 'enable_password = "%s";' > %s && cat %s''' % (rspamd_passphrase_hash, rspamd_password_filename, rspamd_password_filename)
+            cmd_response = self.exec_cmd_container(container, cmd, user="_rspamd")
+            if rspamd_passphrase_hash.startswith("$2$") and rspamd_passphrase_hash in cmd_response:
+              container.restart()
+              matched = True
+        if matched:
+          res = { 'type': 'success', 'msg': 'command completed successfully' }
+          self.logger.info('success changing Rspamd password')
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+        else:
+          self.logger.error('failed changing Rspamd password')
+          res = { 'type': 'danger', 'msg': 'command did not complete' }
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
+
+  # Collect host stats
+  async def get_host_stats(self, wait=5):
+    try:
+      system_time = datetime.now()
+      host_stats = {
+        "cpu": {
+          "cores": psutil.cpu_count(),
+          "usage": psutil.cpu_percent()
+        },
+        "memory": {
+          "total": psutil.virtual_memory().total,
+          "usage": psutil.virtual_memory().percent,
+          "swap": psutil.swap_memory()
+        },
+        "uptime": time.time() - psutil.boot_time(),
+        "system_time": system_time.strftime("%d.%m.%Y %H:%M:%S"),
+        "architecture": platform.machine()
+      }
+
+      await self.redis_client.set('host_stats', json.dumps(host_stats), ex=10)
+    except Exception as e:
+      res = {
+        "type": "danger",
+        "msg": str(e)
+      }
+
+    await asyncio.sleep(wait)
+    self.host_stats_isUpdating = False
+  # Collect container stats
+  async def get_container_stats(self, container_id, wait=5, stop=False):
+    if container_id and container_id.isalnum():
+      try:
+        for container in (await self.async_docker_client.containers.list()):
+          if container._id == container_id:
+            res = await container.stats(stream=False)
+
+            if await self.redis_client.exists(container_id + '_stats'):
+              stats = json.loads(await self.redis_client.get(container_id + '_stats'))
+            else:
+              stats = []
+            stats.append(res[0])
+            if len(stats) > 3:
+              del stats[0]
+            await self.redis_client.set(container_id + '_stats', json.dumps(stats), ex=60)
+      except Exception as e:
+        res = {
+          "type": "danger",
+          "msg": str(e)
+        }
+    else:
+      res = {
+        "type": "danger",
+        "msg": "no or invalid id defined"
+      }
+
+    await asyncio.sleep(wait)
+    if stop == True:
+      # update task was called second time, stop
+      self.containerIds_to_update.remove(container_id)
+    else:
+      # call update task a second time
+      await self.get_container_stats(container_id, wait=0, stop=True)
+
+  def exec_cmd_container(self, container, cmd, user, timeout=2, shell_cmd="/bin/bash"):
+    def recv_socket_data(c_socket, timeout):
+      c_socket.setblocking(0)
+      total_data=[]
+      data=''
+      begin=time.time()
+      while True:
+        if total_data and time.time()-begin > timeout:
+          break
+        elif time.time()-begin > timeout*2:
+          break
+        try:
+          data = c_socket.recv(8192)
+          if data:
+            total_data.append(data.decode('utf-8'))
+            #change the beginning time for measurement
+            begin=time.time()
+          else:
+            #sleep for sometime to indicate a gap
+            time.sleep(0.1)
+            break
+        except:
+          pass
+      return ''.join(total_data)
+      
+    try :
+      socket = container.exec_run([shell_cmd], stdin=True, socket=True, user=user).output._sock
+      if not cmd.endswith("\n"):
+        cmd = cmd + "\n"
+      socket.send(cmd.encode('utf-8'))
+      data = recv_socket_data(socket, timeout)
+      socket.close()
+      return data
+    except Exception as e:
+      self.logger.error("error - exec_cmd_container: %s" % str(e))
+      traceback.print_exc(file=sys.stdout)
+
+  def exec_run_handler(self, type, output):
+    if type == 'generic':
+      if output.exit_code == 0:
+        res = { 'type': 'success', 'msg': 'command completed successfully' }
+        return Response(content=json.dumps(res, indent=4), media_type="application/json")
+      else:
+        res = { 'type': 'danger', 'msg': 'command failed: ' + output.output.decode('utf-8') }
+        return Response(content=json.dumps(res, indent=4), media_type="application/json")
+    if type == 'utf8_text_only':
+      return Response(content=output.output.decode('utf-8'), media_type="text/plain")

data/Dockerfiles/dovecot/Dockerfile

@@ -1,112 +1,125 @@
-FROM debian:buster-slim
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM alpine:3.19
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
 
-ARG DEBIAN_FRONTEND=noninteractive
-ARG DOVECOT=2.3.11.3
+# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
+ARG GOSU_VERSION=1.16
 ENV LC_ALL C
 
+
 # Add groups and users before installing Dovecot to not break compatibility
-RUN groupadd -g 5000 vmail \
-  && groupadd -g 401 dovecot \
-  && groupadd -g 402 dovenull \
-  && useradd -g vmail -u 5000 vmail -d /var/vmail \
-  && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \
-  && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull \
-  && touch /etc/default/locale \
-  && apt-get update \
-  && apt-get -y --no-install-recommends install \
-  apt-transport-https \
+RUN addgroup -g 5000 vmail \
+  && addgroup -g 401 dovecot \
+  && addgroup -g 402 dovenull \
+  && sed -i "s/999/99/" /etc/group \
+  && addgroup -g 999 sogo \
+  && addgroup nobody sogo \
+  && adduser -D -u 5000 -G vmail -h /var/vmail vmail \
+  && adduser -D -G dovecot -u 401 -h /dev/null -s /sbin/nologin dovecot \
+  && adduser -D -G dovenull -u 402 -h /dev/null -s /sbin/nologin dovenull \
+  && apk add --no-cache --update \
+  bash \
+  bind-tools \
+  findutils \
+  envsubst \
   ca-certificates \
-  cpanminus \
-  cron \
   curl \
-  dnsutils \
-  dirmngr \
-  gettext \
-  gnupg2 \
   jq \
-  libauthen-ntlm-perl \
-  libcgi-pm-perl \
-  libcrypt-openssl-rsa-perl \
-  libcrypt-ssleay-perl \
-  libdata-uniqid-perl \
-  libdbd-mysql-perl \
-  libdbi-perl \
-  libdigest-hmac-perl \
-  libdist-checkconflicts-perl \
-  libencode-imaputf7-perl \
-  libfile-copy-recursive-perl \
-  libfile-tail-perl \
-  libhtml-parser-perl \
-  libio-compress-perl \
-  libio-socket-inet6-perl \
-  libio-socket-ssl-perl \
-  libio-tee-perl \
-  libipc-run-perl \
-  libjson-webtoken-perl \
-  liblockfile-simple-perl \
-  libmail-imapclient-perl \
-  libmodule-implementation-perl \
-  libmodule-scandeps-perl \
-  libnet-ssleay-perl \
-  libpackage-stash-perl \
-  libpackage-stash-xs-perl \
-  libpar-packer-perl \
-  libparse-recdescent-perl \
-  libproc-processtable-perl \
-  libreadonly-perl \
-  libregexp-common-perl \
-  libsys-meminfo-perl \
-  libterm-readkey-perl \
-  libtest-deep-perl \
-  libtest-fatal-perl \
-  libtest-mock-guard-perl \
-  libtest-mockobject-perl \
-  libtest-nowarnings-perl \
-  libtest-pod-perl \
-  libtest-requires-perl \
-  libtest-simple-perl \
-  libtest-warn-perl \
-  libtry-tiny-perl \
-  libunicode-string-perl \
-  liburi-perl \
-  libwww-perl \
+  lua \
+  lua-cjson \
+  lua-socket \
   lua-sql-mysql \
+  lua5.3-sql-mysql \
+  icu-data-full \
+  mariadb-connector-c \
+  lua-sec \
+  mariadb-dev \
+  glib-dev \
+  gcompat \
   mariadb-client \
+  perl \
+  perl-dev \
+  perl-ntlm \
+  perl-cgi \
+  perl-crypt-openssl-rsa \
+  perl-utils \
+  perl-crypt-ssleay \
+  perl-data-uniqid \
+  perl-dbd-mysql \
+  perl-dbi \
+  perl-digest-hmac \
+  perl-dist-checkconflicts \
+  perl-encode-imaputf7 \
+  perl-file-copy-recursive \
+  perl-file-tail \
+  perl-io-socket-inet6 \
+  perl-io-gzip \
+  perl-io-socket-ssl \
+  perl-io-tee \
+  perl-ipc-run \
+  perl-json-webtoken \
+  perl-mail-imapclient \
+  perl-module-implementation \
+  perl-module-scandeps \
+  perl-net-ssleay \
+  perl-package-stash \
+  perl-package-stash-xs \
+  perl-par-packer \
+  perl-parse-recdescent \
+  perl-lockfile-simple --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
+  libproc \
+  perl-readonly \
+  perl-regexp-common \
+  perl-sys-meminfo \
+  perl-term-readkey \
+  perl-test-deep \
+  perl-test-fatal \
+  perl-test-mockobject \
+  perl-test-mock-guard \
+  perl-test-pod \
+  perl-test-requires \
+  perl-test-simple \
+  perl-test-warn \
+  perl-try-tiny \
+  perl-unicode-string \
+  perl-proc-processtable \
+  perl-app-cpanminus \
   procps \
-  python3-pip \
-  redis-server \
-  supervisor \
+  python3 \
+  py3-mysqlclient \
+  py3-html2text \
+  py3-jinja2 \
+  py3-redis \
+  redis \
   syslog-ng \
-  syslog-ng-core \
-  syslog-ng-mod-redis \
-  && apt-key adv --fetch-keys https://repo.dovecot.org/DOVECOT-REPO-GPG \
-  && echo "deb https://repo.dovecot.org/ce-${DOVECOT}/debian/buster buster main" > /etc/apt/sources.list.d/dovecot.list \
-  && apt-get update \
-  && apt-get -y --no-install-recommends install \
-  dovecot-lua \
-  dovecot-managesieved \
-  dovecot-sieve \
+  syslog-ng-redis \
+  syslog-ng-json \
+  supervisor \
+  tzdata \
+  wget \
+  dovecot \
+  dovecot-dev \
   dovecot-lmtpd \
+  dovecot-lua \
   dovecot-ldap \
   dovecot-mysql \
-  dovecot-core \
+  dovecot-sql \
+  dovecot-submissiond \
+  dovecot-pigeonhole-plugin \
   dovecot-pop3d \
-  dovecot-imapd \
-  dovecot-solr \
-  && pip3 install mysql-connector-python html2text jinja2 redis \
-  && apt-get autoremove --purge -y \
-  && apt-get autoclean \
-  && rm -rf /var/lib/apt/lists/* \
-  && rm -rf /tmp/* /var/tmp/* /etc/cron.daily/* /root/.cache/
+  dovecot-fts-solr \
+  && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
+  && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
+  && chmod +x /usr/local/bin/gosu \
+  && gosu nobody true
+
+#RUN cpan LockFile::Simple
 
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
-COPY postlogin.sh /usr/local/bin/postlogin.sh
-COPY imapsync_cron.pl /usr/local/bin/imapsync_cron.pl
+COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham
@@ -121,4 +134,4 @@ COPY quota_notify.py /usr/local/bin/quota_notify.py
 COPY repl_health.sh /usr/local/bin/repl_health.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
+CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -7,7 +7,7 @@ while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${D
   sleep 2
 done
 
-until dig +short mailcow.email @unbound > /dev/null; do
+until dig +short mailcow.email > /dev/null; do
   echo "Waiting for DNS..."
   sleep 1
 done
@@ -28,7 +28,7 @@ ${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
 
 # Create missing directories
 [[ ! -d /etc/dovecot/sql/ ]] && mkdir -p /etc/dovecot/sql/
-[[ ! -d /etc/dovecot/lua/ ]] && mkdir -p /etc/dovecot/lua/
+[[ ! -d /etc/dovecot/auth/ ]] && mkdir -p /etc/dovecot/auth/
 [[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage
 [[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve
 [[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo
@@ -60,14 +60,6 @@ map {
 }
 EOF
 
-# Write last logins to Redis
-if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
-  cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
-  echo -n "redis:host=${REDIS_SLAVEOF_IP}:port=${REDIS_SLAVEOF_PORT}" > /etc/dovecot/last_login
-else
-  echo -n "redis:host=${IPV4_NETWORK}.249:port=6379" > /etc/dovecot/last_login
-fi
-
 # Create dict used for sieve pre and postfilters
 cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf
 # Autogenerated by mailcow
@@ -118,13 +110,13 @@ EOF
 echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
 
 if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify replication last_login' > /etc/dovecot/mail_plugins
-echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify replication mail_log last_login' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl notify replication' > /etc/dovecot/mail_plugins_lmtp
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 else
-echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify fts fts_solr replication last_login' > /etc/dovecot/mail_plugins
-echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log fts fts_solr replication last_login' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl fts fts_solr notify replication' > /etc/dovecot/mail_plugins_lmtp
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_solr listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_solr listescape replication' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_solr notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 fi
 chmod 644 /etc/dovecot/mail_plugins /etc/dovecot/mail_plugins_imap /etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl
 
@@ -136,55 +128,6 @@ user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format
 iterate_query = SELECT username FROM mailbox WHERE active = '1' OR active = '2';
 EOF
 
-# Create pass dict for Dovecot
-cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-passdb.conf
-# Autogenerated by mailcow
-driver = mysql
-connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
-default_pass_scheme = SSHA256
-password_query = SELECT password FROM mailbox WHERE active = '1' AND username = '%u' AND domain IN (SELECT domain FROM domain WHERE domain='%d' AND active='1') AND JSON_UNQUOTE(JSON_VALUE(attributes, '$.force_pw_update')) != '1' AND (JSON_UNQUOTE(JSON_VALUE(attributes, '$.%s_access')) = '1' OR ('%s' != 'imap' AND '%s' != 'pop3'))
-EOF
-
-cat <<EOF > /etc/dovecot/lua/app-passdb.lua
-function auth_password_verify(req, pass)
-  if req.domain == nil then
-    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
-  end
-  if cur == nil then
-    script_init()
-  end
-  local cur,errorString = con:execute(string.format([[SELECT mailbox, password FROM app_passwd
-    WHERE mailbox = '%s'
-      AND active = '1'
-      AND domain IN (SELECT domain FROM domain WHERE domain='%s' AND active='1')]], con:escape(req.user), con:escape(req.domain)))
-  local row = cur:fetch ({}, "a")
-  while row do
-    if req.password_verify(req, row.password, pass) == 1 then
-      cur:close()
-      return dovecot.auth.PASSDB_RESULT_OK, "password=" .. pass
-    end
-    row = cur:fetch (row, "a")
-  end
-  return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
-end
-
-function auth_passdb_lookup(req)
-   return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, ""
-end
-
-function script_init()
-  mysql = require "luasql.mysql"
-  env  = mysql.mysql()
-  con = env:connect("__DBNAME__","__DBUSER__","__DBPASS__","localhost")
-  return 0
-end
-
-function script_deinit()
-  con:close()
-  env:close()
-end
-EOF
-
 # Migrate old sieve_after file
 [[ -f /etc/dovecot/sieve_after ]] && mv /etc/dovecot/sieve_after /etc/dovecot/global_sieve_after
 # Create global sieve scripts
@@ -199,6 +142,8 @@ if [[ $(stat -c %U /var/vmail_index) != "vmail" ]] ; then chown -R vmail:vmail /
 
 # Cleanup random user maildirs
 rm -rf /var/vmail/mailcow.local/*
+# Cleanup PIDs
+[[ -f /tmp/quarantine_notify.pid ]] && rm /tmp/quarantine_notify.pid
 
 # create sni configuration
 echo "" > /etc/dovecot/sni.conf
@@ -244,6 +189,7 @@ namespace {
 }
 EOF
 
+
 cat <<EOF > /etc/dovecot/sogo_trusted_ip.conf
 # Autogenerated by mailcow
 remote ${IPV4_NETWORK}.248 {
@@ -251,36 +197,35 @@ remote ${IPV4_NETWORK}.248 {
 }
 EOF
 
-if [[ "${ALLOW_ADMIN_EMAIL_LOGIN}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-    # Create random master Password for SOGo 'login as user' via proxy auth
-    RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1)
-    echo -n ${RAND_PASS} > /etc/phpfpm/sogo-sso.pass
-    cat <<EOF > /etc/dovecot/sogo-sso.conf
+# Create random master Password for SOGo SSO
+RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1)
+echo -n ${RAND_PASS} > /etc/phpfpm/sogo-sso.pass
+cat <<EOF > /etc/dovecot/sogo-sso.conf
 # Autogenerated by mailcow
 passdb {
   driver = static
   args = allow_real_nets=${IPV4_NETWORK}.248/32 password={plain}${RAND_PASS}
 }
 EOF
-else
-    rm -f /etc/dovecot/sogo-sso.pass
-    rm -f /etc/dovecot/sogo-sso.conf
-fi
 
-# Hard-code env vars to scripts due to cron not passing them to the scripts
-sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/imapsync_cron.pl /usr/local/bin/quarantine_notify.py /usr/local/bin/clean_q_aged.sh /etc/dovecot/lua/app-passdb.lua
-sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/imapsync_cron.pl /usr/local/bin/quarantine_notify.py /usr/local/bin/clean_q_aged.sh /etc/dovecot/lua/app-passdb.lua
-sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/imapsync_cron.pl /usr/local/bin/quarantine_notify.py /usr/local/bin/clean_q_aged.sh /etc/dovecot/lua/app-passdb.lua
-sed -i "s/__LOG_LINES__/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh
 if [[ "${MASTER}" =~ ^([nN][oO]|[nN])+$ ]]; then
-# Toggling MASTER will result in a rebuild of containers, so the quota script will be recreated
-cat <<'EOF' > /usr/local/bin/quota_notify.py
+  # Toggling MASTER will result in a rebuild of containers, so the quota script will be recreated
+  cat <<'EOF' > /usr/local/bin/quota_notify.py
 #!/usr/bin/python3
 import sys
 sys.exit()
 EOF
 fi
 
+# Set mail_replica for HA setups
+if [[ -n ${MAILCOW_REPLICA_IP} && -n ${DOVEADM_REPLICA_PORT} ]]; then
+  cat <<EOF > /etc/dovecot/mail_replica.conf
+# Autogenerated by mailcow
+mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
+EOF
+fi
+
+
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
 	openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem
@@ -296,10 +241,18 @@ sievec /var/vmail/sieve/global_sieve_after.sieve
 sievec /usr/lib/dovecot/sieve/report-spam.sieve
 sievec /usr/lib/dovecot/sieve/report-ham.sieve
 
+for file in /var/vmail/*/*/sieve/*.sieve ; do
+  if [[ "$file" == "/var/vmail/*/*/sieve/*.sieve" ]]; then
+    continue
+  fi
+  sievec "$file" "$(dirname "$file")/../.dovecot.svbin"
+  chown vmail:vmail "$(dirname "$file")/../.dovecot.svbin"
+done
+
 # Fix permissions
 chown root:root /etc/dovecot/sql/*.conf
-chown root:dovecot /etc/dovecot/sql/dovecot-dict-sql-sieve* /etc/dovecot/sql/dovecot-dict-sql-quota* /etc/dovecot/lua/app-passdb.lua
-chmod 640 /etc/dovecot/sql/*.conf /etc/dovecot/lua/app-passdb.lua
+chown root:dovecot /etc/dovecot/sql/dovecot-dict-sql-sieve* /etc/dovecot/sql/dovecot-dict-sql-quota* /etc/dovecot/auth/passwd-verify.lua
+chmod 640 /etc/dovecot/sql/*.conf /etc/dovecot/auth/passwd-verify.lua
 chown -R vmail:vmail /var/vmail/sieve
 chown -R vmail:vmail /var/volatile
 chown -R vmail:vmail /var/vmail_index
@@ -308,8 +261,7 @@ chmod g+rw /dev/console
 chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
-  /usr/local/bin/imapsync_cron.pl \
-  /usr/local/bin/postlogin.sh \
+  /usr/local/bin/imapsync_runner.pl \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -319,27 +271,6 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/local/bin/quota_notify.py \
   /usr/local/bin/repl_health.sh
 
-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-# Setup cronjobs
-echo '* * * * *    root  /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync
-#echo '30 3 * * *   vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
-echo '* * * * *    vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs
-echo '25 * * * *   vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
-echo '30 1 * * *   root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
-echo '0 2 * * *    root  /usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize
-echo '*/20 * * * * vmail /usr/local/bin/quarantine_notify.py >> /dev/console 2>&1' > /etc/cron.d/quarantine_notify
-echo '15 4 * * * vmail /usr/local/bin/clean_q_aged.sh >> /dev/console 2>&1' > /etc/cron.d/clean_q_aged
-echo '*/5 * * * *  vmail /usr/local/bin/repl_health.sh >> /dev/console 2>&1' > /etc/cron.d/repl_health
-else
-echo '25 * * * *   vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
-echo '30 1 * * *   root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
-echo '0 2 * * *    root  /usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize
-echo '*/5 * * * *  vmail /usr/local/bin/repl_health.sh >> /dev/console 2>&1' > /etc/cron.d/repl_health
-fi
-
-# Fix more than 1 hardlink issue
-touch /etc/crontab /etc/cron.*/*
-
 # Prepare environment file for cronjobs
 printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh
 
@@ -390,6 +321,10 @@ done
 
 # For some strange, unknown and stupid reason, Dovecot may run into a race condition, when this file is not touched before it is read by dovecot/auth
 # May be related to something inside Docker, I seriously don't know
-touch /etc/dovecot/lua/app-passdb.lua
+touch /etc/dovecot/auth/passwd-verify.lua
+
+if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
+  cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
+fi
 
 exec "$@"

data/Dockerfiles/dovecot/imapsync_runner.pl

@@ -36,11 +36,11 @@ sub qqw($) {
 }
 
 $run_dir="/tmp";
-$dsn = 'DBI:mysql:database=__DBNAME__;mysql_socket=/var/run/mysqld/mysqld.sock';
+$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
 $lock_file = $run_dir . "/imapsync_busy";
 $lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
 $lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, '__DBUSER__', '__DBPASS__', {
+$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
   mysql_auto_reconnect => 1,
   mysql_enable_utf8mb4 => 1
 });
@@ -75,7 +75,8 @@ my $sth = $dbh->prepare("SELECT id,
   custom_params,
   subscribeall,
   timeout1,
-  timeout2
+  timeout2,
+  dry
     FROM imapsync
       WHERE active = 1
         AND is_running = 0
@@ -111,13 +112,16 @@ while ($row = $sth->fetchrow_arrayref()) {
   $subscribeall        = @$row[18];
   $timeout1            = @$row[19];
   $timeout2            = @$row[20];
+  $dry                 = @$row[21];
 
   if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; }
 
   my $template = $run_dir . '/imapsync.XXXXXXX';
   my $passfile1 = File::Temp->new(TEMPLATE => $template);
   my $passfile2 = File::Temp->new(TEMPLATE => $template);
-
+  
+  binmode( $passfile1, ":utf8" );
+  
   print $passfile1 "$password1\n";
   print $passfile2 trim($master_pass) . "\n";
 
@@ -127,6 +131,7 @@ while ($row = $sth->fetchrow_arrayref()) {
   my $generated_cmds = [ "/usr/local/bin/imapsync",
   "--tmpdir", "/tmp",
   "--nofoldersizes",
+  "--addheader",
   ($timeout1 gt "0" ? () : ('--timeout1', $timeout1)),
   ($timeout2 gt "0" ? () : ('--timeout2', $timeout2)),
   ($exclude eq "" ? () : ("--exclude", $exclude)),
@@ -147,22 +152,33 @@ while ($row = $sth->fetchrow_arrayref()) {
   "--host2", "localhost",
   "--user2", $user2 . '*' . trim($master_user),
   "--passfile2", $passfile2->filename,
+  ($dry eq "1" ? ('--dry') : ()),
   '--no-modulesversion',
   '--noreleasecheck'];
 
   try {
-    $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1 WHERE id = ?");
+    $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
     $is_running->bind_param( 1, ${id} );
     $is_running->execute();
-    
+
     run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout;
-    
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = ? WHERE id = ?");
+
+    # check exit code and status
+    ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
+
+    $success = 0;
+    if (defined $exit_code && $exit_code == 0) {
+      $success = 1;
+    }
+
+    $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
     $update->bind_param( 1, ${stdout} );
-    $update->bind_param( 2, ${id} );
+    $update->bind_param( 2, ${success} );
+    $update->bind_param( 3, ${exit_status} );
+    $update->bind_param( 4, ${id} );
     $update->execute();
   } catch {
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync' WHERE id = ?");
+    $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
     $update->bind_param( 1, ${id} );
     $update->execute();
   } finally {

data/Dockerfiles/dovecot/maildir_gc.sh

@@ -1,2 +1,2 @@
-#/bin/bash
+#!/bin/bash
 [ -d /var/vmail/_garbage/ ] && /usr/bin/find /var/vmail/_garbage/ -mindepth 1 -maxdepth 1 -type d -cmin +${MAILDIR_GC_TIME} -exec rm -r {} \;

data/Dockerfiles/dovecot/postlogin.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-export MASTER_USER=$USER
-exec "$@"

data/Dockerfiles/dovecot/quarantine_notify.py

@@ -2,11 +2,11 @@
 
 import smtplib
 import os
-import mysql.connector
+import sys
+import MySQLdb
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import COMMASPACE, formatdate
-import cgi
 import jinja2
 from jinja2 import Template
 import json
@@ -15,132 +15,154 @@ import time
 import html2text
 import socket
 
-while True:
-  try:
-    r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0)
-    r.ping()
-  except Exception as ex:
-    print('%s - trying again...'  % (ex))
-    time.sleep(3)
-  else:
-    break
+pid = str(os.getpid())
+pidfile = "/tmp/quarantine_notify.pid"
 
-time_now = int(time.time())
+if os.path.isfile(pidfile):
+  print("%s already exists, exiting" % (pidfile))
+  sys.exit()
+
+pid = str(os.getpid())
+f = open(pidfile, 'w')
+f.write(pid)
+f.close()
+
+try:
 
-def query_mysql(query, headers = True, update = False):
   while True:
     try:
-      cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user='__DBUSER__', passwd='__DBPASS__', database='__DBNAME__', charset="utf8")
+      r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0)
+      r.ping()
     except Exception as ex:
       print('%s - trying again...'  % (ex))
       time.sleep(3)
     else:
       break
-  cur = cnx.cursor()
-  cur.execute(query)
-  if not update:
-    result = []
-    columns = tuple( [d[0] for d in cur.description] )
-    for row in cur:
-      if headers:
-        result.append(dict(list(zip(columns, row))))
-      else:
-        result.append(row)
-    cur.close()
-    cnx.close()
-    return result
-  else:
-    cnx.commit()
-    cur.close()
-    cnx.close()
 
-def notify_rcpt(rcpt, msg_count, quarantine_acl):
-  meta_query = query_mysql('SELECT SHA2(CONCAT(id, qid), 256) AS qhash, id, subject, score, sender, created FROM quarantine WHERE notified = 0 AND rcpt = "%s"' % (rcpt))
-  if r.get('Q_HTML'):
-    try:
-      template = Template(r.get('Q_HTML'))
-    except:
-      print("Error: Cannot parse quarantine template, falling back to default template.")
+  time_now = int(time.time())
+  mailcow_hostname = os.environ.get('MAILCOW_HOSTNAME')
+
+  max_score = float(r.get('Q_MAX_SCORE') or "9999.0")
+  if max_score == "":
+    max_score = 9999.0
+
+  def query_mysql(query, headers = True, update = False):
+    while True:
+      try:
+        cnx = MySQLdb.connect(user=os.environ.get('DBUSER'), password=os.environ.get('DBPASS'), database=os.environ.get('DBNAME'), charset="utf8mb4", collation="utf8mb4_general_ci")
+      except Exception as ex:
+        print('%s - trying again...'  % (ex))
+        time.sleep(3)
+      else:
+        break
+    cur = cnx.cursor()
+    cur.execute(query)
+    if not update:
+      result = []
+      columns = tuple( [d[0] for d in cur.description] )
+      for row in cur:
+        if headers:
+          result.append(dict(list(zip(columns, row))))
+        else:
+          result.append(row)
+      cur.close()
+      cnx.close()
+      return result
+    else:
+      cnx.commit()
+      cur.close()
+      cnx.close()
+
+  def notify_rcpt(rcpt, msg_count, quarantine_acl, category):
+    if category == "add_header": category = "add header"
+    meta_query = query_mysql('SELECT SHA2(CONCAT(id, qid), 256) AS qhash, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category))
+    print("%s: %d of %d messages qualify for notification" % (rcpt, len(meta_query), msg_count))
+    if len(meta_query) == 0:
+      return
+    msg_count = len(meta_query)
+    if r.get('Q_HTML'):
+      try:
+        template = Template(r.get('Q_HTML'))
+      except:
+        print("Error: Cannot parse quarantine template, falling back to default template.")
+        with open('/templates/quarantine.tpl') as file_:
+          template = Template(file_.read())
+    else:
       with open('/templates/quarantine.tpl') as file_:
         template = Template(file_.read())
-  else:
-    with open('/templates/quarantine.tpl') as file_:
-      template = Template(file_.read())
-  html = template.render(meta=meta_query, username=rcpt, counter=msg_count, hostname=socket.gethostname(), quarantine_acl=quarantine_acl)
-  text = html2text.html2text(html)
-  count = 0
-  while count < 15:
-    count += 1
+    html = template.render(meta=meta_query, username=rcpt, counter=msg_count, hostname=mailcow_hostname, quarantine_acl=quarantine_acl)
+    text = html2text.html2text(html)
+    count = 0
+    while count < 15:
+      count += 1
+      try:
+        server = smtplib.SMTP('postfix', 590, 'quarantine')
+        server.ehlo()
+        msg = MIMEMultipart('alternative')
+        msg_from = r.get('Q_SENDER') or "quarantine@localhost"
+        # Remove non-ascii chars from field
+        msg['From'] = ''.join([i if ord(i) < 128 else '' for i in msg_from])
+        msg['Subject'] = r.get('Q_SUBJ') or "Spam Quarantine Notification"
+        msg['Date'] = formatdate(localtime = True)
+        text_part = MIMEText(text, 'plain', 'utf-8')
+        html_part = MIMEText(html, 'html', 'utf-8')
+        msg.attach(text_part)
+        msg.attach(html_part)
+        msg['To'] = str(rcpt)
+        bcc = r.get('Q_BCC') or ""
+        redirect = r.get('Q_REDIRECT') or ""
+        text = msg.as_string()
+        if bcc == '':
+          if redirect == '':
+            server.sendmail(msg['From'], str(rcpt), text)
+          else:
+            server.sendmail(msg['From'], str(redirect), text)
+        else:
+          if redirect == '':
+            server.sendmail(msg['From'], [str(rcpt)] + [str(bcc)], text)
+          else:
+            server.sendmail(msg['From'], [str(redirect)] + [str(bcc)], text)
+        server.quit()
+        for res in meta_query:
+         query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True)
+        r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now)
+        break
+      except Exception as ex:
+        server.quit()
+        print('%s'  % (ex))
+        time.sleep(3)
+
+  records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND score < %f AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt' % (max_score))
+
+  for record in records:
+    attrs = ''
+    attrs_json = ''
+    time_trans = {
+      "hourly": 3600,
+      "daily": 86400,
+      "weekly": 604800
+    }
     try:
-      server = smtplib.SMTP('postfix', 590, 'quarantine')
-      server.ehlo()
-      msg = MIMEMultipart('alternative')
-      msg_from = r.get('Q_SENDER') or "quarantine@localhost"
-      # Remove non-ascii chars from field
-      msg['From'] = ''.join([i if ord(i) < 128 else '' for i in msg_from])
-      msg['Subject'] = r.get('Q_SUBJ') or "Spam Quarantine Notification"
-      msg['Date'] = formatdate(localtime = True)
-      text_part = MIMEText(text, 'plain', 'utf-8')
-      html_part = MIMEText(html, 'html', 'utf-8')
-      msg.attach(text_part)
-      msg.attach(html_part)
-      msg['To'] = str(rcpt)
-      bcc = r.get('Q_BCC') or ""
-      redirect = r.get('Q_REDIRECT') or ""
-      text = msg.as_string()
-      if bcc == '':
-        if redirect == '':
-          server.sendmail(msg['From'], str(rcpt), text)
-        else:
-          server.sendmail(msg['From'], str(redirect), text)
-      else:
-        if redirect == '':
-          server.sendmail(msg['From'], [str(rcpt)] + [str(bcc)], text)
-        else:
-          server.sendmail(msg['From'], [str(redirect)] + [str(bcc)], text)
-      server.quit()
-      for res in meta_query:
-        query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True)
-      r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now)
-      break
+      last_notification = int(r.hget('Q_LAST_NOTIFIED', record['rcpt']))
+      if last_notification > time_now:
+        print('Last notification is > time now, assuming never')
+        last_notification = 0
     except Exception as ex:
-      server.quit()
-      print('%s'  % (ex))
-      time.sleep(3)
-
-records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt')
-
-for record in records:
-  attrs = ''
-  attrs_json = ''
-  try:
-    last_notification = int(r.hget('Q_LAST_NOTIFIED', record['rcpt']))
-    if last_notification > time_now:
-      print('Last notification is > time now, assuming never')
+      print('Could not determine last notification for %s, assuming never' % (record['rcpt']))
       last_notification = 0
-  except Exception as ex:
-    print('Could not determine last notification for %s, assuming never' % (record['rcpt']))
-    last_notification = 0
-  attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt']))
-  attrs = attrs_json[0]['attributes']
-  if isinstance(attrs, str):
-    # if attr is str then just load it
-    attrs = json.loads(attrs)
-  else:
-    # if it's bytes then decode and load it
-    attrs = json.loads(attrs.decode('utf-8'))
-  if attrs['quarantine_notification'] not in ('hourly', 'daily', 'weekly', 'never'):
-    print('Abnormal quarantine_notification value')
-    continue
-  if attrs['quarantine_notification'] == 'hourly':
-    if last_notification == 0 or (last_notification + 3600) < time_now:
-      print("Notifying %s about %d new items in quarantine" % (record['rcpt'], record['counter']))
-      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'])
-  elif attrs['quarantine_notification'] == 'daily':
-    if last_notification == 0 or (last_notification + 86400) < time_now:
-      print("Notifying %s about %d new items in quarantine" % (record['rcpt'], record['counter']))
-      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'])
-  elif attrs['quarantine_notification'] == 'weekly':
-    if last_notification == 0 or (last_notification + 604800) < time_now:
-      print("Notifying %s about %d new items in quarantine" % (record['rcpt'], record['counter']))
-      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'])
+    attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt']))
+    attrs = attrs_json[0]['attributes']
+    if isinstance(attrs, str):
+      # if attr is str then just load it
+      attrs = json.loads(attrs)
+    else:
+      # if it's bytes then decode and load it
+      attrs = json.loads(attrs.decode('utf-8'))
+    if attrs['quarantine_notification'] not in ('hourly', 'daily', 'weekly'):
+      continue
+    if last_notification == 0 or (last_notification + time_trans[attrs['quarantine_notification']]) <= time_now:
+      print("Notifying %s: Considering %d new items in quarantine (policy: %s)" % (record['rcpt'], record['counter'], attrs['quarantine_notification']))
+      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'], attrs['quarantine_category'])
+
+finally:
+  os.unlink(pidfile)

data/Dockerfiles/dovecot/quota_notify.py

@@ -9,6 +9,7 @@ import jinja2
 from jinja2 import Template
 import redis
 import time
+import json
 import sys
 import html2text
 from subprocess import Popen, PIPE, STDOUT
@@ -54,9 +55,30 @@ try:
   msg.attach(text_part)
   msg.attach(html_part)
   msg['To'] = username
-  p = Popen(['/usr/lib/dovecot/dovecot-lda', '-d', username, '-o', '"plugin/quota=maildir:User quota:noenforcing"'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
+  p = Popen(['/usr/libexec/dovecot/dovecot-lda', '-d', username, '-o', '"plugin/quota=maildir:User quota:noenforcing"'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
   p.communicate(input=bytes(msg.as_string(), 'utf-8'))
 
+  domain = username.split("@")[-1]
+  if domain and r.hget('QW_BCC', domain):
+    bcc_data = json.loads(r.hget('QW_BCC', domain))
+    bcc_rcpts = bcc_data['bcc_rcpts']
+    if bcc_data['active'] == 1:
+      for rcpt in bcc_rcpts:
+        msg = MIMEMultipart('alternative')
+        msg['From'] = username
+        subject = r.get('QW_SUBJ') or "Quota warning"
+        msg['Subject'] = subject + ' (' + username + ')'
+        msg['Date'] = formatdate(localtime = True)
+        text_part = MIMEText(text, 'plain', 'utf-8')
+        html_part = MIMEText(html, 'html', 'utf-8')
+        msg.attach(text_part)
+        msg.attach(html_part)
+        msg['To'] = rcpt
+        server = smtplib.SMTP('postfix', 588, 'quotanotification')
+        server.ehlo()
+        server.sendmail(msg['From'], str(rcpt), msg.as_string())
+        server.quit()
+
 except Exception as ex:
   print('Failed to send quota notification: %s' % (ex))
   sys.exit(1)

data/Dockerfiles/dovecot/repl_health.sh

@@ -11,7 +11,7 @@ fi
 
 # Is replication active?
 # grep on file is less expensive than doveconf
-if ! grep -qi mail_replica /etc/dovecot/dovecot.conf; then
+if [ -n ${MAILCOW_REPLICA_IP} ]; then
   ${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
   exit
 fi

data/Dockerfiles/dovecot/sa-rules.sh

@@ -2,7 +2,6 @@
 
 # Create temp directories
 [[ ! -d /tmp/sa-rules-heinlein ]] && mkdir -p /tmp/sa-rules-heinlein
-#[[ ! -d /tmp/sa-rules-schaal ]] && mkdir -p /tmp/sa-rules-schaal
 
 # Hash current SA rules
 if [[ ! -f /etc/rspamd/custom/sa-rules ]]; then
@@ -12,19 +11,11 @@ else
 fi
 
 # Deploy
-## Heinlein
-curl --connect-timeout 15 --retry 10 --max-time 30 http://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"').tar.gz --output /tmp/sa-rules-heinlein.tar.gz
+curl --connect-timeout 15 --retry 10 --max-time 30 http://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"' | tr -dc '0-9').tar.gz --output /tmp/sa-rules-heinlein.tar.gz
 if gzip -t /tmp/sa-rules-heinlein.tar.gz; then
   tar xfvz /tmp/sa-rules-heinlein.tar.gz -C /tmp/sa-rules-heinlein
   cat /tmp/sa-rules-heinlein/*cf > /etc/rspamd/custom/sa-rules
 fi
-## Schaal
-#curl --connect-timeout 15 --max-time 30 http://sa.schaal-it.net/$(dig txt 1.4.3.sa.schaal-it.net +short | tr -d '"').tar.gz --output /tmp/sa-rules-schaal.tar.gz
-#if gzip -t /tmp/sa-rules-schaal.tar.gz; then
-#  tar xfvz /tmp/sa-rules-schaal.tar.gz -C /tmp/sa-rules-schaal
-#  # Append, do not overwrite
-#  cat /tmp/sa-rules-schaal/*cf >> /etc/rspamd/custom/sa-rules
-#fi
 
 sed -i -e 's/\([^\\]\)\$\([^\/]\)/\1\\$\2/g' /etc/rspamd/custom/sa-rules
 
@@ -40,4 +31,3 @@ fi
 
 # Cleanup
 rm -rf /tmp/sa-rules-heinlein /tmp/sa-rules-heinlein.tar.gz
-#rm -rf /tmp/sa-rules-schaal /tmp/sa-rules-schaal.tar.gz

data/Dockerfiles/dovecot/supervisord.conf

@@ -13,10 +13,10 @@ autostart=true
 
 [program:dovecot]
 command=/usr/sbin/dovecot -F
-autorestart=true
-
-[program:cron]
-command=/usr/sbin/cron -f
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
 autorestart=true
 
 [eventlistener:processes]

data/Dockerfiles/dovecot/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.19
+@version: 4.5
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -6,11 +6,11 @@ options {
   use_dns(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
-source s_src {
-  unix-stream("/dev/log");
+source s_dgram {
+  unix-dgram("/dev/log");
   internal();
 };
 destination d_stdout { pipe("/dev/stdout"); };
@@ -27,7 +27,7 @@ destination d_redis_f2b_channel {
     host("`REDIS_SLAVEOF_IP`")
     persist-name("redis2")
     port(`REDIS_SLAVEOF_PORT`)
-    command("PUBLISH" "F2B_CHANNEL" "$MESSAGE")
+    command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
   );
 };
 filter f_mail { facility(mail); };
@@ -36,7 +36,7 @@ filter f_replica {
   not match("Error: sync: Unknown user in remote" value("MESSAGE"));
 };
 log {
-  source(s_src);
+  source(s_dgram);
   filter(f_replica);
   destination(d_stdout);
   filter(f_mail);

data/Dockerfiles/dovecot/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.19
+@version: 4.5
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -6,11 +6,11 @@ options {
   use_dns(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
-source s_src {
-  unix-stream("/dev/log");
+source s_dgram {
+  unix-dgram("/dev/log");
   internal();
 };
 destination d_stdout { pipe("/dev/stdout"); };
@@ -27,7 +27,7 @@ destination d_redis_f2b_channel {
     host("redis-mailcow")
     persist-name("redis2")
     port(6379)
-    command("PUBLISH" "F2B_CHANNEL" "$MESSAGE")
+    command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
   );
 };
 filter f_mail { facility(mail); };
@@ -36,7 +36,7 @@ filter f_replica {
   not match("Error: sync: Unknown user in remote" value("MESSAGE"));
 };
 log {
-  source(s_src);
+  source(s_dgram);
   filter(f_replica);
   destination(d_stdout);
   filter(f_mail);

data/Dockerfiles/dovecot/trim_logs.sh

@@ -14,12 +14,13 @@ if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
 else
   REDIS_CMDLINE="redis-cli -h redis -p 6379"
 fi
-catch_non_zero "${REDIS_CMDLINE} LTRIM ACME_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM POSTFIX_MAILLOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM DOVECOT_MAILLOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM SOGO_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM NETFILTER_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM AUTODISCOVER_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM API_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM RL_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM WATCHDOG_LOG 0 __LOG_LINES__"
+catch_non_zero "${REDIS_CMDLINE} LTRIM ACME_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM POSTFIX_MAILLOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM DOVECOT_MAILLOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM SOGO_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM NETFILTER_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM AUTODISCOVER_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM API_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM RL_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM WATCHDOG_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM CRON_LOG 0 ${LOG_LINES}"

data/Dockerfiles/netfilter/Dockerfile

@@ -1,15 +1,42 @@
-FROM alpine:3.11
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM alpine:3.19
+LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
+WORKDIR /app
+
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
 ENV XTABLES_LIBDIR /usr/lib/xtables
 ENV PYTHON_IPTABLES_XTABLES_VERSION 12
 ENV IPTABLES_LIBDIR /usr/lib
 
-RUN apk add --virtual .build-deps gcc python3-dev libffi-dev openssl-dev \
-  && apk add -U python3 iptables ip6tables tzdata musl-dev \
-  && pip3 install --upgrade pip python-iptables redis ipaddress dnspython \
-#  && pip3 install --upgrade pip python-iptables==0.13.0 redis ipaddress dnspython \
-  && apk del .build-deps
+RUN apk add --virtual .build-deps \
+  gcc \
+  python3-dev \
+  libffi-dev \
+  openssl-dev \
+&& apk add -U python3 \
+  iptables \
+  iptables-dev \
+  ip6tables \
+  xtables-addons \
+  nftables \
+  tzdata \
+  py3-pip \
+  py3-nftables \
+  musl-dev \
+&& pip3 install --ignore-installed --upgrade pip \
+  jsonschema \
+  python-iptables \
+  redis \
+  ipaddress \
+  dnspython \
+&& apk del .build-deps
 
-COPY server.py /
-CMD ["python3", "-u", "/server.py"]
+#  && pip3 install --upgrade pip python-iptables==0.13.0 redis ipaddress dnspython \
+
+COPY modules /app/modules
+COPY main.py /app/
+COPY ./docker-entrypoint.sh /app/
+
+RUN chmod +x /app/docker-entrypoint.sh
+
+CMD ["/bin/sh", "-c", "/app/docker-entrypoint.sh"]

data/Dockerfiles/netfilter/docker-entrypoint.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+backend=iptables
+
+nft list table ip filter &>/dev/null
+nftables_found=$?
+
+iptables -L -n &>/dev/null
+iptables_found=$?
+
+if [ $nftables_found -lt $iptables_found ]; then
+  backend=nftables
+fi
+
+if [ $nftables_found -gt $iptables_found ]; then
+  backend=iptables
+fi
+
+if [ $nftables_found -eq 0 ] && [ $nftables_found -eq $iptables_found ]; then
+  nftables_lines=$(nft list ruleset | wc -l)
+  iptables_lines=$(iptables-save | wc -l)
+  if [ $nftables_lines -gt $iptables_lines ]; then
+    backend=nftables
+  else
+    backend=iptables
+  fi
+fi
+
+exec python -u /app/main.py $backend

data/Dockerfiles/netfilter/main.py

@@ -0,0 +1,496 @@
+#!/usr/bin/env python3
+
+import re
+import os
+import sys
+import time
+import atexit
+import signal
+import ipaddress
+from collections import Counter
+from random import randint
+from threading import Thread
+from threading import Lock
+import redis
+import json
+import dns.resolver
+import dns.exception
+import uuid
+from modules.Logger import Logger
+from modules.IPTables import IPTables
+from modules.NFTables import NFTables
+
+
+# globals
+WHITELIST = []
+BLACKLIST= []
+bans = {}
+quit_now = False
+exit_code = 0
+lock = Lock()
+chain_name = "MAILCOW"
+r = None
+pubsub = None
+clear_before_quit = False
+
+
+def refreshF2boptions():
+  global f2boptions
+  global quit_now
+  global exit_code
+
+  f2boptions = {}
+
+  if not r.get('F2B_OPTIONS'):
+    f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
+    f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
+    f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
+    f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
+    f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
+    f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
+    f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
+  else:
+    try:
+      f2boptions = json.loads(r.get('F2B_OPTIONS'))
+    except ValueError:
+      logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json')
+      quit_now = True
+      exit_code = 2
+
+  verifyF2boptions(f2boptions)
+  r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
+
+def verifyF2boptions(f2boptions):
+  verifyF2boption(f2boptions,'ban_time', 1800)
+  verifyF2boption(f2boptions,'max_ban_time', 10000)
+  verifyF2boption(f2boptions,'ban_time_increment', True)
+  verifyF2boption(f2boptions,'max_attempts', 10)
+  verifyF2boption(f2boptions,'retry_window', 600)
+  verifyF2boption(f2boptions,'netban_ipv4', 32)
+  verifyF2boption(f2boptions,'netban_ipv6', 128)
+  verifyF2boption(f2boptions,'banlist_id', str(uuid.uuid4()))
+  verifyF2boption(f2boptions,'manage_external', 0)
+
+def verifyF2boption(f2boptions, f2boption, f2bdefault):
+  f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
+
+def refreshF2bregex():
+  global f2bregex
+  global quit_now
+  global exit_code
+  if not r.get('F2B_REGEX'):
+    f2bregex = {}
+    f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
+    f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
+    f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
+    f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
+    f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
+    f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
+    f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
+    f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
+    f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
+    f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
+    r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
+  else:
+    try:
+      f2bregex = {}
+      f2bregex = json.loads(r.get('F2B_REGEX'))
+    except ValueError:
+      logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
+      quit_now = True
+      exit_code = 2
+
+def get_ip(address):
+  ip = ipaddress.ip_address(address)
+  if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
+    ip = ip.ipv4_mapped
+  if ip.is_private or ip.is_loopback:
+    return False
+  
+  return ip
+
+def ban(address):
+  global f2boptions
+  global lock
+
+  refreshF2boptions()
+  BAN_TIME = int(f2boptions['ban_time'])
+  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
+  MAX_ATTEMPTS = int(f2boptions['max_attempts'])
+  RETRY_WINDOW = int(f2boptions['retry_window'])
+  NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
+  NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
+
+  ip = get_ip(address)
+  if not ip: return
+  address = str(ip)
+  self_network = ipaddress.ip_network(address)
+
+  with lock:
+    temp_whitelist = set(WHITELIST)
+  if temp_whitelist:
+    for wl_key in temp_whitelist:
+      wl_net = ipaddress.ip_network(wl_key, False)
+      if wl_net.overlaps(self_network):
+        logger.logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net))
+        return
+
+  net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
+  net = str(net)
+
+  if not net in bans:
+    bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
+
+  current_attempt = time.time()
+  if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
+    bans[net]['attempts'] = 0
+
+  bans[net]['attempts'] += 1
+  bans[net]['last_attempt'] = current_attempt
+
+  if bans[net]['attempts'] >= MAX_ATTEMPTS:
+    cur_time = int(round(time.time()))
+    NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
+    logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
+    if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
+      with lock:
+        tables.banIPv4(net)
+    elif int(f2boptions['manage_external']) != 1:
+      with lock:
+        tables.banIPv6(net)
+
+    r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
+  else:
+    logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
+
+def unban(net):
+  global lock
+
+  if not net in bans:
+   logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net)
+   r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
+   return
+
+  logger.logInfo('Unbanning %s' % net)
+  if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
+    with lock:
+      tables.unbanIPv4(net)
+  else:
+    with lock:
+      tables.unbanIPv6(net)
+
+  r.hdel('F2B_ACTIVE_BANS', '%s' % net)
+  r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
+  if net in bans:
+    bans[net]['attempts'] = 0
+    bans[net]['ban_counter'] += 1
+
+def permBan(net, unban=False):
+  global f2boptions
+  global lock
+
+  is_unbanned = False
+  is_banned = False
+  if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
+    with lock:
+      if unban:
+        is_unbanned = tables.unbanIPv4(net)
+      elif int(f2boptions['manage_external']) != 1:
+        is_banned = tables.banIPv4(net)
+  else:
+    with lock:
+      if unban:
+        is_unbanned = tables.unbanIPv6(net)
+      elif int(f2boptions['manage_external']) != 1:
+        is_banned = tables.banIPv6(net)
+
+
+  if is_unbanned:
+    r.hdel('F2B_PERM_BANS', '%s' % net)
+    logger.logCrit('Removed host/network %s from blacklist' % net)
+  elif is_banned:
+    r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
+    logger.logCrit('Added host/network %s to blacklist' % net)
+
+def clear():
+  global lock
+  logger.logInfo('Clearing all bans')
+  for net in bans.copy():
+    unban(net)
+  with lock:
+    tables.clearIPv4Table()
+    tables.clearIPv6Table()
+    try:
+      if r is not None:
+        r.delete('F2B_ACTIVE_BANS')
+        r.delete('F2B_PERM_BANS')
+    except Exception as ex:
+      logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
+
+def watch():
+  global pubsub
+  global quit_now
+  global exit_code
+
+  logger.logInfo('Watching Redis channel F2B_CHANNEL')
+  pubsub.subscribe('F2B_CHANNEL')
+
+  while not quit_now:
+    try:
+      for item in pubsub.listen():
+        refreshF2bregex()
+        for rule_id, rule_regex in f2bregex.items():
+          if item['data'] and item['type'] == 'message':
+            try:
+              result = re.search(rule_regex, item['data'])
+            except re.error:
+              result = False
+            if result:
+              addr = result.group(1)
+              ip = ipaddress.ip_address(addr)
+              if ip.is_private or ip.is_loopback:
+                continue
+              logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
+              ban(addr)
+    except Exception as ex:
+      logger.logWarn('Error reading log line from pubsub: %s' % ex)
+      pubsub = None
+      quit_now = True
+      exit_code = 2
+
+def snat4(snat_target):
+  global lock
+  global quit_now
+
+  while not quit_now:
+    time.sleep(10)
+    with lock:
+      tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
+
+def snat6(snat_target):
+  global lock
+  global quit_now
+
+  while not quit_now:
+    time.sleep(10)
+    with lock:
+      tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
+
+def autopurge():
+  while not quit_now:
+    time.sleep(10)
+    refreshF2boptions()
+    BAN_TIME = int(f2boptions['ban_time'])
+    MAX_BAN_TIME = int(f2boptions['max_ban_time'])
+    BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
+    MAX_ATTEMPTS = int(f2boptions['max_attempts'])
+    QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
+    if QUEUE_UNBAN:
+      for net in QUEUE_UNBAN:
+        unban(str(net))
+    for net in bans.copy():
+      if bans[net]['attempts'] >= MAX_ATTEMPTS:
+        NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
+        TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt']
+        if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME:
+          unban(net)
+
+def mailcowChainOrder():
+  global lock
+  global quit_now
+  global exit_code
+  while not quit_now:
+    time.sleep(10)
+    with lock:
+      quit_now, exit_code = tables.checkIPv4ChainOrder()
+      if quit_now: return
+      quit_now, exit_code = tables.checkIPv6ChainOrder()
+
+def isIpNetwork(address):
+  try:
+    ipaddress.ip_network(address, False)
+  except ValueError:
+    return False
+  return True
+
+def genNetworkList(list):
+  resolver = dns.resolver.Resolver()
+  hostnames = []
+  networks = []
+  for key in list:
+    if isIpNetwork(key):
+      networks.append(key)
+    else:
+      hostnames.append(key)
+  for hostname in hostnames:
+    hostname_ips = []
+    for rdtype in ['A', 'AAAA']:
+      try:
+        answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
+      except dns.exception.Timeout:
+        logger.logInfo('Hostname %s timedout on resolve' % hostname)
+        break
+      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+        continue
+      except dns.exception.DNSException as dnsexception:
+        logger.logInfo('%s' % dnsexception)
+        continue
+      for rdata in answer:
+        hostname_ips.append(rdata.to_text())
+    networks.extend(hostname_ips)
+  return set(networks)
+
+def whitelistUpdate():
+  global lock
+  global quit_now
+  global WHITELIST
+  while not quit_now:
+    start_time = time.time()
+    list = r.hgetall('F2B_WHITELIST')
+    new_whitelist = []
+    if list:
+      new_whitelist = genNetworkList(list)
+    with lock:
+      if Counter(new_whitelist) != Counter(WHITELIST):
+        WHITELIST = new_whitelist
+        logger.logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST))
+    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
+
+def blacklistUpdate():
+  global quit_now
+  global BLACKLIST
+  while not quit_now:
+    start_time = time.time()
+    list = r.hgetall('F2B_BLACKLIST')
+    new_blacklist = []
+    if list:
+      new_blacklist = genNetworkList(list)
+    if Counter(new_blacklist) != Counter(BLACKLIST):
+      addban = set(new_blacklist).difference(BLACKLIST)
+      delban = set(BLACKLIST).difference(new_blacklist)
+      BLACKLIST = new_blacklist
+      logger.logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST))
+      if addban:
+        for net in addban:
+          permBan(net=net)
+      if delban:
+        for net in delban:
+          permBan(net=net, unban=True)
+    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
+
+def sigterm_quit(signum, frame):
+  global clear_before_quit
+  clear_before_quit = True
+  sys.exit(exit_code)
+
+def berfore_quit():
+  if clear_before_quit:
+    clear()
+  if pubsub is not None:
+    pubsub.unsubscribe()
+
+
+if __name__ == '__main__':
+  atexit.register(berfore_quit)
+  signal.signal(signal.SIGTERM, sigterm_quit)
+
+  # init Logger
+  logger = Logger()
+
+  # init backend
+  backend = sys.argv[1]
+  if backend == "nftables":
+    logger.logInfo('Using NFTables backend')
+    tables = NFTables(chain_name, logger)
+  else:
+    logger.logInfo('Using IPTables backend')
+    tables = IPTables(chain_name, logger)
+
+  # In case a previous session was killed without cleanup
+  clear()
+
+  # Reinit MAILCOW chain
+  # Is called before threads start, no locking
+  logger.logInfo("Initializing mailcow netfilter chain")
+  tables.initChainIPv4()
+  tables.initChainIPv6()
+
+  if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE").lower() in ("y", "yes"):
+    logger.logInfo(f"Skipping {chain_name} isolation")
+  else:
+    logger.logInfo(f"Setting {chain_name} isolation")
+    tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
+
+  # connect to redis
+  while True:
+    try:
+      redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
+      redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
+      if "".__eq__(redis_slaveof_ip):
+        r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
+      else:
+        r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
+      r.ping()
+      pubsub = r.pubsub()
+    except Exception as ex:
+      print('%s - trying again in 3 seconds'  % (ex))
+      time.sleep(3)
+    else:
+      break
+  logger.set_redis(r)
+
+  # rename fail2ban to netfilter
+  if r.exists('F2B_LOG'):
+    r.rename('F2B_LOG', 'NETFILTER_LOG')
+  # clear bans in redis
+  r.delete('F2B_ACTIVE_BANS')
+  r.delete('F2B_PERM_BANS')
+  
+  refreshF2boptions()
+
+  watch_thread = Thread(target=watch)
+  watch_thread.daemon = True
+  watch_thread.start()
+
+  if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
+    try:
+      snat_ip = os.getenv('SNAT_TO_SOURCE')
+      snat_ipo = ipaddress.ip_address(snat_ip)
+      if type(snat_ipo) is ipaddress.IPv4Address:
+        snat4_thread = Thread(target=snat4,args=(snat_ip,))
+        snat4_thread.daemon = True
+        snat4_thread.start()
+    except ValueError:
+      print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
+
+  if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
+    try:
+      snat_ip = os.getenv('SNAT6_TO_SOURCE')
+      snat_ipo = ipaddress.ip_address(snat_ip)
+      if type(snat_ipo) is ipaddress.IPv6Address:
+        snat6_thread = Thread(target=snat6,args=(snat_ip,))
+        snat6_thread.daemon = True
+        snat6_thread.start()
+    except ValueError:
+      print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
+
+  autopurge_thread = Thread(target=autopurge)
+  autopurge_thread.daemon = True
+  autopurge_thread.start()
+
+  mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
+  mailcowchainwatch_thread.daemon = True
+  mailcowchainwatch_thread.start()
+
+  blacklistupdate_thread = Thread(target=blacklistUpdate)
+  blacklistupdate_thread.daemon = True
+  blacklistupdate_thread.start()
+
+  whitelistupdate_thread = Thread(target=whitelistUpdate)
+  whitelistupdate_thread.daemon = True
+  whitelistupdate_thread.start()
+
+  while not quit_now:
+    time.sleep(0.5)
+
+  sys.exit(exit_code)

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -0,0 +1,252 @@
+import iptc
+import time
+import os
+
+class IPTables:
+  def __init__(self, chain_name, logger):
+    self.chain_name = chain_name
+    self.logger = logger
+
+  def initChainIPv4(self):
+    if not iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name) in iptc.Table(iptc.Table.FILTER).chains:
+      iptc.Table(iptc.Table.FILTER).create_chain(self.chain_name)
+    for c in ['FORWARD', 'INPUT']:
+      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), c)
+      rule = iptc.Rule()
+      rule.src = '0.0.0.0/0'
+      rule.dst = '0.0.0.0/0'
+      target = iptc.Target(rule, self.chain_name)
+      rule.target = target
+      if rule not in chain.rules:
+        chain.insert_rule(rule)
+
+  def initChainIPv6(self):
+    if not iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name) in iptc.Table6(iptc.Table6.FILTER).chains:
+      iptc.Table6(iptc.Table6.FILTER).create_chain(self.chain_name)
+    for c in ['FORWARD', 'INPUT']:
+      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), c)
+      rule = iptc.Rule6()
+      rule.src = '::/0'
+      rule.dst = '::/0'
+      target = iptc.Target(rule, self.chain_name)
+      rule.target = target
+      if rule not in chain.rules:
+        chain.insert_rule(rule)
+
+  def checkIPv4ChainOrder(self):
+    filter_table = iptc.Table(iptc.Table.FILTER)
+    filter_table.refresh()
+    return self.checkChainOrder(filter_table)
+
+  def checkIPv6ChainOrder(self):
+    filter_table = iptc.Table6(iptc.Table6.FILTER)
+    filter_table.refresh()
+    return self.checkChainOrder(filter_table)
+
+  def checkChainOrder(self, filter_table):
+    err = False
+    exit_code = None
+
+    forward_chain = iptc.Chain(filter_table, 'FORWARD')
+    input_chain = iptc.Chain(filter_table, 'INPUT')
+    for chain in [forward_chain, input_chain]:
+      target_found = False
+      for position, item in enumerate(chain.rules):
+        if item.target.name == self.chain_name:
+          target_found = True
+          if position > 2:
+            self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
+            err = True
+            exit_code = 2
+      if not target_found:
+        self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
+        err = True
+        exit_code = 2
+
+    return err, exit_code
+
+  def clearIPv4Table(self):
+    self.clearTable(iptc.Table(iptc.Table.FILTER))
+
+  def clearIPv6Table(self):
+    self.clearTable(iptc.Table6(iptc.Table6.FILTER))
+
+  def clearTable(self, filter_table):
+    filter_table.autocommit = False
+    forward_chain = iptc.Chain(filter_table, "FORWARD")
+    input_chain = iptc.Chain(filter_table, "INPUT")
+    mailcow_chain = iptc.Chain(filter_table, self.chain_name)
+    if mailcow_chain in filter_table.chains:
+      for rule in mailcow_chain.rules:
+        mailcow_chain.delete_rule(rule)
+      for rule in forward_chain.rules:
+        if rule.target.name == self.chain_name:
+          forward_chain.delete_rule(rule)
+      for rule in input_chain.rules:
+        if rule.target.name == self.chain_name:
+          input_chain.delete_rule(rule)
+      filter_table.delete_chain(self.chain_name)
+    filter_table.commit()
+    filter_table.refresh()
+    filter_table.autocommit = True
+
+  def banIPv4(self, source):
+    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+    rule = iptc.Rule()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule in chain.rules:
+      return False
+    chain.insert_rule(rule)
+    return True
+
+  def banIPv6(self, source):
+    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
+    rule = iptc.Rule6()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule in chain.rules:
+      return False
+    chain.insert_rule(rule)
+    return True
+
+  def unbanIPv4(self, source):
+    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+    rule = iptc.Rule()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule not in chain.rules: 
+      return False
+    chain.delete_rule(rule)
+    return True
+
+  def unbanIPv6(self, source):
+    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
+    rule = iptc.Rule6()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule not in chain.rules:
+      return False
+    chain.delete_rule(rule)
+    return True
+
+  def snat4(self, snat_target, source):
+    try:
+      table = iptc.Table('nat')
+      table.refresh()
+      chain = iptc.Chain(table, 'POSTROUTING')
+      table.autocommit = False
+      new_rule = self.getSnat4Rule(snat_target, source)
+
+      if not chain.rules:
+        # if there are no rules in the chain, insert the new rule directly
+        self.logger.logInfo(f'Added POSTROUTING rule for source network {new_rule.src} to SNAT target {snat_target}')
+        chain.insert_rule(new_rule)
+      else:
+        for position, rule in enumerate(chain.rules):
+          if not hasattr(rule.target, 'parameter'):
+              continue
+          match = all((
+            new_rule.get_src() == rule.get_src(),
+            new_rule.get_dst() == rule.get_dst(),
+            new_rule.target.parameters == rule.target.parameters,
+            new_rule.target.name == rule.target.name
+          ))
+          if position == 0:
+            if not match:
+              self.logger.logInfo(f'Added POSTROUTING rule for source network {new_rule.src} to SNAT target {snat_target}')
+              chain.insert_rule(new_rule)
+          else:
+            if match:
+              self.logger.logInfo(f'Remove rule for source network {new_rule.src} to SNAT target {snat_target} from POSTROUTING chain at position {position}')
+              chain.delete_rule(rule)
+
+      table.commit()
+      table.autocommit = True
+      return True
+    except:
+      self.logger.logCrit('Error running SNAT4, retrying...')
+      return False
+
+  def snat6(self, snat_target, source):
+    try:
+      table = iptc.Table6('nat')
+      table.refresh()
+      chain = iptc.Chain(table, 'POSTROUTING')
+      table.autocommit = False
+      new_rule = self.getSnat6Rule(snat_target, source)
+
+      if new_rule not in chain.rules:
+        self.logger.logInfo('Added POSTROUTING rule for source network %s to SNAT target %s' % (new_rule.src, snat_target))
+        chain.insert_rule(new_rule)
+      else:
+        for position, item in enumerate(chain.rules):
+          if item == new_rule:
+            if position != 0:
+              chain.delete_rule(new_rule)
+    
+      table.commit()
+      table.autocommit = True
+    except:
+      self.logger.logCrit('Error running SNAT6, retrying...')
+
+
+  def getSnat4Rule(self, snat_target, source):
+    rule = iptc.Rule()
+    rule.src = source
+    rule.dst = '!' + rule.src
+    target = rule.create_target("SNAT")
+    target.to_source = snat_target
+    match = rule.create_match("comment")
+    match.comment = f'{int(round(time.time()))}'
+    return rule
+
+  def getSnat6Rule(self, snat_target, source):
+    rule = iptc.Rule6()
+    rule.src = source
+    rule.dst = '!' + rule.src
+    target = rule.create_target("SNAT")
+    target.to_source = snat_target
+    return rule
+
+  def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
+    try:
+      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+
+      # insert mailcow isolation rule
+      rule = iptc.Rule()
+      rule.in_interface = f'!{_interface}'
+      rule.out_interface = _interface
+      rule.protocol = 'tcp'
+      rule.create_target("DROP")
+      match = rule.create_match("multiport")
+      match.dports = ','.join(map(str, _dports))
+
+      if rule in chain.rules:
+        chain.delete_rule(rule)
+      chain.insert_rule(rule, position=0)
+
+      # insert mailcow isolation exception rule
+      if _allow != "":
+        rule = iptc.Rule()
+        rule.src = _allow
+        rule.in_interface = f'!{_interface}'
+        rule.out_interface = _interface
+        rule.protocol = 'tcp'
+        rule.create_target("ACCEPT")
+        match = rule.create_match("multiport")
+        match.dports = ','.join(map(str, _dports))
+
+        if rule in chain.rules:
+          chain.delete_rule(rule)
+        chain.insert_rule(rule, position=0)
+
+
+      return True
+    except Exception as e:
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
+      return False

data/Dockerfiles/netfilter/modules/Logger.py

@@ -0,0 +1,27 @@
+import time
+import json
+
+class Logger:
+  def __init__(self):
+    self.r = None
+
+  def set_redis(self, redis):
+    self.r = redis
+
+  def log(self, priority, message):
+    tolog = {}
+    tolog['time'] = int(round(time.time()))
+    tolog['priority'] = priority
+    tolog['message'] = message
+    if self.r is not None:
+      self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
+    print(message)
+
+  def logWarn(self, message):
+    self.log('warn', message)
+
+  def logCrit(self, message):
+    self.log('crit', message)
+
+  def logInfo(self, message):
+    self.log('info', message)

data/Dockerfiles/netfilter/modules/NFTables.py

@@ -0,0 +1,656 @@
+import nftables
+import ipaddress
+import os
+
+class NFTables:
+  def __init__(self, chain_name, logger):
+    self.chain_name = chain_name
+    self.logger = logger
+
+    self.nft = nftables.Nftables()
+    self.nft.set_json_output(True)
+    self.nft.set_handle_output(True)
+    self.nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
+                            'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
+
+    self.search_current_chains()
+
+  def initChainIPv4(self):
+    self.insert_mailcow_chains("ip")
+
+  def initChainIPv6(self):
+    self.insert_mailcow_chains("ip6")
+
+  def checkIPv4ChainOrder(self):
+    return self.checkChainOrder("ip")
+
+  def checkIPv6ChainOrder(self):
+    return self.checkChainOrder("ip6")
+
+  def checkChainOrder(self, filter_table):
+    err = False
+    exit_code = None
+
+    for chain in ['input', 'forward']:
+      chain_position = self.check_mailcow_chains(filter_table, chain)
+      if chain_position is None: continue
+
+      if chain_position is False:
+        self.logger.logCrit(f'MAILCOW target not found in {filter_table} {chain} table, restarting container to fix it...')
+        err = True
+        exit_code = 2
+
+      if chain_position > 0:
+        self.logger.logCrit(f'MAILCOW target is in position {chain_position} in the {filter_table} {chain} table, restarting container to fix it...')
+        err = True
+        exit_code = 2
+
+    return err, exit_code
+
+  def clearIPv4Table(self):
+    self.clearTable("ip")
+
+  def clearIPv6Table(self):
+    self.clearTable("ip6")
+
+  def clearTable(self, _family):
+    is_empty_dict = True
+    json_command = self.get_base_dict()
+    chain_handle = self.get_chain_handle(_family, "filter", self.chain_name)
+    # if no handle, the chain doesn't exists
+    if chain_handle is not None:
+      is_empty_dict = False
+      # flush chain
+      mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+      flush_chain = {'flush': {'chain': mailcow_chain}}
+      json_command["nftables"].append(flush_chain)
+
+    # remove rule in forward chain
+    # remove rule in input chain
+    chains_family = [self.nft_chain_names[_family]['filter']['input'],
+                    self.nft_chain_names[_family]['filter']['forward'] ]
+
+    for chain_base in chains_family:
+      if not chain_base: continue
+
+      rules_handle = self.get_rules_handle(_family, "filter", chain_base)
+      if rules_handle is not None:
+        for r_handle in rules_handle:
+          is_empty_dict = False
+          mailcow_rule = {'family':_family,
+                          'table': 'filter',
+                          'chain': chain_base,
+                          'handle': r_handle }
+          delete_rules = {'delete': {'rule': mailcow_rule} }
+          json_command["nftables"].append(delete_rules)
+
+    # remove chain
+    # after delete all rules referencing this chain
+    if chain_handle is not None:
+      mc_chain_handle = {'family':_family,
+                        'table': 'filter',
+                        'name': self.chain_name,
+                        'handle': chain_handle }
+      delete_chain = {'delete': {'chain': mc_chain_handle} }
+      json_command["nftables"].append(delete_chain)
+
+    if is_empty_dict == False:
+      if self.nft_exec_dict(json_command):
+        self.logger.logInfo(f"Clear completed: {_family}")
+
+  def banIPv4(self, source):
+    ban_dict = self.get_ban_ip_dict(source, "ip")
+    return self.nft_exec_dict(ban_dict)
+
+  def banIPv6(self, source):
+    ban_dict = self.get_ban_ip_dict(source, "ip6")
+    return self.nft_exec_dict(ban_dict)
+
+  def unbanIPv4(self, source):
+    unban_dict = self.get_unban_ip_dict(source, "ip")
+    if not unban_dict:
+      return False
+    return self.nft_exec_dict(unban_dict)
+
+  def unbanIPv6(self, source):
+    unban_dict = self.get_unban_ip_dict(source, "ip6")
+    if not unban_dict:
+      return False
+    return self.nft_exec_dict(unban_dict)
+
+  def snat4(self, snat_target, source):
+    self.snat_rule("ip", snat_target, source)
+
+  def snat6(self, snat_target, source):
+    self.snat_rule("ip6", snat_target, source)
+
+
+  def nft_exec_dict(self, query: dict):
+    if not query: return False
+
+    rc, output, error = self.nft.json_cmd(query)
+    if rc != 0:
+      #self.logger.logCrit(f"Nftables Error: {error}")
+      return False
+
+    # Prevent returning False or empty string on commands that do not produce output
+    if rc == 0 and len(output) == 0:
+      return True
+
+    return output
+
+  def get_base_dict(self):
+    return {'nftables': [{ 'metainfo': { 'json_schema_version': 1} } ] }
+
+  def search_current_chains(self):
+    nft_chain_priority = {'ip': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} },
+                      'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+
+    # Command: 'nft list chains'
+    _list = {'list' : {'chains': 'null'} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset['nftables']:
+        chain = _object.get("chain")
+        if not chain: continue
+
+        _family = chain['family']
+        _table = chain['table']
+        _hook = chain.get("hook")
+        _priority = chain.get("prio")
+        _name = chain['name']
+
+        if _family not in self.nft_chain_names: continue
+        if _table not in self.nft_chain_names[_family]: continue
+        if _hook not in self.nft_chain_names[_family][_table]: continue
+        if _priority is None: continue
+
+        _saved_priority = nft_chain_priority[_family][_table][_hook]
+        if _saved_priority is None or _priority < _saved_priority:
+          # at this point, we know the chain has:
+          # hook and priority set
+          # and it has the lowest priority
+          nft_chain_priority[_family][_table][_hook] = _priority
+          self.nft_chain_names[_family][_table][_hook] = _name
+
+  def search_for_chain(self, kernel_ruleset: dict, chain_name: str):
+    found = False
+    for _object in kernel_ruleset["nftables"]:
+      chain = _object.get("chain")
+      if not chain:
+        continue
+      ch_name = chain.get("name")
+      if ch_name == chain_name:
+        found = True
+        break
+    return found
+
+  def get_chain_dict(self, _family: str, _name: str):
+    # nft (add | create) chain [<family>] <table> <name> 
+    _chain_opts = {'family': _family, 'table': 'filter', 'name': _name  }
+    _add = {'add': {'chain': _chain_opts} }
+    final_chain = self.get_base_dict()
+    final_chain["nftables"].append(_add)
+    return final_chain
+
+  def get_mailcow_jump_rule_dict(self, _family: str, _chain: str):
+    _jump_rule = self.get_base_dict()
+    _expr_opt=[]
+    _expr_counter = {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}
+    _counter_dict = {'counter': _expr_counter}
+    _expr_opt.append(_counter_dict)
+
+    _jump_opts = {'jump': {'target': self.chain_name} }
+
+    _expr_opt.append(_jump_opts)
+
+    _rule_params = {'family': _family,
+                    'table': 'filter',
+                    'chain': _chain,
+                    'expr': _expr_opt,
+                    'comment': "mailcow" }
+
+    _add_rule = {'insert': {'rule': _rule_params} }
+
+    _jump_rule["nftables"].append(_add_rule)
+
+    return _jump_rule
+
+  def insert_mailcow_chains(self, _family: str):
+    nft_input_chain = self.nft_chain_names[_family]['filter']['input']
+    nft_forward_chain = self.nft_chain_names[_family]['filter']['forward']
+    # Command: 'nft list table <family> filter'
+    _table_opts = {'family': _family, 'name': 'filter'}
+    _list = {'list': {'table': _table_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      # chain
+      if not self.search_for_chain(kernel_ruleset, self.chain_name):
+        cadena = self.get_chain_dict(_family, self.chain_name)
+        if self.nft_exec_dict(cadena):
+          self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+
+      input_jump_found, forward_jump_found = False, False
+
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+
+        rule = _object["rule"]
+        if nft_input_chain and rule["chain"] == nft_input_chain:
+          if rule.get("comment") and rule["comment"] == "mailcow":
+            input_jump_found = True
+        if nft_forward_chain and rule["chain"] == nft_forward_chain:
+          if rule.get("comment") and rule["comment"] == "mailcow":
+            forward_jump_found = True
+
+      if not input_jump_found:
+        command = self.get_mailcow_jump_rule_dict(_family, nft_input_chain)
+        self.nft_exec_dict(command)
+
+      if not forward_jump_found:
+        command = self.get_mailcow_jump_rule_dict(_family, nft_forward_chain)
+        self.nft_exec_dict(command)
+
+  def delete_nat_rule(self, _family:str, _chain: str, _handle:str):
+    delete_command = self.get_base_dict()
+    _rule_opts = {'family': _family,
+                  'table': 'nat',
+                  'chain': _chain,
+                  'handle': _handle  }
+    _delete = {'delete': {'rule': _rule_opts} }
+    delete_command["nftables"].append(_delete)
+
+    return self.nft_exec_dict(delete_command)
+
+  def delete_filter_rule(self, _family:str, _chain: str, _handle:str):
+    delete_command = self.get_base_dict()
+    _rule_opts = {'family': _family,
+                  'table': 'filter',
+                  'chain': _chain,
+                  'handle': _handle  }
+    _delete = {'delete': {'rule': _rule_opts} }
+    delete_command["nftables"].append(_delete)
+
+    return self.nft_exec_dict(delete_command)
+
+  def snat_rule(self, _family: str, snat_target: str, source_address: str):
+    chain_name = self.nft_chain_names[_family]['nat']['postrouting']
+
+    # no postrouting chain, may occur if docker has ipv6 disabled.
+    if not chain_name: return
+
+    # Command: nft list chain <family> nat <chain_name>
+    _chain_opts = {'family': _family, 'table': 'nat', 'name': chain_name}
+    _list = {'list':{'chain': _chain_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if not kernel_ruleset:
+      return
+
+    rule_position = 0
+    rule_handle = None
+    rule_found = False
+    for _object in kernel_ruleset["nftables"]:
+      if not _object.get("rule"):
+        continue
+
+      rule = _object["rule"]
+      if not rule.get("comment") or not rule["comment"] == "mailcow":
+        rule_position +=1
+        continue
+
+      rule_found = True
+      rule_handle = rule["handle"]
+      break
+
+    dest_net = ipaddress.ip_network(source_address)
+    target_net = ipaddress.ip_network(snat_target)
+
+    if rule_found:
+      saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
+      saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
+
+      daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
+      daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
+
+      target_ip = rule["expr"][3]["snat"]["addr"]
+
+      saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len))
+      daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len))
+      current_target_net = ipaddress.ip_network(target_ip)
+
+      match = all((
+                dest_net == saddr_net,
+                dest_net == daddr_net,
+                target_net == current_target_net
+              ))
+      try:
+        if rule_position == 0:
+          if not match:
+            # Position 0 , it is a mailcow rule , but it does not have the same parameters
+            if self.delete_nat_rule(_family, chain_name, rule_handle):
+              self.logger.logInfo(f'Remove rule for source network {saddr_net} to SNAT target {target_net} from {_family} nat {chain_name} chain, rule does not match configured parameters')
+        else:
+          # Position > 0 and is mailcow rule
+          if self.delete_nat_rule(_family, chain_name, rule_handle):
+            self.logger.logInfo(f'Remove rule for source network {saddr_net} to SNAT target {target_net} from {_family} nat {chain_name} chain, rule is at position {rule_position}')
+      except:
+          self.logger.logCrit(f"Error running SNAT on {_family}, retrying..." )
+    else:
+      # rule not found
+      json_command = self.get_base_dict()
+      try:
+        snat_dict = {'snat': {'addr': str(target_net.network_address)} }
+
+        expr_counter = {'family': _family, 'table': 'nat', 'packets': 0, 'bytes': 0}
+        counter_dict = {'counter': expr_counter}
+
+        prefix_dict = {'prefix': {'addr': str(dest_net.network_address), 'len': int(dest_net.prefixlen)} }
+        payload_dict = {'payload': {'protocol': _family, 'field': "saddr"} }
+        match_dict1 = {'match': {'op': '==', 'left': payload_dict, 'right': prefix_dict} }
+
+        payload_dict2 = {'payload': {'protocol': _family, 'field': "daddr"} }
+        match_dict2 = {'match': {'op': '!=', 'left': payload_dict2, 'right': prefix_dict } }
+        expr_list = [
+                    match_dict1,
+                    match_dict2,
+                    counter_dict,
+                    snat_dict
+                    ]
+        rule_fields = {'family': _family,
+                        'table': 'nat',
+                        'chain': chain_name,
+                        'comment': "mailcow",
+                        'expr': expr_list }
+
+        insert_dict = {'insert': {'rule': rule_fields} }
+        json_command["nftables"].append(insert_dict)
+        if self.nft_exec_dict(json_command):
+          self.logger.logInfo(f'Added {_family} nat {chain_name} rule for source network {dest_net} to {target_net}')
+      except:
+        self.logger.logCrit(f"Error running SNAT on {_family}, retrying...")
+
+  def get_chain_handle(self, _family: str, _table: str, chain_name: str):
+    chain_handle = None
+    # Command: 'nft list chains {family}'
+    _list = {'list': {'chains': {'family': _family} } }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("chain"):
+          continue
+        chain = _object["chain"]
+        if chain["family"] == _family and chain["table"] == _table and chain["name"] == chain_name:
+          chain_handle = chain["handle"]
+          break
+    return chain_handle
+
+  def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
+    rule_handle = []
+    # Command: 'nft list chain {family} {table} {chain_name}'
+    _chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
+    _list = {'list': {'chain': _chain_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+
+        rule = _object["rule"]
+        if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
+          if rule.get("comment") and rule["comment"] == _comment_filter:
+            rule_handle.append(rule["handle"])
+    return rule_handle
+
+  def get_ban_ip_dict(self, ipaddr: str, _family: str):
+    json_command = self.get_base_dict()
+
+    expr_opt = []
+    ipaddr_net = ipaddress.ip_network(ipaddr)
+    right_dict = {'prefix': {'addr': str(ipaddr_net.network_address), 'len': int(ipaddr_net.prefixlen) } }
+
+    left_dict = {'payload': {'protocol': _family, 'field': 'saddr'} }
+    match_dict = {'op': '==', 'left': left_dict, 'right': right_dict }
+    expr_opt.append({'match': match_dict})
+
+    counter_dict = {'counter': {'family': _family, 'table': "filter", 'packets': 0, 'bytes': 0} }
+    expr_opt.append(counter_dict)
+
+    expr_opt.append({'drop': "null"})
+
+    rule_dict = {'family': _family, 'table': "filter", 'chain': self.chain_name, 'expr': expr_opt}
+
+    base_dict = {'insert': {'rule': rule_dict} }
+    json_command["nftables"].append(base_dict)
+
+    return json_command
+
+  def get_unban_ip_dict(self, ipaddr:str, _family: str):
+    json_command = self.get_base_dict()
+    # Command: 'nft list chain {s_family} filter  MAILCOW'
+    _chain_opts = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+    _list = {'list': {'chain': _chain_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    rule_handle = None
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+
+        rule = _object["rule"]["expr"][0]["match"]
+        left_opt = rule["left"]["payload"]
+        if not left_opt["protocol"] == _family:
+          continue
+        if not left_opt["field"] =="saddr":
+          continue
+
+        # ip currently banned
+        rule_right = rule["right"]
+        if isinstance(rule_right, dict):
+          current_rule_ip = rule_right["prefix"]["addr"] + '/' + str(rule_right["prefix"]["len"])
+        else:
+          current_rule_ip = rule_right
+        current_rule_net = ipaddress.ip_network(current_rule_ip)
+
+        # ip to ban
+        candidate_net = ipaddress.ip_network(ipaddr)
+
+        if current_rule_net == candidate_net:
+          rule_handle = _object["rule"]["handle"]
+          break
+
+      if rule_handle is not None:
+        mailcow_rule = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule_handle}
+        delete_rule = {'delete': {'rule': mailcow_rule} }
+        json_command["nftables"].append(delete_rule)
+      else:
+        return False
+
+    return json_command
+
+  def check_mailcow_chains(self, family: str, chain: str):
+    position = 0
+    rule_found = False
+    chain_name = self.nft_chain_names[family]['filter'][chain]
+
+    if not chain_name: return None
+
+    _chain_opts = {'family': family, 'table': 'filter', 'name': chain_name}
+    _list = {'list': {'chain': _chain_opts}}
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+        rule = _object["rule"]
+        if rule.get("comment") and rule["comment"] == "mailcow":
+          rule_found = True
+          break
+
+        position+=1
+
+    return position if rule_found else False
+
+  def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
+    family = "ip"
+    table = "filter"
+    comment_filter_drop = "mailcow isolation"
+    comment_filter_allow = "mailcow isolation allow"
+    json_command = self.get_base_dict()
+
+    # Delete old mailcow isolation rules
+    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
+    for handle in handles:
+      self.delete_filter_rule(family, self.chain_name, handle)
+    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
+    for handle in handles:
+      self.delete_filter_rule(family, self.chain_name, handle)
+
+    # insert mailcow isolation rule
+    _match_dict_drop = [
+      {
+        "match": {
+          "op": "!=",
+          "left": {
+            "meta": {
+              "key": "iifname"
+            }
+          },
+          "right": _interface
+        }
+      },
+      {
+        "match": {
+          "op": "==",
+          "left": {
+            "meta": {
+              "key": "oifname"
+            }
+          },
+          "right": _interface
+        }
+      },
+      {
+        "match": {
+          "op": "==",
+          "left": {
+            "payload": {
+              "protocol": "tcp",
+              "field": "dport"
+            }
+          },
+          "right": {
+            "set": _dports
+          }
+        }
+      },
+      {
+        "counter": {
+          "packets": 0,
+          "bytes": 0
+        }
+      },
+      {
+        "drop": None
+      }
+    ]
+    rule_drop = { "insert": { "rule": {
+      "family": family,
+      "table": table,
+      "chain": self.chain_name,
+      "comment": comment_filter_drop,
+      "expr": _match_dict_drop
+    }}}
+    json_command["nftables"].append(rule_drop)
+
+    # insert mailcow isolation allow rule
+    if _allow != "":
+      _match_dict_allow = [
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "payload": {
+                "protocol": "ip",
+                "field": "saddr"
+              }
+            },
+            "right": _allow
+          }
+        },
+        {
+          "match": {
+            "op": "!=",
+            "left": {
+              "meta": {
+                "key": "iifname"
+              }
+            },
+            "right": _interface
+          }
+        },
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "meta": {
+                "key": "oifname"
+              }
+            },
+            "right": _interface
+          }
+        },
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "payload": {
+                "protocol": "tcp",
+                "field": "dport"
+              }
+            },
+            "right": {
+              "set": _dports
+            }
+          }
+        },
+        {
+          "counter": {
+            "packets": 0,
+            "bytes": 0
+          }
+        },
+        {
+          "accept": None
+        }
+      ]
+      rule_allow = { "insert": { "rule": {
+        "family": family,
+        "table": table,
+        "chain": self.chain_name,
+        "comment": comment_filter_allow,
+        "expr": _match_dict_allow
+      }}}
+      json_command["nftables"].append(rule_allow)
+
+    success = self.nft_exec_dict(json_command)
+    if success == False:
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation")
+      return False
+
+    return True

data/Dockerfiles/netfilter/server.py

@@ -1,557 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import time
-import atexit
-import signal
-import ipaddress
-from collections import Counter
-from random import randint
-from threading import Thread
-from threading import Lock
-import redis
-import json
-import iptc
-import dns.resolver
-import dns.exception
-
-while True:
-  try:
-    redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
-    redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
-    if "".__eq__(redis_slaveof_ip):
-      r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
-    else:
-      r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
-    r.ping()
-  except Exception as ex:
-    print('%s - trying again in 3 seconds'  % (ex))
-    time.sleep(3)
-  else:
-    break
-
-pubsub = r.pubsub()
-
-WHITELIST = []
-BLACKLIST= []
-
-bans = {}
-
-quit_now = False
-lock = Lock()
-
-def log(priority, message):
-  tolog = {}
-  tolog['time'] = int(round(time.time()))
-  tolog['priority'] = priority
-  tolog['message'] = message
-  r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
-  print(message)
-
-def logWarn(message):
-  log('warn', message)
-
-def logCrit(message):
-  log('crit', message)
-
-def logInfo(message):
-  log('info', message)
-
-def refreshF2boptions():
-  global f2boptions
-  global quit_now
-  if not r.get('F2B_OPTIONS'):
-    f2boptions = {}
-    f2boptions['ban_time'] = int
-    f2boptions['max_attempts'] = int
-    f2boptions['retry_window'] = int
-    f2boptions['netban_ipv4'] = int
-    f2boptions['netban_ipv6'] = int
-    f2boptions['ban_time'] = r.get('F2B_BAN_TIME') or 1800
-    f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS') or 10
-    f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW') or 600
-    f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4') or 32
-    f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6') or 128
-    r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
-  else:
-    try:
-      f2boptions = {}
-      f2boptions = json.loads(r.get('F2B_OPTIONS'))
-    except ValueError:
-      print('Error loading F2B options: F2B_OPTIONS is not json')
-      quit_now = True
-
-def refreshF2bregex():
-  global f2bregex
-  global quit_now
-  if not r.get('F2B_REGEX'):
-    f2bregex = {}
-    f2bregex[1] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed'
-    f2bregex[2] = '-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
-    f2bregex[3] = '-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
-    f2bregex[4] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
-    f2bregex[5] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
-    f2bregex[6] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
-    f2bregex[7] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
-    f2bregex[8] = '-login: Aborted login \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
-    r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
-  else:
-    try:
-      f2bregex = {}
-      f2bregex = json.loads(r.get('F2B_REGEX'))
-    except ValueError:
-      print('Error loading F2B options: F2B_REGEX is not json')
-      quit_now = True
-
-if r.exists('F2B_LOG'):
-  r.rename('F2B_LOG', 'NETFILTER_LOG')
-
-def mailcowChainOrder():
-  global lock
-  global quit_now
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      filter4_table = iptc.Table(iptc.Table.FILTER)
-      filter6_table = iptc.Table6(iptc.Table6.FILTER)
-      filter4_table.refresh()
-      filter6_table.refresh()
-      for f in [filter4_table, filter6_table]:
-        forward_chain = iptc.Chain(f, 'FORWARD')
-        input_chain = iptc.Chain(f, 'INPUT')
-        for chain in [forward_chain, input_chain]:
-          target_found = False
-          for position, item in enumerate(chain.rules):
-            if item.target.name == 'MAILCOW':
-              target_found = True
-              if position > 2:
-                logCrit('Error in %s chain order: MAILCOW on position %d, restarting container' % (chain.name, position))
-                quit_now = True
-          if not target_found:
-            logCrit('Error in %s chain: MAILCOW target not found, restarting container' % (chain.name))
-            quit_now = True
-
-def ban(address):
-  global lock
-  refreshF2boptions()
-  BAN_TIME = int(f2boptions['ban_time'])
-  MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-  RETRY_WINDOW = int(f2boptions['retry_window'])
-  NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
-  NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
-
-  ip = ipaddress.ip_address(address)
-  if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
-    ip = ip.ipv4_mapped
-    address = str(ip)
-  if ip.is_private or ip.is_loopback:
-    return
-
-  self_network = ipaddress.ip_network(address)
-
-  with lock:
-    temp_whitelist = set(WHITELIST)
-
-  if temp_whitelist:
-    for wl_key in temp_whitelist:
-      wl_net = ipaddress.ip_network(wl_key, False)
-      if wl_net.overlaps(self_network):
-        logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net))
-        return
-
-  net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
-  net = str(net)
-
-  if not net in bans or time.time() - bans[net]['last_attempt'] > RETRY_WINDOW:
-    bans[net] = { 'attempts': 0 }
-    active_window = RETRY_WINDOW
-  else:
-    active_window = time.time() - bans[net]['last_attempt']
-
-  bans[net]['attempts'] += 1
-  bans[net]['last_attempt'] = time.time()
-
-  active_window = time.time() - bans[net]['last_attempt']
-
-  if bans[net]['attempts'] >= MAX_ATTEMPTS:
-    cur_time = int(round(time.time()))
-    logCrit('Banning %s for %d minutes' % (net, BAN_TIME / 60))
-    if type(ip) is ipaddress.IPv4Address:
-      with lock:
-        chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), 'MAILCOW')
-        rule = iptc.Rule()
-        rule.src = net
-        target = iptc.Target(rule, "REJECT")
-        rule.target = target
-        if rule not in chain.rules:
-          chain.insert_rule(rule)
-    else:
-      with lock:
-        chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), 'MAILCOW')
-        rule = iptc.Rule6()
-        rule.src = net
-        target = iptc.Target(rule, "REJECT")
-        rule.target = target
-        if rule not in chain.rules:
-          chain.insert_rule(rule)
-    r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + BAN_TIME)
-  else:
-    logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
-
-def unban(net):
-  global lock
-  if not net in bans:
-   logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net)
-   r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-   return
-  logInfo('Unbanning %s' % net)
-  if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
-    with lock:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), 'MAILCOW')
-      rule = iptc.Rule()
-      rule.src = net
-      target = iptc.Target(rule, "REJECT")
-      rule.target = target
-      if rule in chain.rules:
-        chain.delete_rule(rule)
-  else:
-    with lock:
-      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), 'MAILCOW')
-      rule = iptc.Rule6()
-      rule.src = net
-      target = iptc.Target(rule, "REJECT")
-      rule.target = target
-      if rule in chain.rules:
-        chain.delete_rule(rule)
-  r.hdel('F2B_ACTIVE_BANS', '%s' % net)
-  r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-  if net in bans:
-    del bans[net]
-
-def permBan(net, unban=False):
-  global lock
-  if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
-    with lock:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), 'MAILCOW')
-      rule = iptc.Rule()
-      rule.src = net
-      target = iptc.Target(rule, "REJECT")
-      rule.target = target
-      if rule not in chain.rules and not unban:
-        logCrit('Add host/network %s to blacklist' % net)
-        chain.insert_rule(rule)
-        r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time()))) 
-      elif rule in chain.rules and unban:
-        logCrit('Remove host/network %s from blacklist' % net)
-        chain.delete_rule(rule)
-        r.hdel('F2B_PERM_BANS', '%s' % net)
-  else:
-    with lock:
-      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), 'MAILCOW')
-      rule = iptc.Rule6()
-      rule.src = net
-      target = iptc.Target(rule, "REJECT")
-      rule.target = target
-      if rule not in chain.rules and not unban:
-        logCrit('Add host/network %s to blacklist' % net)
-        chain.insert_rule(rule)
-        r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time()))) 
-      elif rule in chain.rules and unban:
-        logCrit('Remove host/network %s from blacklist' % net)
-        chain.delete_rule(rule)
-        r.hdel('F2B_PERM_BANS', '%s' % net)
-
-def quit(signum, frame):
-  global quit_now
-  quit_now = True
-
-def clear():
-  global lock
-  logInfo('Clearing all bans')
-  for net in bans.copy():
-    unban(net)
-  with lock:
-    filter4_table = iptc.Table(iptc.Table.FILTER)
-    filter6_table = iptc.Table6(iptc.Table6.FILTER)
-    for filter_table in [filter4_table, filter6_table]:
-      filter_table.autocommit = False
-      forward_chain = iptc.Chain(filter_table, "FORWARD")
-      input_chain = iptc.Chain(filter_table, "INPUT")
-      mailcow_chain = iptc.Chain(filter_table, "MAILCOW")
-      if mailcow_chain in filter_table.chains:
-        for rule in mailcow_chain.rules:
-          mailcow_chain.delete_rule(rule)
-        for rule in forward_chain.rules:
-          if rule.target.name == 'MAILCOW':
-            forward_chain.delete_rule(rule)
-        for rule in input_chain.rules:
-          if rule.target.name == 'MAILCOW':
-            input_chain.delete_rule(rule)
-        filter_table.delete_chain("MAILCOW")
-      filter_table.commit()
-      filter_table.refresh()
-      filter_table.autocommit = True
-    r.delete('F2B_ACTIVE_BANS')
-    r.delete('F2B_PERM_BANS')
-    pubsub.unsubscribe()
-
-def watch():
-  logInfo('Watching Redis channel F2B_CHANNEL')
-  pubsub.subscribe('F2B_CHANNEL')
-
-  while not quit_now:
-    for item in pubsub.listen():
-      refreshF2bregex()
-      for rule_id, rule_regex in f2bregex.items():
-        if item['data'] and item['type'] == 'message':
-          try:
-            result = re.search(rule_regex, item['data'])
-          except re.error:
-            result = False
-          if result:
-            addr = result.group(1)
-            ip = ipaddress.ip_address(addr)
-            if ip.is_private or ip.is_loopback:
-              continue
-            logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
-            ban(addr)
-
-def snat4(snat_target):
-  global lock
-  global quit_now
-
-  def get_snat4_rule():
-    rule = iptc.Rule()
-    rule.src = os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24'
-    rule.dst = '!' + rule.src
-    target = rule.create_target("SNAT")
-    target.to_source = snat_target
-    return rule
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      try:
-        table = iptc.Table('nat')
-        table.refresh()
-        chain = iptc.Chain(table, 'POSTROUTING')
-        table.autocommit = False
-        if get_snat4_rule() not in chain.rules:
-          logCrit('Added POSTROUTING rule for source network %s to SNAT target %s' % (get_snat4_rule().src, snat_target))
-          chain.insert_rule(get_snat4_rule())
-          table.commit()
-        else:
-          for position, item in enumerate(chain.rules):
-            if item == get_snat4_rule():
-              if position != 0:
-                chain.delete_rule(get_snat4_rule())
-          table.commit()
-        table.autocommit = True
-      except:
-        print('Error running SNAT4, retrying...') 
-
-def snat6(snat_target):
-  global lock
-  global quit_now
-
-  def get_snat6_rule():
-    rule = iptc.Rule6()
-    rule.src = os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64')
-    rule.dst = '!' + rule.src
-    target = rule.create_target("SNAT")
-    target.to_source = snat_target
-    return rule
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      try:
-        table = iptc.Table6('nat')
-        table.refresh()
-        chain = iptc.Chain(table, 'POSTROUTING')
-        table.autocommit = False
-        if get_snat6_rule() not in chain.rules:
-          logInfo('Added POSTROUTING rule for source network %s to SNAT target %s' % (get_snat6_rule().src, snat_target))
-          chain.insert_rule(get_snat6_rule())
-          table.commit()
-        else:
-          for position, item in enumerate(chain.rules):
-            if item == get_snat6_rule():
-              if position != 0:
-                chain.delete_rule(get_snat6_rule())
-          table.commit()
-        table.autocommit = True
-      except:
-        print('Error running SNAT6, retrying...') 
-
-def autopurge():
-  while not quit_now:
-    time.sleep(10)
-    refreshF2boptions()
-    BAN_TIME = int(f2boptions['ban_time'])
-    MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-    QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
-    if QUEUE_UNBAN:
-      for net in QUEUE_UNBAN:
-        unban(str(net))
-    for net in bans.copy():
-      if bans[net]['attempts'] >= MAX_ATTEMPTS:
-        if time.time() - bans[net]['last_attempt'] > BAN_TIME:
-          unban(net)
-
-def isIpNetwork(address):
-  try:
-    ipaddress.ip_network(address, False)
-  except ValueError:
-    return False
-  return True
-
-
-def genNetworkList(list):
-  resolver = dns.resolver.Resolver()
-  hostnames = []
-  networks = []
-  for key in list:
-    if isIpNetwork(key):
-      networks.append(key)
-    else:
-      hostnames.append(key)
-  for hostname in hostnames:
-    hostname_ips = []
-    for rdtype in ['A', 'AAAA']:
-      try:
-        answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
-      except dns.exception.Timeout:
-        logInfo('Hostname %s timedout on resolve' % hostname)
-        break
-      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-        continue
-      except dns.exception.DNSException as dnsexception:
-        logInfo('%s' % dnsexception)
-        continue
-      for rdata in answer:
-        hostname_ips.append(rdata.to_text())
-    networks.extend(hostname_ips)
-  return set(networks)
-
-def whitelistUpdate():
-  global lock
-  global quit_now
-  global WHITELIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_WHITELIST')
-    new_whitelist = []
-    if list:
-      new_whitelist = genNetworkList(list)
-    with lock:
-      if Counter(new_whitelist) != Counter(WHITELIST):
-        WHITELIST = new_whitelist
-        logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST))
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0)) 
-
-def blacklistUpdate():
-  global quit_now
-  global BLACKLIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_BLACKLIST')
-    new_blacklist = []
-    if list:
-      new_blacklist = genNetworkList(list)
-    if Counter(new_blacklist) != Counter(BLACKLIST): 
-      addban = set(new_blacklist).difference(BLACKLIST)
-      delban = set(BLACKLIST).difference(new_blacklist)
-      BLACKLIST = new_blacklist
-      logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST))
-      if addban:
-        for net in addban:
-          permBan(net=net)
-      if delban:
-        for net in delban:
-          permBan(net=net, unban=True)
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0)) 
-
-def initChain():
-  # Is called before threads start, no locking
-  print("Initializing mailcow netfilter chain")
-  # IPv4
-  if not iptc.Chain(iptc.Table(iptc.Table.FILTER), "MAILCOW") in iptc.Table(iptc.Table.FILTER).chains:
-    iptc.Table(iptc.Table.FILTER).create_chain("MAILCOW")
-  for c in ['FORWARD', 'INPUT']:
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), c)
-    rule = iptc.Rule()
-    rule.src = '0.0.0.0/0'
-    rule.dst = '0.0.0.0/0'
-    target = iptc.Target(rule, "MAILCOW")
-    rule.target = target
-    if rule not in chain.rules:
-      chain.insert_rule(rule)
-  # IPv6
-  if not iptc.Chain(iptc.Table6(iptc.Table6.FILTER), "MAILCOW") in iptc.Table6(iptc.Table6.FILTER).chains:
-    iptc.Table6(iptc.Table6.FILTER).create_chain("MAILCOW")
-  for c in ['FORWARD', 'INPUT']:
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), c)
-    rule = iptc.Rule6()
-    rule.src = '::/0'
-    rule.dst = '::/0'
-    target = iptc.Target(rule, "MAILCOW")
-    rule.target = target
-    if rule not in chain.rules:
-      chain.insert_rule(rule)
-
-if __name__ == '__main__':
-
-  # In case a previous session was killed without cleanup
-  clear()
-  # Reinit MAILCOW chain
-  initChain()
-
-  watch_thread = Thread(target=watch)
-  watch_thread.daemon = True
-  watch_thread.start()
-
-  if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv4Address:
-        snat4_thread = Thread(target=snat4,args=(snat_ip,))
-        snat4_thread.daemon = True
-        snat4_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
-
-  if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT6_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv6Address:
-        snat6_thread = Thread(target=snat6,args=(snat_ip,))
-        snat6_thread.daemon = True
-        snat6_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
-
-  autopurge_thread = Thread(target=autopurge)
-  autopurge_thread.daemon = True
-  autopurge_thread.start()
-
-  mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
-  mailcowchainwatch_thread.daemon = True
-  mailcowchainwatch_thread.start()
-
-  blacklistupdate_thread = Thread(target=blacklistUpdate)
-  blacklistupdate_thread.daemon = True
-  blacklistupdate_thread.start()
-
-  whitelistupdate_thread = Thread(target=whitelistUpdate)
-  whitelistupdate_thread.daemon = True
-  whitelistupdate_thread.start()
-
-  signal.signal(signal.SIGTERM, quit)
-  atexit.register(clear)
-
-  while not quit_now:
-    time.sleep(0.5)

data/Dockerfiles/olefy/Dockerfile

@@ -1,18 +1,19 @@
-FROM alpine:3.11
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM alpine:3.19
+LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
 WORKDIR /app
 
 #RUN addgroup -S olefy && adduser -S olefy -G olefy \
-RUN apk add --virtual .build-deps gcc python3-dev musl-dev libffi-dev openssl-dev \
-  && apk add --update --no-cache python3 openssl tzdata libmagic \
+RUN apk add --virtual .build-deps gcc musl-dev python3-dev libffi-dev openssl-dev cargo \
+  && apk add --update --no-cache python3 py3-pip openssl tzdata libmagic \
   && pip3 install --upgrade pip \
   && pip3 install --upgrade asyncio python-magic \
-  && pip3 install --upgrade https://github.com/HeinleinSupport/oletools/archive/master.zip \
+  && pip3 install --upgrade https://github.com/decalage2/oletools/archive/master.zip \
   && apk del .build-deps
-#  && sed -i 's/decompress_stream(bytearray(compressed_code))/bytes2str(decompress_stream(bytearray(compressed_code)))/g' /usr/lib/python3.8/site-packages/oletools/olevba.py
+#  && sed -i 's/template_injection_detected = True/template_injection_detected = False/g' /usr/lib/python3.9/site-packages/oletools/olevba.py
 
-ADD https://raw.githubusercontent.com/HeinleinSupport/olefy/master/olefy.py /app/
+ADD olefy.py /app/
 
 RUN chown -R nobody:nobody /app /tmp
 

data/Dockerfiles/olefy/olefy.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2020, Dennis Kalbhen <d.kalbhen@heinlein-support.de>
+# Copyright (c) 2020, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+#
+#  olefy is a little helper socket to use oletools with rspamd. (https://rspamd.com)
+#  Please find updates and issues here: https://github.com/HeinleinSupport/olefy
+#
+###
+
+from subprocess import Popen, PIPE
+import sys
+import os
+import logging
+import asyncio
+import time
+import magic
+import re
+
+# merge variables from /etc/olefy.conf and the defaults
+olefy_listen_addr_string = os.getenv('OLEFY_BINDADDRESS', '127.0.0.1,::1')
+olefy_listen_port = int(os.getenv('OLEFY_BINDPORT', '10050'))
+olefy_tmp_dir = os.getenv('OLEFY_TMPDIR', '/tmp')
+olefy_python_path = os.getenv('OLEFY_PYTHON_PATH', '/usr/bin/python3')
+olefy_olevba_path = os.getenv('OLEFY_OLEVBA_PATH', '/usr/local/bin/olevba3')
+# 10:DEBUG, 20:INFO, 30:WARNING, 40:ERROR, 50:CRITICAL
+olefy_loglvl = int(os.getenv('OLEFY_LOGLVL', 20))
+olefy_min_length = int(os.getenv('OLEFY_MINLENGTH', 500))
+olefy_del_tmp = int(os.getenv('OLEFY_DEL_TMP', 1))
+olefy_del_tmp_failed = int(os.getenv('OLEFY_DEL_TMP_FAILED', 1))
+
+# internal used variables
+request_time = '0000000000.000000'
+olefy_protocol = 'OLEFY'
+olefy_ping = 'PING'
+olefy_protocol_sep = '\n\n'
+olefy_headers = {}
+
+# init logging
+logger = logging.getLogger('olefy')
+logging.basicConfig(stream=sys.stdout, level=olefy_loglvl, format='olefy %(levelname)s %(funcName)s %(message)s')
+
+logger.debug('olefy listen address string: {} (type {})'.format(olefy_listen_addr_string, type(olefy_listen_addr_string)))
+
+if not olefy_listen_addr_string:
+    olefy_listen_addr = ""
+else:
+    addr_re = re.compile('[\[" \]]')
+    olefy_listen_addr = addr_re.sub('', olefy_listen_addr_string.replace("'", "")).split(',')
+
+# log runtime variables
+logger.info('olefy listen address: {} (type: {})'.format(olefy_listen_addr, type(olefy_listen_addr)))
+logger.info('olefy listen port: {}'.format(olefy_listen_port))
+logger.info('olefy tmp dir: {}'.format(olefy_tmp_dir))
+logger.info('olefy python path: {}'.format(olefy_python_path))
+logger.info('olefy olvba path: {}'.format(olefy_olevba_path))
+logger.info('olefy log level: {}'.format(olefy_loglvl))
+logger.info('olefy min file length: {}'.format(olefy_min_length))
+logger.info('olefy delete tmp file: {}'.format(olefy_del_tmp))
+logger.info('olefy delete tmp file when failed: {}'.format(olefy_del_tmp_failed))
+
+if not os.path.isfile(olefy_python_path):
+    logger.critical('python path not found: {}'.format(olefy_python_path))
+    exit(1)
+if not os.path.isfile(olefy_olevba_path):
+    logger.critical('olevba path not found: {}'.format(olefy_olevba_path))
+    exit(1)
+
+# olefy protocol function
+def protocol_split( olefy_line ):
+    header_lines = olefy_line.split('\n')
+    for line in header_lines:
+        if line == 'OLEFY/1.0':
+            olefy_headers['olefy'] = line
+        elif line != '':
+            kv = line.split(': ')
+            if kv[0] != '' and kv[1] != '':
+                olefy_headers[kv[0]] = kv[1]
+    logger.debug('olefy_headers: {}'.format(olefy_headers))
+
+# calling oletools
+def oletools( stream, tmp_file_name, lid ):
+    if olefy_min_length > stream.__len__():
+        logger.error('{} {} bytes (Not Scanning! File smaller than {!r})'.format(lid, stream.__len__(), olefy_min_length))
+        out = b'[ { "error": "File too small" } ]'
+    else:
+        tmp_file = open(tmp_file_name, 'wb')
+        tmp_file.write(stream)
+        tmp_file.close()
+
+        file_magic = magic.Magic(mime=True, uncompress=True)
+        file_mime = file_magic.from_file(tmp_file_name)
+        logger.info('{} {} (libmagic output)'.format(lid, file_mime))
+
+        # do the olefy
+        cmd_tmp = Popen([olefy_python_path, olefy_olevba_path, '-a', '-j' , '-l', 'error', tmp_file_name], stdout=PIPE, stderr=PIPE)
+        out, err = cmd_tmp.communicate()
+        out = bytes(out.decode('utf-8', 'ignore').replace('  ', ' ').replace('\t', '').replace('\n', '').replace('XLMMacroDeobfuscator: pywin32 is not installed (only is required if you want to use MS Excel)', ''), encoding="utf-8")
+        failed = False
+        if out.__len__() < 30:
+            logger.error('{} olevba returned <30 chars - rc: {!r}, response: {!r}, error: {!r}'.format(lid,cmd_tmp.returncode, 
+                out.decode('utf-8', 'ignore'), err.decode('utf-8', 'ignore')))
+            out = b'[ { "error": "Unhandled error - too short olevba response" } ]'
+            failed = True
+        elif err.__len__() > 10 and cmd_tmp.returncode == 9:
+            logger.error("{} olevba stderr >10 chars - rc: {!r}, response: {!r}".format(lid, cmd_tmp.returncode, err.decode("utf-8", "ignore")))
+            out = b'[ { "error": "Decrypt failed" } ]'
+            failed = True
+        elif err.__len__() > 10 and cmd_tmp.returncode > 9:
+            logger.error('{} olevba stderr >10 chars - rc: {!r}, response: {!r}'.format(lid, cmd_tmp.returncode, err.decode('utf-8', 'ignore')))
+            out = b'[ { "error": "Unhandled oletools error" } ]'
+            failed = True
+        elif cmd_tmp.returncode != 0:
+            logger.error('{} olevba exited with code {!r}; err: {!r}'.format(lid, cmd_tmp.returncode, err.decode('utf-8', 'ignore')))
+            failed = True
+
+        if failed and olefy_del_tmp_failed == 0:
+            logger.debug('{} {} FAILED: not deleting tmp file'.format(lid, tmp_file_name))
+        elif olefy_del_tmp == 1:
+            logger.debug('{} {} deleting tmp file'.format(lid, tmp_file_name))
+            os.remove(tmp_file_name)
+
+    logger.debug('{} response: {}'.format(lid, out.decode('utf-8', 'ignore')))
+    return out + b'\t\n\n\t'
+
+# Asyncio data handling, default AIO-Functions
+class AIO(asyncio.Protocol):
+    def __init__(self):
+        self.extra = bytearray()
+
+    def connection_made(self, transport):
+        global request_time
+        peer = transport.get_extra_info('peername')
+        logger.debug('{} new connection was made'.format(peer))
+        self.transport = transport
+        request_time = str(time.time())
+
+    def data_received(self, request, msgid=1):
+        peer = self.transport.get_extra_info('peername')
+        logger.debug('{} data received from new connection'.format(peer))
+        self.extra.extend(request)
+
+    def eof_received(self):
+        peer = self.transport.get_extra_info('peername')
+        olefy_protocol_err = False
+        proto_ck = self.extra[0:2000].decode('utf-8', 'ignore')
+
+        headers = proto_ck[0:proto_ck.find(olefy_protocol_sep)]
+
+        if olefy_protocol == headers[0:5]:
+            self.extra = bytearray(self.extra[len(headers)+2:len(self.extra)])
+            protocol_split(headers)
+        else:
+            olefy_protocol_err = True
+
+        if olefy_ping == headers[0:4]:
+            is_ping = True
+        else:
+            is_ping = False
+            rspamd_id = olefy_headers['Rspamd-ID'][:6] or ''
+            lid = 'Rspamd-ID' in olefy_headers and '<'+rspamd_id+'>'
+            tmp_file_name = olefy_tmp_dir+'/'+request_time+'.'+str(peer[1])+'.'+rspamd_id
+            logger.debug('{} {} choosen as tmp filename'.format(lid, tmp_file_name))
+
+        if not is_ping or olefy_loglvl == 10:
+            logger.info('{} {} bytes (stream size)'.format(lid, self.extra.__len__()))
+
+        if olefy_ping == headers[0:4]:
+            logger.debug('{} PING request'.format(peer))
+            out = b'PONG'
+        elif olefy_protocol_err == True or olefy_headers['olefy'] != 'OLEFY/1.0':
+            logger.error('{} Protocol ERROR: no OLEFY/1.0 found'.format(lid))
+            out = b'[ { "error": "Protocol error" } ]'
+        elif 'Method' in olefy_headers:
+            if olefy_headers['Method'] == 'oletools':
+                out = oletools(self.extra, tmp_file_name, lid)
+        else:
+            logger.error('Protocol ERROR: Method header not found')
+            out = b'[ { "error": "Protocol error: Method header not found" } ]'
+
+        self.transport.write(out)
+        if not is_ping or olefy_loglvl == 10:
+            logger.info('{} {} response send: {!r}'.format(lid, peer, out))
+        self.transport.close()
+
+
+# start the listeners
+loop = asyncio.get_event_loop()
+# each client connection will create a new protocol instance
+coro = loop.create_server(AIO, olefy_listen_addr, olefy_listen_port)
+server = loop.run_until_complete(coro)
+for sockets in server.sockets:
+    logger.info('serving on {}'.format(sockets.getsockname()))
+
+# XXX serve requests until KeyboardInterrupt, not needed for production
+try:
+    loop.run_forever()
+except KeyboardInterrupt:
+    pass
+
+# graceful shutdown/reload
+server.close()
+loop.run_until_complete(server.wait_closed())
+loop.close()
+logger.info('stopped serving')

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,12 +1,18 @@
-FROM php:7.4-fpm-alpine3.11
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM php:8.2-fpm-alpine3.18
+LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
-ENV APCU_PECL 5.1.18
-ENV IMAGICK_PECL 3.4.4
-# Mailparse is pulled from master branch
-#ENV MAILPARSE_PECL 3.0.2
-ENV MEMCACHED_PECL 3.1.5
-ENV REDIS_PECL 5.3.1
+# renovate: datasource=github-tags depName=krakjoe/apcu versioning=semver-coerced extractVersion=^v(?<version>.*)$
+ARG APCU_PECL_VERSION=5.1.23
+# renovate: datasource=github-tags depName=Imagick/imagick versioning=semver-coerced extractVersion=(?<version>.*)$
+ARG IMAGICK_PECL_VERSION=3.7.0
+# renovate: datasource=github-tags depName=php/pecl-mail-mailparse versioning=semver-coerced extractVersion=^v(?<version>.*)$
+ARG MAILPARSE_PECL_VERSION=3.1.6
+# renovate: datasource=github-tags depName=php-memcached-dev/php-memcached versioning=semver-coerced extractVersion=^v(?<version>.*)$
+ARG MEMCACHED_PECL_VERSION=3.2.0
+# renovate: datasource=github-tags depName=phpredis/phpredis versioning=semver-coerced extractVersion=(?<version>.*)$
+ARG REDIS_PECL_VERSION=6.0.2
+# renovate: datasource=github-tags depName=composer/composer versioning=semver-coerced extractVersion=(?<version>.*)$
+ARG COMPOSER_VERSION=2.6.6
 
 RUN apk add -U --no-cache autoconf \
   aspell-dev \
@@ -18,6 +24,7 @@ RUN apk add -U --no-cache autoconf \
   freetype-dev \
   g++ \
   git \
+  gettext \
   gettext-dev \
   gmp-dev \
   gnupg \
@@ -27,8 +34,11 @@ RUN apk add -U --no-cache autoconf \
   imagemagick-dev \
   imap-dev \
   jq \
+  libavif \
+  libavif-dev \
   libjpeg-turbo \
   libjpeg-turbo-dev \
+  libmemcached \
   libmemcached-dev \
   libpng \
   libpng-dev \
@@ -38,8 +48,11 @@ RUN apk add -U --no-cache autoconf \
   libtool \
   libwebp-dev \
   libxml2-dev \
+  libxpm \
   libxpm-dev \
+  libzip \
   libzip-dev \
+  linux-headers \
   make \
   mysql-client \
   openldap-dev \
@@ -49,22 +62,24 @@ RUN apk add -U --no-cache autoconf \
   samba-client \
   zlib-dev \
   tzdata \
-  && git clone https://github.com/php/pecl-mail-mailparse \
-  && cd pecl-mail-mailparse \
-  && pecl install package.xml \
-  && cd .. \
-  && rm -r pecl-mail-mailparse \
-  && pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} \
+  && pecl install APCu-${APCU_PECL_VERSION} \
+  && pecl install imagick-${IMAGICK_PECL_VERSION} \
+  && pecl install mailparse-${MAILPARSE_PECL_VERSION} \
+  && pecl install memcached-${MEMCACHED_PECL_VERSION} \
+  && pecl install redis-${REDIS_PECL_VERSION} \
   && docker-php-ext-enable apcu imagick memcached mailparse redis \
   && pecl clear-cache \
   && docker-php-ext-configure intl \
   && docker-php-ext-configure exif \
   && docker-php-ext-configure gd --with-freetype=/usr/include/ \  
     --with-jpeg=/usr/include/ \
-  && docker-php-ext-install -j 4 exif gd gettext intl ldap opcache pcntl pdo pdo_mysql pspell soap sockets xmlrpc zip bcmath gmp \
+    --with-webp \
+    --with-xpm \
+    --with-avif \
+  && docker-php-ext-install -j 4 exif gd gettext intl ldap opcache pcntl pdo pdo_mysql pspell soap sockets sysvsem zip bcmath gmp \
   && docker-php-ext-configure imap --with-imap --with-imap-ssl \
   && docker-php-ext-install -j 4 imap \
-  && curl --silent --show-error https://getcomposer.org/installer | php \
+  && curl --silent --show-error https://getcomposer.org/installer | php -- --version=${COMPOSER_VERSION} \
   && mv composer.phar /usr/local/bin/composer \
   && chmod +x /usr/local/bin/composer \
   && apk del --purge autoconf \
@@ -72,15 +87,22 @@ RUN apk add -U --no-cache autoconf \
     cyrus-sasl-dev \
     freetype-dev \
     g++ \
+    gettext-dev \
     icu-dev \
     imagemagick-dev \
     imap-dev \
+    libavif-dev \
     libjpeg-turbo-dev \
+    libmemcached-dev \
     libpng-dev \
     libressl-dev \
     libwebp-dev \
     libxml2-dev \
+    libxpm-dev \
+    libzip-dev \
+    linux-headers \
     make \
+    openldap-dev \
     pcre-dev \
     zlib-dev
 

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -24,6 +24,7 @@ done
 CONTAINER_ID=
 until [[ ! -z "${CONTAINER_ID}" ]] && [[ "${CONTAINER_ID}" =~ ^[[:alnum:]]*$ ]]; do
   CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], project: .Config.Labels[\"com.docker.compose.project\"], id: .Id}" 2> /dev/null | jq -rc "select( .name | tostring | contains(\"mysql-mailcow\")) | select( .project | tostring | contains(\"${COMPOSE_PROJECT_NAME,,}\")) | .id" 2> /dev/null)
+  sleep 2
 done
 echo "MySQL @ ${CONTAINER_ID}"
 SQL_LOOP_C=0
@@ -89,6 +90,15 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
     ${REDIS_CMDLINE} --raw SET Q_MAX_AGE 365
   fi
 
+  # Set default password policy - if unset
+  if [[ -z $(${REDIS_CMDLINE} --raw HGET PASSWD_POLICY length) ]]; then
+    ${REDIS_CMDLINE} --raw HSET PASSWD_POLICY length 6
+    ${REDIS_CMDLINE} --raw HSET PASSWD_POLICY chars 0
+    ${REDIS_CMDLINE} --raw HSET PASSWD_POLICY special_chars 0
+    ${REDIS_CMDLINE} --raw HSET PASSWD_POLICY lowerupper 0
+    ${REDIS_CMDLINE} --raw HSET PASSWD_POLICY numbers 0
+  fi
+
   # Trigger db init
   echo "Running DB init..."
   php -c /usr/local/etc/php -f /web/inc/init_db.inc.php
@@ -162,6 +172,24 @@ BEGIN
 END;
 //
 DELIMITER ;
+DROP EVENT IF EXISTS clean_sasl_log;
+DELIMITER //
+CREATE EVENT clean_sasl_log
+ON SCHEDULE EVERY 1 DAY DO
+BEGIN
+  DELETE sasl_log.* FROM sasl_log
+    LEFT JOIN (
+      SELECT username, service, MAX(datetime) AS lastdate
+      FROM sasl_log
+      GROUP BY username, service
+    ) AS last ON sasl_log.username = last.username AND sasl_log.service = last.service
+    WHERE datetime < DATE_SUB(NOW(), INTERVAL 31 DAY) AND datetime < lastdate;
+  DELETE FROM sasl_log
+    WHERE username NOT IN (SELECT username FROM mailbox) AND
+    datetime < DATE_SUB(NOW(), INTERVAL 31 DAY);
+END;
+//
+DELIMITER ;
 EOF
 fi
 
@@ -171,6 +199,22 @@ fi
 # Fix permissions for global filters
 chown -R 82:82 /global_sieve/*
 
+# Fix permissions on twig cache folder
+chown -R 82:82 /web/templates/cache
+# Clear cache
+find /web/templates/cache/* -not -name '.gitkeep' -delete
+
+# list client ca of all domains for
+CA_LIST="/etc/nginx/conf.d/client_cas.crt"
+# Clear the output file
+> "$CA_LIST"
+# Execute the query and append each value to the output file
+mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT ssl_client_ca FROM domain;" | while read -r ca; do
+    echo "$ca" >> "$CA_LIST"
+done
+echo "SSL client CAs have been appended to $CA_LIST"
+
+
 # Run hooks
 for file in /hooks/*; do
   if [ -x "${file}" ]; then

data/Dockerfiles/postfix/Dockerfile

@@ -1,5 +1,5 @@
-FROM debian:buster-slim
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM debian:bullseye-slim
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
 ENV LC_ALL C
@@ -17,10 +17,10 @@ RUN groupadd -g 102 postfix \
 	ca-certificates \
 	curl \
 	dirmngr \
-  dnsutils \
+  	dnsutils \
 	gnupg \
 	libsasl2-modules \
-  mariadb-client \
+  	mariadb-client \
 	perl \
 	postfix \
 	postfix-mysql \
@@ -32,7 +32,7 @@ RUN groupadd -g 102 postfix \
 	syslog-ng \
 	syslog-ng-core \
 	syslog-ng-mod-redis \
-  tzdata \
+  	tzdata \
 	&& rm -rf /var/lib/apt/lists/* \
 	&& touch /etc/default/locale \
   && printf '#!/bin/bash\n/usr/sbin/postconf -c /opt/postfix/conf "$@"' > /usr/local/sbin/postconf \
@@ -45,7 +45,6 @@ COPY postfix.sh /opt/postfix.sh
 COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
 COPY whitelist_forwardinghosts.sh /usr/local/bin/whitelist_forwardinghosts.sh
-COPY smtpd_last_login.sh /usr/local/bin/smtpd_last_login.sh
 COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
@@ -53,7 +52,6 @@ RUN chmod +x /opt/postfix.sh \
   /usr/local/bin/rspamd-pipe-ham \
   /usr/local/bin/rspamd-pipe-spam \
   /usr/local/bin/whitelist_forwardinghosts.sh \
-  /usr/local/bin/smtpd_last_login.sh \
   /usr/local/sbin/stop-supervisor.sh
 RUN rm -rf /tmp/* /var/tmp/*
 

data/Dockerfiles/postfix/postfix.sh

@@ -10,7 +10,7 @@ while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${D
   sleep 2
 done
 
-until dig +short mailcow.email @unbound > /dev/null; do
+until dig +short mailcow.email > /dev/null; do
   echo "Waiting for DNS..."
   sleep 1
 done
@@ -25,17 +25,21 @@ EOF
 newaliases;
 
 # create sni configuration
-echo -n "" > /opt/postfix/conf/sni.map;
-for cert_dir in /etc/ssl/mail/*/ ; do
-  if [[ ! -f ${cert_dir}domains ]] || [[ ! -f ${cert_dir}cert.pem ]] || [[ ! -f ${cert_dir}key.pem ]]; then
-    continue;
-  fi
-  IFS=" " read -r -a domains <<< "$(cat "${cert_dir}domains")"
-  for domain in "${domains[@]}"; do
-    echo -n "${domain} ${cert_dir}key.pem ${cert_dir}cert.pem" >> /opt/postfix/conf/sni.map;
-    echo "" >> /opt/postfix/conf/sni.map;
+if [[ "${SKIP_LETS_ENCRYPT}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  echo -n "" > /opt/postfix/conf/sni.map
+else
+  echo -n "" > /opt/postfix/conf/sni.map;
+  for cert_dir in /etc/ssl/mail/*/ ; do
+    if [[ ! -f ${cert_dir}domains ]] || [[ ! -f ${cert_dir}cert.pem ]] || [[ ! -f ${cert_dir}key.pem ]]; then
+      continue;
+    fi
+    IFS=" " read -r -a domains <<< "$(cat "${cert_dir}domains")"
+    for domain in "${domains[@]}"; do
+      echo -n "${domain} ${cert_dir}key.pem ${cert_dir}cert.pem" >> /opt/postfix/conf/sni.map;
+      echo "" >> /opt/postfix/conf/sni.map;
+    done
   done
-done
+fi
 postmap -F hash:/opt/postfix/conf/sni.map;
 
 cat <<EOF > /opt/postfix/conf/sql/mysql_relay_ne.cf
@@ -121,16 +125,31 @@ query = SELECT GROUP_CONCAT(transport SEPARATOR '') AS transport_maps
         AND mailbox.active = '1'
     ), 'smtp_enforced_tls:', 'smtp:') AS 'transport'
     UNION ALL
-    SELECT hostname AS transport FROM relayhosts
+    SELECT COALESCE(
+      (SELECT hostname FROM relayhosts
+      LEFT OUTER JOIN mailbox ON JSON_UNQUOTE(JSON_VALUE(mailbox.attributes, '$.relayhost')) = relayhosts.id
+        WHERE relayhosts.active = '1'
+          AND (
+            mailbox.username IN (SELECT alias.goto from alias
+              JOIN mailbox ON mailbox.username = alias.goto
+                WHERE alias.active = '1'
+                  AND alias.address = '%s'
+                  AND alias.address NOT LIKE '@%%'
+            )
+          )
+      ),
+      (SELECT hostname FROM relayhosts
       LEFT OUTER JOIN domain ON domain.relayhost = relayhosts.id
         WHERE relayhosts.active = '1'
-          AND domain = '%d'
-          OR domain IN (
-            SELECT target_domain FROM alias_domain
-              WHERE alias_domain = '%d'
+          AND (domain.domain = '%d'
+            OR domain.domain IN (
+              SELECT target_domain FROM alias_domain
+                WHERE alias_domain = '%d'
+            )
           )
-  )
-  AS transport_view;
+      )
+    )
+  ) AS transport_view;
 EOF
 
 cat <<EOF > /opt/postfix/conf/sql/mysql_transport_maps.cf
@@ -162,11 +181,31 @@ hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM relayhosts
   WHERE id IN (
-    SELECT relayhost FROM domain
-      WHERE CONCAT('@', domain) = '%s'
-      OR domain IN (
-        SELECT target_domain FROM alias_domain WHERE CONCAT('@', alias_domain) =  '%s'
+    SELECT COALESCE(
+      (SELECT id FROM relayhosts
+      LEFT OUTER JOIN domain ON domain.relayhost = relayhosts.id
+      WHERE relayhosts.active = '1'
+        AND (domain.domain = '%d'
+          OR domain.domain IN (
+            SELECT target_domain FROM alias_domain
+            WHERE alias_domain = '%d'
+          )
+        )
+      ),
+      (SELECT id FROM relayhosts
+      LEFT OUTER JOIN mailbox ON JSON_UNQUOTE(JSON_VALUE(mailbox.attributes, '$.relayhost')) = relayhosts.id
+      WHERE relayhosts.active = '1'
+        AND (
+          mailbox.username IN (
+            SELECT alias.goto from alias
+              JOIN mailbox ON mailbox.username = alias.goto
+                WHERE alias.active = '1'
+                  AND alias.address = '%s'
+                  AND alias.address NOT LIKE '@%%'
+          )
+        )
       )
+    )
   )
   AND active = '1'
   AND username != '';
@@ -284,7 +323,19 @@ hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 # First select queries domain and alias_domain to determine if domains are active.
 query = SELECT goto FROM alias
-  WHERE address='%s'
+  WHERE id IN (
+      SELECT COALESCE (
+        (
+          SELECT id FROM alias
+            WHERE address='%s'
+            AND (active='1' OR active='2')
+        ), (
+          SELECT id FROM alias
+            WHERE address='@%d'
+            AND (active='1' OR active='2')
+        )
+      )
+    )
     AND active='1'
     AND (domain IN
       (SELECT domain FROM domain
@@ -315,17 +366,20 @@ query = SELECT goto FROM alias
     WHERE alias_domain.alias_domain = '%d'
       AND mailbox.username = CONCAT('%u','@',alias_domain.target_domain)
       AND (mailbox.active = '1' OR mailbox.active ='2')
-      AND alias_domain.active='1'
+      AND alias_domain.active='1';
 EOF
 
-# Reject sasl usernames with smtp disabled
-cat <<EOF > /opt/postfix/conf/sql/mysql_sasl_access_maps.cf
+# MX based routing
+cat <<EOF > /opt/postfix/conf/sql/mysql_mbr_access_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}
 password = ${DBPASS}
 hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
-query = SELECT 'REJECT' FROM mailbox WHERE username = '%u' AND JSON_UNQUOTE(JSON_VALUE(attributes, '$.smtp_access')) = '0';
+query = SELECT CONCAT('FILTER smtp_via_transport_maps:', nexthop) as transport FROM transports
+  WHERE '%s' REGEXP destination
+    AND active='1'
+    AND is_mx_based='1';
 EOF
 
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_spamalias_maps.cf
@@ -339,11 +393,102 @@ query = SELECT goto FROM spamalias
     AND validity >= UNIX_TIMESTAMP()
 EOF
 
-sed -i '/User overrides/q' /opt/postfix/conf/main.cf
-echo >> /opt/postfix/conf/main.cf
-if [ -f /opt/postfix/conf/extra.cf ]; then
-  cat /opt/postfix/conf/extra.cf >> /opt/postfix/conf/main.cf
+if [ ! -f /opt/postfix/conf/dns_blocklists.cf ]; then
+  cat <<EOF > /opt/postfix/conf/dns_blocklists.cf
+# This file can be edited. 
+# Delete this file and restart postfix container to revert any changes.
+postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
+  hostkarma.junkemailfilter.com=127.0.0.1*-2
+  list.dnswl.org=127.0.[0..255].0*-2
+  list.dnswl.org=127.0.[0..255].1*-4
+  list.dnswl.org=127.0.[0..255].2*-6
+  list.dnswl.org=127.0.[0..255].3*-8
+  ix.dnsbl.manitu.net*2
+  bl.spamcop.net*2
+  bl.suomispam.net*2
+  hostkarma.junkemailfilter.com=127.0.0.2*3
+  hostkarma.junkemailfilter.com=127.0.0.4*2
+  hostkarma.junkemailfilter.com=127.0.1.2*1
+  backscatter.spameatingmonkey.net*2
+  bl.ipv6.spameatingmonkey.net*2
+  bl.spameatingmonkey.net*2
+  b.barracudacentral.org=127.0.0.2*7
+  bl.mailspike.net=127.0.0.2*5
+  bl.mailspike.net=127.0.0.[10;11;12]*4
+  dnsbl.sorbs.net=127.0.0.10*8
+  dnsbl.sorbs.net=127.0.0.5*6
+  dnsbl.sorbs.net=127.0.0.7*3
+  dnsbl.sorbs.net=127.0.0.8*2
+  dnsbl.sorbs.net=127.0.0.6*2
+  dnsbl.sorbs.net=127.0.0.9*2
+EOF
 fi
+DNSBL_CONFIG=$(grep -v '^#' /opt/postfix/conf/dns_blocklists.cf | grep '\S')
+
+if [ ! -z "$DNSBL_CONFIG" ]; then
+  echo -e "\e[33mChecking if ASN for your IP is listed for Spamhaus Bad ASN List...\e[0m"
+  if [ -n "$SPAMHAUS_DQS_KEY" ]; then
+    echo -e "\e[32mDetected SPAMHAUS_DQS_KEY variable from mailcow.conf...\e[0m"
+    echo -e "\e[33mUsing DQS Blocklists from Spamhaus!\e[0m"
+    SPAMHAUS_DNSBL_CONFIG=$(cat <<EOF
+  ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.net=127.0.0.[4..7]*6
+  ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.net=127.0.0.[10;11]*8
+  ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.net=127.0.0.3*4
+  ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.net=127.0.0.2*3
+postscreen_dnsbl_reply_map = texthash:/opt/postfix/conf/dnsbl_reply.map
+EOF
+
+  cat <<EOF > /opt/postfix/conf/dnsbl_reply.map
+# Autogenerated by mailcow, using Spamhaus DQS reply domains
+${SPAMHAUS_DQS_KEY}.sbl.dq.spamhaus.net     sbl.spamhaus.org
+${SPAMHAUS_DQS_KEY}.xbl.dq.spamhaus.net     xbl.spamhaus.org
+${SPAMHAUS_DQS_KEY}.pbl.dq.spamhaus.net     pbl.spamhaus.org
+${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.net     zen.spamhaus.org
+${SPAMHAUS_DQS_KEY}.dbl.dq.spamhaus.net     dbl.spamhaus.org
+${SPAMHAUS_DQS_KEY}.zrd.dq.spamhaus.net     zrd.spamhaus.org
+EOF
+    )
+  else
+    if [ -f "/opt/postfix/conf/dnsbl_reply.map" ]; then
+      rm /opt/postfix/conf/dnsbl_reply.map
+    fi
+    response=$(curl --connect-timeout 15 --max-time 30 -s -o /dev/null -w "%{http_code}" "https://asn-check.mailcow.email")
+    if [ "$response" -eq 503 ]; then
+      echo -e "\e[31mThe AS of your IP is listed as a banned AS from Spamhaus!\e[0m"
+      echo -e "\e[33mNo SPAMHAUS_DQS_KEY found... Skipping Spamhaus blocklists entirely!\e[0m"
+      SPAMHAUS_DNSBL_CONFIG=""
+    elif [ "$response" -eq 200 ]; then
+      echo -e "\e[32mThe AS of your IP is NOT listed as a banned AS from Spamhaus!\e[0m"
+      echo -e "\e[33mUsing the open Spamhaus blocklists.\e[0m"
+      SPAMHAUS_DNSBL_CONFIG=$(cat <<EOF
+  zen.spamhaus.org=127.0.0.[10;11]*8
+  zen.spamhaus.org=127.0.0.[4..7]*6
+  zen.spamhaus.org=127.0.0.3*4
+  zen.spamhaus.org=127.0.0.2*3
+EOF
+      )
+
+    else
+      echo -e "\e[31mWe couldn't determine your AS... (maybe DNS/Network issue?) Response Code: $response\e[0m"
+      echo -e "\e[33mDeactivating Spamhaus DNS Blocklists to be on the safe site!\e[0m"
+      SPAMHAUS_DNSBL_CONFIG=""
+    fi
+  fi
+fi
+
+# Reset main.cf
+sed -i '/Overrides/q' /opt/postfix/conf/main.cf
+echo >> /opt/postfix/conf/main.cf
+# Append postscreen dnsbl sites to main.cf
+if [ ! -z "$DNSBL_CONFIG" ]; then
+  echo -e "${DNSBL_CONFIG}\n${SPAMHAUS_DNSBL_CONFIG}" >> /opt/postfix/conf/main.cf
+fi
+# Append user overrides
+echo -e "\n# User Overrides" >> /opt/postfix/conf/main.cf
+touch /opt/postfix/conf/extra.cf
+sed -i '/\$myhostname/! { /myhostname/d }' /opt/postfix/conf/extra.cf
+echo -e "myhostname = ${MAILCOW_HOSTNAME}\n$(cat /opt/postfix/conf/extra.cf)" > /opt/postfix/conf/extra.cf
+cat /opt/postfix/conf/extra.cf >> /opt/postfix/conf/main.cf
 
 if [ ! -f /opt/postfix/conf/custom_transport.pcre ]; then
   echo "Creating dummy custom_transport.pcre"
@@ -352,12 +497,15 @@ fi
 
 if [[ ! -f /opt/postfix/conf/custom_postscreen_whitelist.cidr ]]; then
   echo "Creating dummy custom_postscreen_whitelist.cidr"
-  echo '# Autogenerated by mailcow' > /opt/postfix/conf/custom_postscreen_whitelist.cidr
+  cat <<EOF > /opt/postfix/conf/custom_postscreen_whitelist.cidr
+# Autogenerated by mailcow
+# Rules are evaluated in the order as specified.
+# Blacklist 192.168.* except 192.168.0.1.
+# 192.168.0.1          permit
+# 192.168.0.0/16       reject
+EOF
 fi
 
-# Fix SMTP last login on slaves
-sed -i "s/__REDIS_SLAVEOF_IP__/${REDIS_SLAVEOF_IP}/g" /usr/local/bin/smtpd_last_login.sh
-
 # Fix Postfix permissions
 chown -R root:postfix /opt/postfix/conf/sql/ /opt/postfix/conf/custom_transport.pcre
 chmod 640 /opt/postfix/conf/sql/*.cf /opt/postfix/conf/custom_transport.pcre

data/Dockerfiles/postfix/smtpd_last_login.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-REDIS_SLAVEOF_IP=__REDIS_SLAVEOF_IP__
-
-# Do not attempt to write to slave
-if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
-  REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT}"
-else
-  REDIS_CMDLINE="redis-cli -h redis -p 6379"
-fi
-
-while read QUERY; do
-  QUERY=($QUERY)
-  # If nothing matched, end here - Postfix last line will be empty
-  if [[ -z "$(echo ${QUERY[0]} | tr -d '\040\011\012\015')" ]]; then
-    echo -ne "action=dunno\n\n"
-  # We found a username, log and return
-  elif [[ "${QUERY[0]}" =~ sasl_username ]]; then
-    ${REDIS_CMDLINE} SET "last-login/smtp/$(echo ${QUERY[0]#sasl_username=})" "$(date +%s)"
-    echo -ne "action=dunno\n\n"
-  fi
-done

data/Dockerfiles/postfix/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.19
+@version: 3.28
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -28,7 +28,7 @@ destination d_redis_f2b_channel {
     host("`REDIS_SLAVEOF_IP`")
     persist-name("redis2")
     port(`REDIS_SLAVEOF_PORT`)
-    command("PUBLISH" "F2B_CHANNEL" "$MESSAGE")
+    command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
   );
 };
 filter f_mail { facility(mail); };

data/Dockerfiles/postfix/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.19
+@version: 3.28
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -28,7 +28,7 @@ destination d_redis_f2b_channel {
     host("redis-mailcow")
     persist-name("redis2")
     port(6379)
-    command("PUBLISH" "F2B_CHANNEL" "$MESSAGE")
+    command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
   );
 };
 filter f_mail { facility(mail); };

data/Dockerfiles/rspamd/Dockerfile

@@ -1,29 +1,32 @@
-FROM debian:buster-slim
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM debian:bookworm-slim
+LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG CODENAME=buster
+ARG CODENAME=bookworm
 ENV LC_ALL C
 
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y --no-install-recommends \
   tzdata \
   ca-certificates \
   gnupg2 \
   apt-transport-https \
   dnsutils \
-  netcat \
+  netcat-traditional \
   && apt-key adv --fetch-keys https://rspamd.com/apt-stable/gpg.key \
-  && echo "deb [arch=amd64] https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \
+  && echo "deb https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \
   && apt-get update \
-  && apt-get --no-install-recommends -y install rspamd redis-tools \
+  && apt-get --no-install-recommends -y install rspamd redis-tools procps nano \
   && rm -rf /var/lib/apt/lists/* \
   && apt-get autoremove --purge \
   && apt-get clean \
   && mkdir -p /run/rspamd \
-  && chown _rspamd:_rspamd /run/rspamd
+  && chown _rspamd:_rspamd /run/rspamd \
+  && echo 'alias ll="ls -la --color"' >> ~/.bashrc \
+  && sed -i 's/#analysis_keyword_table > 0/analysis_cat_table.macro_exist == "M"/g' /usr/share/rspamd/lualib/lua_scanners/oletools.lua
 
 COPY settings.conf /etc/rspamd/settings.conf
 COPY metadata_exporter.lua /usr/share/rspamd/plugins/metadata_exporter.lua
+COPY set_worker_password.sh /set_worker_password.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]

data/Dockerfiles/rspamd/docker-entrypoint.sh

@@ -79,6 +79,9 @@ EOF
   redis-cli -h redis-mailcow SLAVEOF NO ONE
 fi
 
+# Provide additional lua modules
+ln -s /usr/lib/$(uname -m)-linux-gnu/liblua5.1-cjson.so.0.0.0 /usr/lib/rspamd/cjson.so
+
 chown -R _rspamd:_rspamd /var/lib/rspamd \
   /etc/rspamd/local.d \
   /etc/rspamd/override.d \
@@ -104,7 +107,8 @@ touch /etc/rspamd/custom/global_mime_from_blacklist.map \
   /etc/rspamd/custom/bad_words.map \
   /etc/rspamd/custom/bad_asn.map \
   /etc/rspamd/custom/bad_words_de.map \
-  /etc/rspamd/custom/bulk_header.map
+  /etc/rspamd/custom/bulk_header.map \
+  /etc/rspamd/custom/bad_header.map
 
 # www-data (82) group needs to write to these files
 chown _rspamd:_rspamd /etc/rspamd/custom/

data/Dockerfiles/rspamd/set_worker_password.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+password_file='/etc/rspamd/override.d/worker-controller-password.inc'
+password_hash=`/usr/bin/rspamadm pw -e -p $1`
+
+echo 'enable_password = "'$password_hash'";' > $password_file
+
+if grep -q "$password_hash" "$password_file"; then
+    echo "OK"
+else
+    echo "ERROR"
+fi

data/Dockerfiles/sogo/Dockerfile

@@ -1,17 +1,18 @@
-FROM debian:buster-slim
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+FROM debian:bookworm-slim
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SOGO_DEBIAN_REPOSITORY=http://packages.inverse.ca/SOGo/nightly/5/debian/
+ARG DEBIAN_VERSION=bookworm
+ARG SOGO_DEBIAN_REPOSITORY=http://www.axis.cz/linux/debian
+# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
+ARG GOSU_VERSION=1.17
 ENV LC_ALL C
-ENV GOSU_VERSION 1.11
 
 # Prerequisites
 RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
   && apt-get update && apt-get install -y --no-install-recommends \
   apt-transport-https \
   ca-certificates \
-  cron \
   gettext \
   gnupg \
   mariadb-client \
@@ -21,7 +22,7 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
   syslog-ng-core \
   syslog-ng-mod-redis \
   dirmngr \
-  netcat \
+  netcat-traditional \
   psmisc \
   wget \
   patch \
@@ -31,8 +32,8 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
   && gosu nobody true \
   && mkdir /usr/share/doc/sogo \
   && touch /usr/share/doc/sogo/empty.sh \
-  && apt-key adv --keyserver keyserver.ubuntu.com --recv-key 0x810273C4 \
-  && echo "deb ${SOGO_DEBIAN_REPOSITORY} buster buster" > /etc/apt/sources.list.d/sogo.list \
+  && apt-key adv --keyserver keys.openpgp.org --recv-key 74FFC6D72B925A34B5D356BDF8A27B36A6E2EAE9 \
+  && echo "deb [trusted=yes] ${SOGO_DEBIAN_REPOSITORY} ${DEBIAN_VERSION} sogo-v5" > /etc/apt/sources.list.d/sogo.list \
   && apt-get update && apt-get install -y --no-install-recommends \
     sogo \
     sogo-activesync \
@@ -53,4 +54,4 @@ RUN chmod +x /bootstrap-sogo.sh \
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
-CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
+CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -24,114 +24,6 @@ while [[ "${DBV_NOW}" != "${DBV_NEW}" ]]; do
 done
 echo "DB schema is ${DBV_NOW}"
 
-# Recreate view
-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-  echo "We are master, preparing sogo_view..."
-  mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP VIEW IF EXISTS sogo_view"
-  while [[ ${VIEW_OK} != 'OK' ]]; do
-    mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-CREATE VIEW sogo_view (c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, ext_acl, kind, multiple_bookings) AS 
-SELECT
-   mailbox.username,
-   mailbox.domain,
-   mailbox.username,
-   IF(JSON_UNQUOTE(JSON_VALUE(attributes, '$.force_pw_update')) = '0', IF(JSON_UNQUOTE(JSON_VALUE(attributes, '$.sogo_access')) = 1, password, '{SSHA256}A123A123A321A321A321B321B321B123B123B321B432F123E321123123321321'), '{SSHA256}A123A123A321A321A321B321B321B123B123B321B432F123E321123123321321'),
-   mailbox.name,
-   mailbox.username,
-   IFNULL(GROUP_CONCAT(ga.aliases ORDER BY ga.aliases SEPARATOR ' '), ''),
-   IFNULL(gda.ad_alias, ''),
-   IFNULL(external_acl.send_as_acl, ''),
-   mailbox.kind,
-   mailbox.multiple_bookings
-FROM
-   mailbox
-   LEFT OUTER JOIN
-      grouped_mail_aliases ga
-      ON ga.username REGEXP CONCAT('(^|,)', mailbox.username, '($|,)')
-   LEFT OUTER JOIN
-      grouped_domain_alias_address gda
-      ON gda.username = mailbox.username
-   LEFT OUTER JOIN
-      grouped_sender_acl_external external_acl
-      ON external_acl.username = mailbox.username
-WHERE
-   mailbox.active = '1'
-GROUP BY
-   mailbox.username;
-EOF
-    if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'sogo_view'") ]]; then
-      VIEW_OK=OK
-    else
-      echo "Will retry to setup SOGo view in 3s..."
-      sleep 3
-    fi
-  done
-else
-  while [[ ${VIEW_OK} != 'OK' ]]; do
-    if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'sogo_view'") ]]; then
-      VIEW_OK=OK
-    else
-      echo "Waiting for SOGo view to be created by master..."
-      sleep 3
-    fi
-  done
-fi
-
-# Wait for static view table if missing after update and update content
-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-  echo "We are master, preparing _sogo_static_view..."
-  while [[ ${STATIC_VIEW_OK} != 'OK' ]]; do
-    if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '_sogo_static_view'") ]]; then
-      STATIC_VIEW_OK=OK
-      echo "Updating _sogo_static_view content..."
-      mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "REPLACE INTO _sogo_static_view (c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, ext_acl, kind, multiple_bookings) SELECT c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, ext_acl, kind, multiple_bookings from sogo_view;"
-      mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "DELETE FROM _sogo_static_view WHERE c_uid NOT IN (SELECT username FROM mailbox WHERE active = '1')"
-    else
-      echo "Waiting for database initialization..."
-      sleep 3
-    fi
-  done
-else
-  while [[ ${STATIC_VIEW_OK} != 'OK' ]]; do
-    if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '_sogo_static_view'") ]]; then
-      STATIC_VIEW_OK=OK
-    else
-      echo "Waiting for database initialization by master..."
-      sleep 3
-    fi
-  done
-fi
-
-
-# Recreate password update trigger
-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-  echo "We are master, preparing update trigger..."
-  mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP TRIGGER IF EXISTS sogo_update_password"
-  while [[ ${TRIGGER_OK} != 'OK' ]]; do
-  mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-DELIMITER -
-CREATE TRIGGER sogo_update_password AFTER UPDATE ON _sogo_static_view
-FOR EACH ROW
-BEGIN
-UPDATE mailbox SET password = NEW.c_password WHERE NEW.c_uid = username;
-END;
--
-DELIMITER ;
-EOF
-    if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_NAME = 'sogo_update_password'") ]]; then
-      TRIGGER_OK=OK
-    else
-      echo "Will retry to setup SOGo password update trigger in 3s"
-      sleep 3
-    fi
-  done
-fi
-
-if [[ "${ALLOW_ADMIN_EMAIL_LOGIN}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-  TRUST_PROXY="YES"
-else
-  TRUST_PROXY="NO"
-fi
 # cat /dev/urandom seems to hang here occasionally and is not recommended anyway, better use openssl
 RAND_PASS=$(openssl rand -base64 16 | tr -dc _A-Z-a-z-0-9)
 
@@ -146,8 +38,12 @@ cat <<EOF > /var/lib/sogo/GNUstep/Defaults/sogod.plist
     <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_acl</string>
     <key>SOGoIMAPServer</key>
     <string>imap://${IPV4_NETWORK}.250:143/?TLS=YES&amp;tlsVerifyMode=none</string>
+    <key>SOGoSieveServer</key>
+    <string>sieve://${IPV4_NETWORK}.250:4190/?TLS=YES&amp;tlsVerifyMode=none</string>
+    <key>SOGoSMTPServer</key>
+    <string>smtp://${IPV4_NETWORK}.253:588/?TLS=YES&amp;tlsVerifyMode=none</string>
     <key>SOGoTrustProxyAuthentication</key>
-    <string>${TRUST_PROXY}</string>
+    <string>YES</string>
     <key>SOGoEncryptionKey</key>
     <string>${RAND_PASS}</string>
     <key>OCSCacheFolderURL</key>
@@ -203,7 +99,7 @@ while read -r line gal
                     <key>type</key>
                     <string>sql</string>
                     <key>userPasswordAlgorithm</key>
-                    <string>ssha256</string>
+                    <string>${MAILCOW_PASS_SCHEME}</string>
                     <key>prependPasswordScheme</key>
                     <string>YES</string>
                     <key>viewURL</key>
@@ -248,14 +144,4 @@ rsync -a /usr/lib/GNUstep/SOGo/. /sogo_web/
 # Chown backup path
 chown -R sogo:sogo /sogo_backup
 
-# Creating cronjobs
-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-  echo "* * * * *   sogo   /usr/sbin/sogo-ealarms-notify -p /etc/sogo/sieve.creds 2>/dev/null" > /etc/cron.d/sogo
-  echo "* * * * *   sogo   /usr/sbin/sogo-tool expire-sessions ${SOGO_EXPIRE_SESSION}" >> /etc/cron.d/sogo
-  echo "0 0 * * *   sogo   /usr/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds" >> /etc/cron.d/sogo
-  echo "0 2 * * *   sogo   /usr/sbin/sogo-tool backup /sogo_backup ALL" >> /etc/cron.d/sogo
-else
-  rm /etc/cron.d/sogo
-fi
-
 exec gosu sogo /usr/sbin/sogod

data/Dockerfiles/sogo/docker-entrypoint.sh

@@ -10,4 +10,12 @@ if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
   cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
 fi
 
+# Run hooks
+for file in /hooks/*; do
+  if [ -x "${file}" ]; then
+    echo "Running hook ${file}"
+    "${file}"
+  fi
+done
+
 exec "$@"

data/Dockerfiles/sogo/supervisord.conf

@@ -11,18 +11,13 @@ stderr_logfile_maxbytes=0
 autostart=true
 priority=1
 
-[program:cron]
-command=/usr/sbin/cron -f
-autorestart=true
-priority=2
-
 [program:bootstrap-sogo]
 command=/bootstrap-sogo.sh
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
-priority=3
+priority=2
 startretries=10
 autorestart=true
 stopwaitsecs=120

data/Dockerfiles/sogo/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.19
+@version: 3.28
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -30,7 +30,7 @@ destination d_redis_f2b_channel {
     host("`REDIS_SLAVEOF_IP`")
     persist-name("redis2")
     port(`REDIS_SLAVEOF_PORT`)
-    command("PUBLISH" "F2B_CHANNEL" "$MESSAGE")
+    command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
   );
 };
 log {

data/Dockerfiles/sogo/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.19
+@version: 3.28
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -30,7 +30,7 @@ destination d_redis_f2b_channel {
     host("redis-mailcow")
     persist-name("redis2")
     port(6379)
-    command("PUBLISH" "F2B_CHANNEL" "$MESSAGE")
+    command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
   );
 };
 log {

data/Dockerfiles/solr/Dockerfile

@@ -1,8 +1,10 @@
 FROM solr:7.7-slim
+LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 USER root
 
-ENV GOSU_VERSION 1.11
+# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=(?<version>.*)$
+ARG GOSU_VERSION=1.17
 
 COPY solr.sh /
 COPY solr-config-7.7.0.xml /
@@ -16,10 +18,15 @@ RUN dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
   tzdata \
   curl \
   bash \
+  zip \
   && apt-get autoclean \
   && rm -rf /var/lib/apt/lists/* \
   && chmod +x /solr.sh \
   && sync \
   && bash /solr.sh --bootstrap
+  
+RUN zip -q -d /opt/solr/server/lib/ext/log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class
+
+RUN apt remove zip -y  
 
 CMD ["/solr.sh"]

data/Dockerfiles/unbound/Dockerfile

@@ -1,9 +1,11 @@
-FROM alpine:3.11
+FROM alpine:3.18
 
-LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
 
 RUN apk add --update --no-cache \
 	curl \
+	bind-tools \
+	netcat-openbsd \
 	unbound \
 	bash \
 	openssl \
@@ -18,6 +20,11 @@ EXPOSE 53/udp 53/tcp
 
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
+# healthcheck (nslookup)
+COPY healthcheck.sh /healthcheck.sh
+RUN chmod +x /healthcheck.sh
+HEALTHCHECK --interval=5s --timeout=30s CMD [ "/healthcheck.sh" ]
+
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
 CMD ["/usr/sbin/unbound"]

data/Dockerfiles/unbound/healthcheck.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+# Skip Unbound (DNS Resolver) Healthchecks (NOT Recommended!)
+if [[ "${SKIP_UNBOUND_HEALTHCHECK}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    SKIP_UNBOUND_HEALTHCHECK=y
+fi
+
+# Declare log function for logfile inside container
+function log_to_file() {
+    echo "$(date +"%Y-%m-%d %H:%M:%S"): $1" > /var/log/healthcheck.log
+}
+
+# General Ping function to check general pingability
+function check_ping() {
+    declare -a ipstoping=("1.1.1.1" "8.8.8.8" "9.9.9.9")
+
+    for ip in "${ipstoping[@]}" ; do
+            ping -q -c 3 -w 5 "$ip"
+            if [ $? -ne 0 ]; then
+                log_to_file "Healthcheck: Couldn't ping $ip for 5 seconds... Gave up!"
+                log_to_file "Please check your internet connection or firewall rules to fix this error, because a simple ping test should always go through from the unbound container!"
+                return 1
+            fi
+    done
+
+    log_to_file "Healthcheck: Ping Checks WORKING properly!"
+    return 0
+}
+
+# General DNS Resolve Check against Unbound Resolver himself
+function check_dns() {
+    declare -a domains=("mailcow.email" "github.com" "hub.docker.com")
+
+    for domain in "${domains[@]}" ; do
+        for ((i=1; i<=3; i++)); do
+            dig +short +timeout=2 +tries=1 "$domain" @127.0.0.1 > /dev/null
+        if [ $? -ne 0 ]; then
+            log_to_file "Healthcheck: DNS Resolution Failed on $i attempt! Trying again..."
+            if [ $i -eq 3 ]; then
+                log_to_file "Healthcheck: DNS Resolution not possible after $i attempts... Gave up!"
+                log_to_file "Maybe check your outbound firewall, as it needs to resolve DNS over TCP AND UDP!"
+                return 1
+            fi
+        fi
+        done
+    done
+
+    log_to_file "Healthcheck: DNS Resolver WORKING properly!"
+    return 0
+    
+}
+
+# Simple Netcat Check to connect to common webports
+function check_netcat() {
+    declare -a domains=("mailcow.email" "github.com" "hub.docker.com")
+    declare -a ports=("80" "443")
+
+    for domain in "${domains[@]}" ; do
+        for port in "${ports[@]}" ; do
+            nc -z -w 2 $domain $port
+            if [ $? -ne 0 ]; then
+                log_to_file "Healthcheck: Could not reach $domain on Port $port... Gave up!"
+                log_to_file "Please check your internet connection or firewall rules to fix this error."
+                return 1
+            fi
+        done
+    done
+
+    log_to_file "Healthcheck: Netcat Checks WORKING properly!"
+    return 0
+
+}
+
+if [[ ${SKIP_UNBOUND_HEALTHCHECK} == "y" ]]; then
+    log_to_file "Healthcheck: ALL CHECKS WERE SKIPPED! Unbound is healthy!"
+    exit 0
+fi
+
+# run checks, if check is not returning 0 (return value if check is ok), healthcheck will exit with 1 (marked in docker as unhealthy)
+check_ping
+
+if [ $? -ne 0 ]; then
+    exit 1
+fi
+
+check_dns
+
+if [ $? -ne 0 ]; then
+    exit 1
+fi
+
+check_netcat
+
+if [ $? -ne 0 ]; then
+    exit 1
+fi
+
+log_to_file "Healthcheck: ALL CHECKS WERE SUCCESSFUL! Unbound is healthy!"
+exit 0

data/Dockerfiles/watchdog/Dockerfile

@@ -1,5 +1,5 @@
-FROM alpine:3.11
-LABEL maintainer "André Peters <andre.peters@servercow.de>"
+FROM alpine:3.18
+LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 # Installation
 RUN apk add --update \
@@ -36,4 +36,4 @@ RUN apk add --update \
 COPY watchdog.sh /watchdog.sh
 COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
 
-CMD /watchdog.sh 2> /dev/null
+CMD /watchdog.sh

data/Dockerfiles/watchdog/watchdog.sh

@@ -6,7 +6,10 @@ trap "kill 0" EXIT
 # Prepare
 BACKGROUND_TASKS=()
 echo "Waiting for containers to settle..."
-sleep 30
+for i in {30..1}; do
+  echo "${i}"
+  sleep 1
+done
 
 if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
   echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
@@ -14,6 +17,16 @@ if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
   exec $(readlink -f "$0")
 fi
 
+if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  SMTP_VERBOSE="--verbose"
+  CURL_VERBOSE="--verbose"
+  set -xv
+else
+  SMTP_VERBOSE=""
+  CURL_VERBOSE=""
+  exec 2>/dev/null
+fi
+
 # Checks pipe their corresponding container name in this pipe
 if [[ ! -p /tmp/com_pipe ]]; then
   mkfifo /tmp/com_pipe
@@ -44,8 +57,8 @@ get_ipv6(){
   local IPV6=
   local IPV6_SRCS=
   local TRY=
-  IPV6_SRCS[0]="ip6.korves.net"
-  IPV6_SRCS[1]="ip6.mailcow.email"
+  IPV6_SRCS[0]="ip6.mailcow.email"
+  IPV6_SRCS[1]="ip6.nevondo.com"
   until [[ ! -z ${IPV6} ]] || [[ ${TRY} -ge 10 ]]; do
     IPV6=$(curl --connect-timeout 3 -m 10 -L6s ${IPV6_SRCS[$RANDOM % ${#IPV6_SRCS[@]} ]} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$")
     [[ ! -z ${TRY} ]] && sleep 1
@@ -86,7 +99,9 @@ log_msg() {
   echo $(date) $(printf '%s\n' "${1}")
 }
 
-function mail_error() {
+function notify_error() {
+  # Check if one of the notification options is enabled
+  [[ -z ${WATCHDOG_NOTIFY_EMAIL} ]] && [[ -z ${WATCHDOG_NOTIFY_WEBHOOK} ]] && return 0
   THROTTLE=
   [[ -z ${1} ]] && return 1
   # If exists, body will be the content of "/tmp/${1}", even if ${2} is set
@@ -109,32 +124,59 @@ function mail_error() {
     SUBJECT="${BODY}"
     BODY="Please see netfilter-mailcow for more details and triggered rules."
   else
-    SUBJECT="Watchdog ALERT: ${1}"
+    SUBJECT="${WATCHDOG_SUBJECT}: ${1}"
+  fi
+
+  # Send mail notification if enabled
+  if [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]]; then
+    IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
+    for rcpt in "${MAIL_RCPTS[@]}"; do
+      RCPT_DOMAIN=
+      RCPT_MX=
+      RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
+      CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
+      if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
+        log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
+        return 1
+      fi
+      [ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
+      timeout 10s ./smtp-cli --missing-modules-ok \
+        "${SMTP_VERBOSE}" \
+        --charset=UTF-8 \
+        --subject="${SUBJECT}" \
+        --body-plain="${BODY}" \
+        --add-header="X-Priority: 1" \
+        --to=${rcpt} \
+        --from="watchdog@${MAILCOW_HOSTNAME}" \
+        --hello-host=${MAILCOW_HOSTNAME} \
+        --ipv4
+      if [[ $? -eq 1 ]]; then # exit code 1 is fine
+        log_msg "Sent notification email to ${rcpt}"
+      else
+        if [[ "${SMTP_VERBOSE}" == "" ]]; then
+          log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
+        else
+          log_msg "Error while sending notification email to ${rcpt}."
+        fi
+      fi
+    done
+  fi
+
+  # Send webhook notification if enabled
+  if [[ ! -z ${WATCHDOG_NOTIFY_WEBHOOK} ]]; then
+    if [[ -z ${WATCHDOG_NOTIFY_WEBHOOK_BODY} ]]; then
+      log_msg "No webhook body set, skipping webhook notification..."
+      return 1
+    fi
+
+    # Replace subject and body placeholders
+    WEBHOOK_BODY=$(echo ${WATCHDOG_NOTIFY_WEBHOOK_BODY} | sed "s/\$SUBJECT\|\${SUBJECT}/$SUBJECT/g" | sed "s/\$BODY\|\${BODY}/$BODY/g")
+    
+    # POST to webhook
+    curl -X POST -H "Content-Type: application/json" ${CURL_VERBOSE} -d "${WEBHOOK_BODY}" ${WATCHDOG_NOTIFY_WEBHOOK}
+
+    log_msg "Sent notification using webhook"
   fi
-  IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
-  for rcpt in "${MAIL_RCPTS[@]}"; do
-    RCPT_DOMAIN=
-    #RCPT_MX=
-    RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
-    # Latest smtp-cli looks up mx via dns
-    #RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
-    #if [[ -z ${RCPT_MX} ]]; then
-    #  log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
-    #  return 1
-    #fi
-    [ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
-    timeout 10s ./smtp-cli --missing-modules-ok \
-      --charset=UTF-8 \
-      --subject="${SUBJECT}" \
-      --body-plain="${BODY}" \
-      --add-header="X-Priority: 1" \
-      --to=${rcpt} \
-      --from="watchdog@${MAILCOW_HOSTNAME}" \
-      --hello-host=${MAILCOW_HOSTNAME} \
-      --ipv4
-      #--server="${RCPT_MX}"
-    log_msg "Sent notification email to ${rcpt}"
-  done
 }
 
 get_container_ip() {
@@ -154,7 +196,7 @@ get_container_ip() {
       CONTAINER_ID=($(printf "%s\n" "${CONTAINER_ID[@]}" | shuf))
       if [[ ! -z ${CONTAINER_ID} ]]; then
         for matched_container in "${CONTAINER_ID[@]}"; do
-          CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress')) 
+          CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress'))
           for ip_match in "${CONTAINER_IPS[@]}"; do
             # grep will do nothing if one of these vars is empty
             [[ -z ${ip_match} ]] && continue
@@ -179,7 +221,7 @@ get_container_ip() {
 # One-time check
 if grep -qi "$(echo ${IPV6_NETWORK} | cut -d: -f1-3)" <<< "$(ip a s)"; then
   if [[ -z "$(get_ipv6)" ]]; then
-    mail_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection."
+    notify_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection."
   fi
 fi
 
@@ -210,7 +252,7 @@ external_checks() {
       sleep 60
     else
       diff_c=0
-      sleep $(( ( RANDOM % 20 ) + 120 ))
+      sleep $(( ( RANDOM % 20 ) + 1800 ))
     fi
   done
   return 1
@@ -358,7 +400,7 @@ sogo_checks() {
     touch /tmp/sogo-mailcow; echo "$(tail -50 /tmp/sogo-mailcow)" > /tmp/sogo-mailcow
     host_ip=$(get_container_ip sogo-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 2>> /tmp/sogo-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 2>> /tmp/sogo-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
@@ -660,39 +702,6 @@ acme_checks() {
   return 1
 }
 
-ipv6nat_checks() {
-  err_count=0
-  diff_c=0
-  THRESHOLD=${IPV6NAT_THRESHOLD}
-  # Reduce error count by 2 after restarting an unhealthy container
-  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
-  while [ ${err_count} -lt ${THRESHOLD} ]; do
-    err_c_cur=${err_count}
-    CONTAINERS=$(curl --silent --insecure https://dockerapi/containers/json)
-    IPV6NAT_CONTAINER_ID=$(echo ${CONTAINERS} | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], project: .Config.Labels[\"com.docker.compose.project\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"ipv6nat-mailcow\")) | select( .project | tostring | contains(\"${COMPOSE_PROJECT_NAME,,}\")) | .id")
-    if [[ ! -z ${IPV6NAT_CONTAINER_ID} ]]; then
-      LATEST_STARTED="$(echo ${CONTAINERS} | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], project: .Config.Labels[\"com.docker.compose.project\"], StartedAt: .State.StartedAt}" | jq -rc "select( .project | tostring | contains(\"${COMPOSE_PROJECT_NAME,,}\")) | select( .name | tostring | contains(\"ipv6nat-mailcow\") | not)" | jq -rc .StartedAt | xargs -n1 date +%s -d | sort | tail -n1)"
-      LATEST_IPV6NAT="$(echo ${CONTAINERS} | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], project: .Config.Labels[\"com.docker.compose.project\"], StartedAt: .State.StartedAt}" | jq -rc "select( .project | tostring | contains(\"${COMPOSE_PROJECT_NAME,,}\")) | select( .name | tostring | contains(\"ipv6nat-mailcow\"))" | jq -rc .StartedAt | xargs -n1 date +%s -d | sort | tail -n1)"
-      DIFFERENCE_START_TIME=$(expr ${LATEST_IPV6NAT} - ${LATEST_STARTED} 2>/dev/null)
-      if [[ "${DIFFERENCE_START_TIME}" -lt 30 ]]; then
-        err_count=$(( ${err_count} + 1 ))
-      fi
-    fi
-    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
-    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
-    progress "IPv6 NAT" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    if [[ $? == 10 ]]; then
-      diff_c=0
-      sleep 30
-    else
-      diff_c=0
-      sleep 300
-    fi
-  done
-  return 1
-}
-
-
 rspamd_checks() {
   err_count=0
   diff_c=0
@@ -707,12 +716,12 @@ rspamd_checks() {
 From: watchdog@localhost
 
 Empty
-' | usr/bin/curl --max-time 10 -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan | jq -rc .default.required_score)
-    if [[ ${SCORE} != "9999" ]]; then
-      echo "Rspamd settings check failed" 2>> /tmp/rspamd-mailcow 1>&2
+' | usr/bin/curl --max-time 10 -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan | jq -rc .default.required_score | sed 's/\..*//' )
+    if [[ ${SCORE} -ne 9999 ]]; then
+      echo "Rspamd settings check failed, score returned: ${SCORE}" 2>> /tmp/rspamd-mailcow 1>&2
       err_count=$(( ${err_count} + 1))
     else
-      echo "Rspamd settings check succeeded" 2>> /tmp/rspamd-mailcow 1>&2
+      echo "Rspamd settings check succeeded, score returned: ${SCORE}" 2>> /tmp/rspamd-mailcow 1>&2
     fi
     # A dirty hack until a PING PONG event is implemented to worker proxy
     # We expect an empty response, not a timeout
@@ -761,8 +770,8 @@ olefy_checks() {
 }
 
 # Notify about start
-if [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]]; then
-  mail_error "watchdog-mailcow" "Watchdog started monitoring mailcow."
+if [[ ${WATCHDOG_NOTIFY_START} =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  notify_error "watchdog-mailcow" "Watchdog started monitoring mailcow."
 fi
 
 # Create watchdog agents
@@ -1005,18 +1014,6 @@ PID=$!
 echo "Spawned acme_checks with PID ${PID}"
 BACKGROUND_TASKS+=(${PID})
 
-(
-while true; do
-  if ! ipv6nat_checks; then
-    log_msg "IPv6 NAT warning: ipv6nat-mailcow container was not started at least 30s after siblings (not an error)"
-    echo ipv6nat-mailcow > /tmp/com_pipe
-  fi
-done
-) &
-PID=$!
-echo "Spawned ipv6nat_checks with PID ${PID}"
-BACKGROUND_TASKS+=(${PID})
-
 # Monitor watchdog agents, stop script when agents fails and wait for respawn by Docker (restart:always:n)
 (
 while true; do
@@ -1056,33 +1053,33 @@ while true; do
   fi
   if [[ ${com_pipe_answer} == "ratelimit" ]]; then
     log_msg "At least one ratelimit was applied"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
+    notify_error "${com_pipe_answer}"
   elif [[ ${com_pipe_answer} == "mail_queue_status" ]]; then
     log_msg "Mail queue status is critical"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
+    notify_error "${com_pipe_answer}"
   elif [[ ${com_pipe_answer} == "external_checks" ]]; then
     log_msg "Your mailcow is an open relay!"
     # Define $2 to override message text, else print service was restarted at ...
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please stop mailcow now and check your network configuration!"
+    notify_error "${com_pipe_answer}" "Please stop mailcow now and check your network configuration!"
   elif [[ ${com_pipe_answer} == "mysql_repl_checks" ]]; then
     log_msg "MySQL replication is not working properly"
     # Define $2 to override message text, else print service was restarted at ...
     # Once mail per 10 minutes
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check the SQL replication status" 600
+    notify_error "${com_pipe_answer}" "Please check the SQL replication status" 600
   elif [[ ${com_pipe_answer} == "dovecot_repl_checks" ]]; then
     log_msg "Dovecot replication is not working properly"
     # Define $2 to override message text, else print service was restarted at ...
     # Once mail per 10 minutes
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check the Dovecot replicator status" 600
+    notify_error "${com_pipe_answer}" "Please check the Dovecot replicator status" 600
   elif [[ ${com_pipe_answer} == "certcheck" ]]; then
     log_msg "Certificates are about to expire"
     # Define $2 to override message text, else print service was restarted at ...
     # Only mail once a day
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please renew your certificate" 86400
+    notify_error "${com_pipe_answer}" "Please renew your certificate" 86400
   elif [[ ${com_pipe_answer} == "acme-mailcow" ]]; then
     log_msg "acme-mailcow did not complete successfully"
     # Define $2 to override message text, else print service was restarted at ...
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
+    notify_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
   elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
     F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null))
     if [[ ! -z "${F2B_RES}" ]]; then
@@ -1092,7 +1089,7 @@ while true; do
         log_msg "Banned ${host}"
         rm /tmp/fail2ban 2> /dev/null
         timeout 2s whois "${host}" > /tmp/fail2ban
-        [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && [[ ${WATCHDOG_NOTIFY_BAN} =~ ^([yY][eE][sS]|[yY])+$ ]] && mail_error "${com_pipe_answer}" "IP ban: ${host}"
+        [[ ${WATCHDOG_NOTIFY_BAN} =~ ^([yY][eE][sS]|[yY])+$ ]] && notify_error "${com_pipe_answer}" "IP ban: ${host}"
       done
     fi
   elif [[ ${com_pipe_answer} =~ .+-mailcow ]]; then
@@ -1112,9 +1109,7 @@ while true; do
       else
         log_msg "Sending restart command to ${CONTAINER_ID}..."
         curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/restart
-        if [[ ${com_pipe_answer} != "ipv6nat-mailcow" ]]; then
-          [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
-        fi
+        notify_error "${com_pipe_answer}"
         log_msg "Wait for restarted container to settle and continue watching..."
         sleep 35
       fi
@@ -1124,3 +1119,4 @@ while true; do
     kill -USR1 ${BACKGROUND_TASKS[*]}
   fi
 done
+

data/assets/nextcloud/nextcloud.conf

@@ -24,7 +24,7 @@ server {
   add_header X-Download-Options "noopen" always;
   add_header X-Frame-Options "SAMEORIGIN" always;
   add_header X-Permitted-Cross-Domain-Policies "none" always;
-  add_header X-Robots-Tag "none" always;
+  add_header X-Robots-Tag "noindex, nofollow" always;
   add_header X-XSS-Protection "1; mode=block" always;
 
   fastcgi_hide_header X-Powered-By;
@@ -47,6 +47,14 @@ server {
     return 301 $client_req_scheme_nc://$host/remote.php/dav;
   }
 
+  location = /.well-known/webfinger {
+    return 301 $client_req_scheme_nc://$host/index.php/.well-known/webfinger;
+  }
+
+  location = /.well-known/nodeinfo {
+    return 301 $client_req_scheme_nc://$host/index.php/.well-known/nodeinfo;
+  }
+
   location ^~ /.well-known/acme-challenge/ {
     default_type "text/plain";
     root /web;
@@ -78,7 +86,7 @@ server {
     deny all;
   }
 
-  location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
+  location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|ocs-provider\/.+)\.php(?:$|\/) {
     fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
     set $path_info $fastcgi_path_info;
     try_files $fastcgi_script_name =404;
@@ -97,7 +105,7 @@ server {
     fastcgi_read_timeout 1200;
   }
 
-  location ~ ^\/(?:updater|oc[ms]-provider)(?:$|\/) {
+  location ~ ^\/(?:updater|ocs-provider)(?:$|\/) {
     try_files $uri/ =404;
     index index.php;
   }

data/assets/templates/quarantine.tpl

@@ -42,15 +42,24 @@
     There are {{counter}} new messages waiting in quarantine:<br>
     {% endif %}
     <table>
-    <tr><th>Subject</th><th>Sender</th><th class="mob">Score</th><th class="mob">Arrived on</th>{% if quarantine_acl == 1 %}<th>Actions</th>{% endif %}</tr>
-    {% for line in meta %}
+    <tr><th>Subject</th><th>Sender</th><th class="mob">Score</th><th class="mob">Action</th><th class="mob">Arrived on</th>{% if quarantine_acl == 1 %}<th>Actions</th>{% endif %}</tr>
+    {% for line in meta|reverse %}
     <tr>
     <td>{{ line.subject|e }}</td>
     <td>{{ line.sender|e }}</td>
     <td class="mob">{{ line.score }}</td>
+    {% if line.action == "reject" %}
+      <td class="mob">Rejected</td>
+    {% else %}
+      <td class="mob">Sent to Junk folder</td>
+    {% endif %}
     <td class="mob">{{ line.created }}</td>
     {% if quarantine_acl == 1 %}
-    <td class="fixed"><a href="https://{{ hostname }}/qhandler/release/{{ line.qhash }}">release</a> | <a href="https://{{ hostname }}/qhandler/delete/{{ line.qhash }}">delete</a></td>
+      {% if line.action == "reject" %}
+        <td class="fixed"><a href="https://{{ hostname }}/qhandler/release/{{ line.qhash }}">Release to inbox</a> | <a href="https://{{ hostname }}/qhandler/delete/{{ line.qhash }}">delete</a></td>
+      {% else %}
+        <td class="fixed"><a href="https://{{ hostname }}/qhandler/release/{{ line.qhash }}">Send copy to inbox</a> | <a href="https://{{ hostname }}/qhandler/delete/{{ line.qhash }}">delete</a></td>
+      {% endif %}
     {% endif %}
     </tr>
     {% endfor %}

data/conf/dovecot/auth/mailcowauth.php

@@ -0,0 +1,75 @@
+<?php
+ini_set('error_reporting', 0);
+header('Content-Type: application/json');
+
+$post = trim(file_get_contents('php://input'));
+if ($post) {
+  $post = json_decode($post, true);
+}
+
+
+$return = array("success" => false);
+if(!isset($post['username']) || !isset($post['password']) || !isset($post['real_rip'])){
+  error_log("MAILCOWAUTH: Bad Request");
+  http_response_code(400); // Bad Request
+  echo json_encode($return); 
+  exit();
+}
+
+require_once('../../../web/inc/vars.inc.php');
+if (file_exists('../../../web/inc/vars.local.inc.php')) {
+  include_once('../../../web/inc/vars.local.inc.php');
+}
+require_once '../../../web/inc/lib/vendor/autoload.php';
+
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("MAILCOWAUTH: " . $e . PHP_EOL);
+  http_response_code(500); // Internal Server Error
+  echo json_encode($return); 
+  exit;
+}
+
+// Load core functions first
+require_once 'functions.inc.php';
+require_once 'functions.auth.inc.php';
+require_once 'sessions.inc.php';
+require_once 'functions.mailbox.inc.php';
+
+// Init provider
+$iam_provider = identity_provider('init');
+
+
+$protocol = $post['protocol'];
+if ($post['real_rip'] == getenv('IPV4_NETWORK') . '.248') {
+  $protocol = null;
+}
+$result = user_login($post['username'], $post['password'], $protocol, array('is_internal' => true));
+if ($result === false){
+  $result = apppass_login($post['username'], $post['password'], $protocol, array(
+    'is_internal' => true,
+    'remote_addr' => $post['real_rip']
+  ));
+}
+
+if ($result) {
+  http_response_code(200); // OK
+  $return['success'] = true;
+} else {
+  error_log("MAILCOWAUTH: Login failed for user " . $post['username']);
+  http_response_code(401); // Unauthorized
+}
+
+
+echo json_encode($return); 
+session_destroy();
+exit;

data/conf/dovecot/auth/passwd-verify.lua

@@ -0,0 +1,42 @@
+function auth_password_verify(request, password)
+  if request.domain == nil then
+    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
+  end
+
+  json = require "cjson"
+  ltn12 = require "ltn12"
+  https = require "ssl.https"
+  https.TIMEOUT = 5
+
+  local req = {
+    username = request.user,
+    password = password,
+    real_rip = request.real_rip,
+    protocol = {}
+  }
+  req.protocol[request.service] = true
+  local req_json = json.encode(req)
+  local res = {} 
+  
+  local b, c = https.request {
+    method = "POST",
+    url = "https://nginx:9082",
+    source = ltn12.source.string(req_json),
+    headers = {
+      ["content-type"] = "application/json",
+      ["content-length"] = tostring(#req_json)
+    },
+    sink = ltn12.sink.table(res),
+    insecure = true
+  }
+  local api_response = json.decode(table.concat(res))
+  if api_response.success == true then
+    return dovecot.auth.PASSDB_RESULT_OK, ""
+  end
+  
+  return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, "Failed to authenticate"
+end
+
+function auth_passdb_lookup(req)
+   return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, ""
+end

data/conf/dovecot/dovecot.conf

@@ -24,6 +24,11 @@ mail_plugins = </etc/dovecot/mail_plugins
 mail_attachment_fs = crypt:set_prefix=mail_crypt_global:posix:
 mail_attachment_dir = /var/attachments
 mail_attachment_min_size = 128k
+# Significantly speeds up very large mailboxes, but is only safe to enable if
+# you do not manually modify the files in the `cur` directories in
+# mailcowdockerized_vmail-vol-1.
+# https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-performance/
+maildir_very_dirty_syncs = yes
 
 # Dovecot 2.2
 #ssl_protocols = !SSLv3
@@ -45,36 +50,25 @@ recipient_delimiter = +
 auth_master_user_separator = *
 mail_shared_explicit_inbox = yes
 mail_prefetch_count = 30
+passdb {
+  driver = lua
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes
+  result_success = return-ok
+  result_failure = continue
+  result_internalfail = continue
+}
 # try a master passwd
 passdb {
   driver = passwd-file
   args = /etc/dovecot/dovecot-master.passwd
   master = yes
-  pass = yes
-  result_failure = continue
-  result_internalfail = continue
-}
-# try an app passwd
-passdb {
-  driver = lua
-  args = file=/etc/dovecot/lua/app-passdb.lua blocking=yes
-  pass = yes
-  result_failure = continue
-  result_internalfail = continue
+  skip = authenticated
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
 passdb {
-  args = /etc/dovecot/sql/dovecot-dict-sql-passdb.conf
-  driver = sql
-  result_success = return-ok
-  result_failure = continue
-  result_internalfail = continue
-}
-passdb {
-  driver = passwd-file
-  args = /etc/dovecot/dovecot-master.passwd
-  skip = authenticated
+  driver = lua
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes
 }
 # Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing)
 service doveadm {
@@ -83,224 +77,7 @@ service doveadm {
   }
   vsz_limit=2048 MB
 }
-namespace inbox {
-  inbox = yes
-  location =
-  separator = /
-  mailbox "Trash" {
-    auto = subscribe
-    special_use = \Trash
-  }
-  mailbox "Deleted Messages" {
-    special_use = \Trash
-  }
-  mailbox "Deleted Items" {
-    special_use = \Trash
-  }
-  mailbox "Rubbish" {
-    special_use = \Trash
-  }
-  mailbox "Gelöschte Objekte" {
-    special_use = \Trash
-  }
-  mailbox "Gelöschte Elemente" {
-    special_use = \Trash
-  }
-  mailbox "Papierkorb" {
-    special_use = \Trash
-  }
-  mailbox "Itens Excluidos" {
-    special_use = \Trash
-  }
-  mailbox "Itens Excluídos" {
-    special_use = \Trash
-  }
-  mailbox "Lixeira" {
-    special_use = \Trash
-  }
-  mailbox "Prullenbak" {
-    special_use = \Trash
-  }
-  mailbox "Odstránené položky" {
-    special_use = \Trash
-  }
-  mailbox "Koš" {
-    special_use = \Trash
-  }
-  mailbox "Verwijderde items" {
-    special_use = \Trash
-  }
-  mailbox "废件箱" {
-    special_use = \Trash
-  }
-  mailbox "已删除消息" {
-    special_use = \Trash
-  }
-  mailbox "已删除邮件" {
-    special_use = \Trash
-  }
-  mailbox "Archive" {
-    auto = subscribe
-    special_use = \Archive
-  }
-  mailbox "Archiv" {
-    special_use = \Archive
-  }
-  mailbox "Archives" {
-    special_use = \Archive
-  }
-  mailbox "Arquivo" {
-    special_use = \Archive
-  }
-  mailbox "Arquivos" {
-    special_use = \Archive
-  }
-  mailbox "Archief" {
-    special_use = \Archive
-  }
-  mailbox "Archív" {
-    special_use = \Archive
-  }
-  mailbox "Archivovať" {
-    special_use = \Archive
-  }
-  mailbox "归档" {
-    special_use = \Archive
-  }
-  mailbox "Sent" {
-    auto = subscribe
-    special_use = \Sent
-  }
-  mailbox "Sent Messages" {
-    special_use = \Sent
-  }
-  mailbox "Sent Items" {
-    special_use = \Sent
-  }
-  mailbox "已发送" {
-    special_use = \Sent
-  }
-  mailbox "已发送消息" {
-    special_use = \Sent
-  }
-  mailbox "已发送邮件" {
-    special_use = \Sent
-  }
-  mailbox "Gesendet" {
-    special_use = \Sent
-  }
-  mailbox "Gesendete Objekte" {
-    special_use = \Sent
-  }
-  mailbox "Gesendete Elemente" {
-    special_use = \Sent
-  }
-  mailbox "Itens Enviados" {
-    special_use = \Sent
-  }
-  mailbox "Enviados" {
-    special_use = \Sent
-  }
-  mailbox "Verzonden items" {
-    special_use = \Sent
-  }
-  mailbox "Verzonden" {
-    special_use = \Sent
-  }
-  mailbox "Odoslaná pošta" {
-    special_use = \Sent
-  }
-  mailbox "Odoslané" {
-    special_use = \Sent
-  }
-  mailbox "Drafts" {
-    auto = subscribe
-    special_use = \Drafts
-  }
-  mailbox "Entwürfe" {
-    special_use = \Drafts
-  }
-  mailbox "Rascunhos" {
-    special_use = \Drafts
-  }
-  mailbox "Concepten" {
-    special_use = \Drafts
-  }
-  mailbox "Koncepty" {
-    special_use = \Drafts
-  }
-  mailbox "草稿" {
-    special_use = \Drafts
-  }
-  mailbox "草稿箱" {
-    special_use = \Drafts
-  }
-  mailbox "Junk" {
-    auto = subscribe
-    special_use = \Junk
-  }
-  mailbox "Junk-E-Mail" {
-    special_use = \Junk
-  }
-  mailbox "Junk E-Mail" {
-    special_use = \Junk
-  }
-  mailbox "Spam" {
-    special_use = \Junk
-  }
-  mailbox "Lixo Eletrônico" {
-    special_use = \Junk
-  }
-  mailbox "Nevyžiadaná pošta" {
-    special_use = \Junk
-  }
-  mailbox "Infikované položky" {
-    special_use = \Junk
-  }
-  mailbox "Ongewenste e-mail" {
-    special_use = \Junk
-  }
-  mailbox "垃圾" {
-    special_use = \Junk
-  }
-  mailbox "垃圾箱" {
-    special_use = \Junk
-  }
-  mailbox "Koncepty" {
-    special_use = \Drafts
-  }
-  mailbox "Nevyžádaná pošta" {
-    special_use = \Junk
-  }
-  mailbox "Odstraněná pošta" {
-    special_use = \Trash
-  }
-  mailbox "Odeslaná pošta" {
-    special_use = \Sent
-  }
-  mailbox "Skräp" {
-    special_use = \Trash
-  }
-  mailbox "Borttagna Meddelanden" {
-    special_use = \Trash
-  }
-  mailbox "Arkiv" {
-    special_use = \Archive
-  }
-  mailbox "Arkeverat" {
-    special_use = \Archive
-  }
-  mailbox "Skickat" {
-    special_use = \Sent
-  }
-  mailbox "Skickade Meddelanden" {
-    special_use = \Sent
-  }
-  mailbox "Utkast" {
-    special_use = \Drafts
-  }
-  prefix =
-}
+!include /etc/dovecot/dovecot.folders.conf
 protocols = imap sieve lmtp pop3
 service dict {
   unix_listener dict {
@@ -331,11 +108,16 @@ service auth {
     mode = 0600
     user = vmail
   }
+  vsz_limit = 2G
 }
 service managesieve-login {
   inet_listener sieve {
     port = 4190
   }
+  inet_listener sieve_haproxy {
+    port = 14190
+    haproxy = yes
+  }
   service_count = 1
   process_min_avail = 2
   vsz_limit = 1G
@@ -369,7 +151,7 @@ service pop3-login {
   }
 }
 service imap {
-  executable = imap imap-postlogin
+  executable = imap
   user = vmail
   vsz_limit = 1G
 }
@@ -385,8 +167,6 @@ service lmtp {
 listen = *,[::]
 ssl_cert = </etc/ssl/mail/cert.pem
 ssl_key = </etc/ssl/mail/key.pem
-!include_try /etc/dovecot/sni.conf
-!include_try /etc/dovecot/sogo_trusted_ip.conf
 userdb {
   driver = passwd-file
   args = /etc/dovecot/dovecot-master.userdb
@@ -413,8 +193,7 @@ plugin {
   acl_anyone = </etc/dovecot/acl_anyone
   acl_shared_dict = file:/var/vmail/shared-mailboxes.db
   acl = vfile
-  last_login_dict = </etc/dovecot/last_login
-  last_login_key = last-login/%s/%u
+  acl_user = %u
   fts = solr
   fts_autoindex = yes
   fts_solr = url=http://solr:8983/solr/dovecot-fts/
@@ -435,6 +214,7 @@ plugin {
   imapsieve_mailbox2_causes = COPY
   imapsieve_mailbox2_before = file:/usr/lib/dovecot/sieve/report-ham.sieve
   # END
+  master_user = %u
   quota_warning = storage=95%% quota-warning 95 %u
   quota_warning2 = storage=80%% quota-warning 80 %u
   sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
@@ -453,7 +233,7 @@ plugin {
   sieve_before2 = dict:proxy::sieve_before;name=active;bindir=/var/vmail/sieve_before_bindir
   sieve_after = dict:proxy::sieve_after;name=active;bindir=/var/vmail/sieve_after_bindir
   sieve_after2 = /var/vmail/sieve/global_sieve_after.sieve
-  sieve_duplicate_default_period = 60m
+  sieve_duplicate_default_period = 1m
   sieve_duplicate_max_period = 7d
 
   # -- Global keys
@@ -461,12 +241,15 @@ plugin {
   mail_crypt_global_public_key = </mail_crypt/ecpubkey.pem
   mail_crypt_save_version = 2
 
-  # Enable compression while saving, lz4 Dovecot v2.2.11+
+  # Enable compression while saving, lz4 Dovecot v2.3.17+
   zlib_save = lz4
 
   mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
   mail_log_fields = uid box msgid size
   mail_log_cached_only = yes
+
+  # Try set mail_replica
+  !include_try /etc/dovecot/mail_replica.conf
 }
 service quota-warning {
   executable = script /usr/local/bin/quota_notify.py
@@ -486,13 +269,6 @@ remote 127.0.0.1 {
 }
 submission_host = postfix:588
 mail_max_userip_connections = 500
-service imap-postlogin {
-  executable = script-login /usr/local/bin/postlogin.sh
-  unix_listener imap-postlogin {
-    user = vmail
-    mode = 0660
-  }
-}
 service stats {
   unix_listener stats-writer {
     mode = 0660
@@ -523,8 +299,12 @@ service replicator {
 replication_max_conns = 10
 doveadm_port = 12345
 replication_dsync_parameters = -d -l 30 -U -n INBOX
+# <Includes>
+!include_try /etc/dovecot/sni.conf
+!include_try /etc/dovecot/sogo_trusted_ip.conf
 !include_try /etc/dovecot/extra.conf
 !include_try /etc/dovecot/sogo-sso.conf
 !include_try /etc/dovecot/shared_namespace.conf
+# </Includes>
 default_client_limit = 10400
 default_vsz_limit = 1024 M

data/conf/dovecot/dovecot.folders.conf

@@ -0,0 +1,293 @@
+namespace inbox {
+  inbox = yes
+  location =
+  separator = /
+  mailbox "Trash" {
+    auto = subscribe
+    special_use = \Trash
+  }
+  mailbox "Deleted Messages" {
+    special_use = \Trash
+  }
+  mailbox "Deleted Items" {
+    special_use = \Trash
+  }
+  mailbox "Rubbish" {
+    special_use = \Trash
+  }
+  mailbox "Gelöschte Objekte" {
+    special_use = \Trash
+  }
+  mailbox "Gelöschte Elemente" {
+    special_use = \Trash
+  }
+  mailbox "Papierkorb" {
+    special_use = \Trash
+  }
+  mailbox "Itens Excluidos" {
+    special_use = \Trash
+  }
+  mailbox "Itens Excluídos" {
+    special_use = \Trash
+  }
+  mailbox "Lixeira" {
+    special_use = \Trash
+  }
+  mailbox "Prullenbak" {
+    special_use = \Trash
+  }
+  mailbox "Odstránené položky" {
+    special_use = \Trash
+  }
+  mailbox "Koš" {
+    special_use = \Trash
+  }
+  mailbox "Verwijderde items" {
+    special_use = \Trash
+  }
+  mailbox "Удаленные" {
+    special_use = \Trash
+  }
+  mailbox "Удаленные элементы" {
+    special_use = \Trash
+  }
+  mailbox "Корзина" {
+    special_use = \Trash
+  }
+  mailbox "Видалені" {
+    special_use = \Trash
+  }
+  mailbox "Видалені елементи" {
+    special_use = \Trash
+  }
+  mailbox "Кошик" {
+    special_use = \Trash
+  }
+  mailbox "废件箱" {
+    special_use = \Trash
+  }
+  mailbox "已删除消息" {
+    special_use = \Trash
+  }
+  mailbox "已删除邮件" {
+    special_use = \Trash
+  }
+  mailbox "Archive" {
+    auto = subscribe
+    special_use = \Archive
+  }
+  mailbox "Archiv" {
+    special_use = \Archive
+  }
+  mailbox "Archives" {
+    special_use = \Archive
+  }
+  mailbox "Arquivo" {
+    special_use = \Archive
+  }
+  mailbox "Arquivos" {
+    special_use = \Archive
+  }
+  mailbox "Archief" {
+    special_use = \Archive
+  }
+  mailbox "Archív" {
+    special_use = \Archive
+  }
+  mailbox "Archivovať" {
+    special_use = \Archive
+  }
+  mailbox "归档" {
+    special_use = \Archive
+  }
+  mailbox "Архив" {
+    special_use = \Archive
+  }
+  mailbox "Архів" {
+    special_use = \Archive
+  }
+  mailbox "Sent" {
+    auto = subscribe
+    special_use = \Sent
+  }
+  mailbox "Sent Messages" {
+    special_use = \Sent
+  }
+  mailbox "Sent Items" {
+    special_use = \Sent
+  }
+  mailbox "已发送" {
+    special_use = \Sent
+  }
+  mailbox "已发送消息" {
+    special_use = \Sent
+  }
+  mailbox "已发送邮件" {
+    special_use = \Sent
+  }
+  mailbox "Отправленные" {
+    special_use = \Sent
+  }
+  mailbox "Отправленные элементы" {
+    special_use = \Sent
+  }
+  mailbox "Надіслані" {
+    special_use = \Sent
+  }
+  mailbox "Надіслані елементи" {
+    special_use = \Sent
+  }
+  mailbox "Gesendet" {
+    special_use = \Sent
+  }
+  mailbox "Gesendete Objekte" {
+    special_use = \Sent
+  }
+  mailbox "Gesendete Elemente" {
+    special_use = \Sent
+  }
+  mailbox "Itens Enviados" {
+    special_use = \Sent
+  }
+  mailbox "Enviados" {
+    special_use = \Sent
+  }
+  mailbox "Verzonden items" {
+    special_use = \Sent
+  }
+  mailbox "Verzonden" {
+    special_use = \Sent
+  }
+  mailbox "Odoslaná pošta" {
+    special_use = \Sent
+  }
+  mailbox "Odoslané" {
+    special_use = \Sent
+  }
+  mailbox "Drafts" {
+    auto = subscribe
+    special_use = \Drafts
+  }
+  mailbox "Entwürfe" {
+    special_use = \Drafts
+  }
+  mailbox "Rascunhos" {
+    special_use = \Drafts
+  }
+  mailbox "Concepten" {
+    special_use = \Drafts
+  }
+  mailbox "Koncepty" {
+    special_use = \Drafts
+  }
+  mailbox "草稿" {
+    special_use = \Drafts
+  }
+  mailbox "草稿箱" {
+    special_use = \Drafts
+  }
+  mailbox "Черновики" {
+    special_use = \Drafts
+  }
+  mailbox "Чернетки" {
+    special_use = \Drafts
+  }
+  mailbox "Junk" {
+    auto = subscribe
+    special_use = \Junk
+  }
+  mailbox "Junk-E-Mail" {
+    special_use = \Junk
+  }
+  mailbox "Junk E-Mail" {
+    special_use = \Junk
+  }
+  mailbox "Spam" {
+    special_use = \Junk
+  }
+  mailbox "Lixo Eletrônico" {
+    special_use = \Junk
+  }
+  mailbox "Nevyžiadaná pošta" {
+    special_use = \Junk
+  }
+  mailbox "Infikované položky" {
+    special_use = \Junk
+  }
+  mailbox "Ongewenste e-mail" {
+    special_use = \Junk
+  }
+  mailbox "垃圾" {
+    special_use = \Junk
+  }
+  mailbox "垃圾箱" {
+    special_use = \Junk
+  }
+  mailbox "Нежелательная почта" {
+    special_use = \Junk
+  }
+  mailbox "Спам" {
+    special_use = \Junk
+  }
+  mailbox "Небажана пошта" {
+    special_use = \Junk
+  }
+  mailbox "Koncepty" {
+    special_use = \Drafts
+  }
+  mailbox "Nevyžádaná pošta" {
+    special_use = \Junk
+  }
+  mailbox "Odstraněná pošta" {
+    special_use = \Trash
+  }
+  mailbox "Odeslaná pošta" {
+    special_use = \Sent
+  }
+  mailbox "Skräp" {
+    special_use = \Trash
+  }
+  mailbox "Borttagna Meddelanden" {
+    special_use = \Trash
+  }
+  mailbox "Arkiv" {
+    special_use = \Archive
+  }
+  mailbox "Arkeverat" {
+    special_use = \Archive
+  }
+  mailbox "Skickat" {
+    special_use = \Sent
+  }
+  mailbox "Skickade Meddelanden" {
+    special_use = \Sent
+  }
+  mailbox "Utkast" {
+    special_use = \Drafts
+  }
+  mailbox "Skraldespand" {
+    special_use = \Trash
+  }
+  mailbox "Slettet mails" {
+    special_use = \Trash
+  }
+  mailbox "Arkiv" {
+    special_use = \Archive
+  }
+  mailbox "Arkiveret mails" {
+    special_use = \Archive
+  }
+  mailbox "Sendt" {
+    special_use = \Sent
+  }
+  mailbox "Sendte mails" {
+    special_use = \Sent
+  }
+  mailbox "Udkast" {
+    special_use = \Drafts
+  }
+  mailbox "Kladde" {
+    special_use = \Drafts
+  }
+  prefix =
+}

data/conf/dovecot/global_sieve_before

@@ -1,2 +1,13 @@
 # global_sieve_before script
 # global_sieve_before -> user sieve_before (mailcow UI) -> user sieve_after (mailcow UI) -> global_sieve_after
+
+require ["mailbox", "fileinto"];
+
+if header :contains ["Chat-Version"] [""] {
+  if mailboxexists "DeltaChat" {
+    fileinto "DeltaChat";
+  } else {
+    fileinto :create "DeltaChat";
+  }
+  stop;
+}

data/conf/mysql/my.cnf

@@ -7,7 +7,7 @@ collation-server               = utf8mb4_unicode_ci
 #innodb_large_prefix            = TRUE
 #sql_mode=IGNORE_SPACE,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
 max_allowed_packet      = 192M
-max-connections         = 350
+max-connections         = 550
 key_buffer_size         = 0
 read_buffer_size        = 192K
 sort_buffer_size        = 2M
@@ -25,6 +25,8 @@ skip-host-cache
 skip-name-resolve
 log-warnings            = 0
 event_scheduler         = 1
+interactive_timeout     = 3610
+wait_timeout            = 3610
 
 [client]
 default-character-set = utf8mb4

data/conf/nginx/includes/site-defaults.conf

@@ -3,6 +3,8 @@
   charset utf-8;
   override_charset on;
 
+  server_tokens off;
+
   ssl_protocols TLSv1.2 TLSv1.3;
   ssl_prefer_server_ciphers on;
   ssl_ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305;
@@ -11,6 +13,8 @@
   ssl_session_timeout 1d;
   ssl_session_tickets off;
 
+  include /etc/nginx/conf.d/includes/ssl_client_auth.conf;
+
   add_header Strict-Transport-Security "max-age=15768000;";
   add_header X-Content-Type-Options nosniff;
   add_header X-XSS-Protection "1; mode=block";
@@ -63,7 +67,7 @@
   }
 
   location ~ ^/api/v1/(.*)$ {
-    try_files $uri $uri/ /json_api.php?query=$1;
+    try_files $uri $uri/ /json_api.php?query=$1&$args;
   }
 
   location ^~ /.well-known/acme-challenge/ {
@@ -86,6 +90,11 @@
     return 301 /SOGo/dav;
   }
 
+  location ^~ /inc/lib/ {
+    deny all;
+    return 403;
+  }
+
   location ~ \.php$ {
     try_files $uri =404;
     fastcgi_split_path_info ^(.+\.php)(/.+)$;
@@ -94,7 +103,12 @@
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     fastcgi_param PATH_INFO $fastcgi_path_info;
-    fastcgi_read_timeout 1200;
+    fastcgi_param TLS_SUCCESS $ssl_client_verify;
+    fastcgi_param TLS_ISSUER  $ssl_client_i_dn;
+    fastcgi_param TLS_DN      $ssl_client_s_dn;
+    fastcgi_param TLS_CERT    $ssl_client_cert;
+    fastcgi_read_timeout 3600;
+    fastcgi_send_timeout 3600;
   }
 
   location /rspamd/ {
@@ -106,7 +120,7 @@
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_redirect off;
-      error_page 403 /_rspamderror.php;
+      error_page 401 /_rspamderror.php;
     }
     proxy_pass       http://rspamd:11334/;
     proxy_set_header Host      $http_host;
@@ -139,7 +153,6 @@
     try_files /autoconfig.php =404;
   }
 
-  # auth_request endpoint if ALLOW_ADMIN_EMAIL_LOGIN is set
   location /sogo-auth-verify {
     internal;
     proxy_set_header  X-Original-URI $request_uri;
@@ -151,12 +164,14 @@
   }
 
   location ^~ /Microsoft-Server-ActiveSync {
-    include /etc/nginx/conf.d/sogo_proxy_auth.active;
+    include /etc/nginx/conf.d/includes/sogo_proxy_auth.conf;
     include /etc/nginx/conf.d/sogo_eas.active;
     proxy_connect_timeout 75;
     proxy_send_timeout 3600;
     proxy_read_timeout 3600;
-    proxy_buffers 64 256k;
+    proxy_buffer_size 128k;
+    proxy_buffers 64 512k;
+    proxy_busy_buffers_size 512k;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header Host $http_host;
@@ -165,7 +180,22 @@
   }
 
   location ^~ /SOGo {
-    include /etc/nginx/conf.d/sogo_proxy_auth.active;
+    location ~* ^/SOGo/so/.*\.(xml|js|html|xhtml)$ {
+      include /etc/nginx/conf.d/includes/sogo_proxy_auth.conf;
+      include /etc/nginx/conf.d/sogo.active;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header Host $http_host;
+      proxy_set_header x-webobjects-server-protocol HTTP/1.0;
+      proxy_set_header x-webobjects-remote-host $remote_addr;
+      proxy_set_header x-webobjects-server-name $server_name;
+      proxy_set_header x-webobjects-server-url $client_req_scheme://$http_host;
+      proxy_set_header x-webobjects-server-port $server_port;
+      proxy_hide_header Content-Type;
+      add_header Content-Type text/plain;
+      break;
+    }
+    include /etc/nginx/conf.d/includes/sogo_proxy_auth.conf;
     include /etc/nginx/conf.d/sogo.active;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -175,8 +205,11 @@
     proxy_set_header x-webobjects-server-name $server_name;
     proxy_set_header x-webobjects-server-url $client_req_scheme://$http_host;
     proxy_set_header x-webobjects-server-port $server_port;
-    proxy_send_timeout 300;
-    proxy_read_timeout 300;
+    proxy_buffer_size 128k;
+    proxy_buffers 64 512k;
+    proxy_busy_buffers_size 512k;
+    proxy_send_timeout 3600;
+    proxy_read_timeout 3600;
     client_body_buffer_size 128k;
     client_max_body_size 0;
     break;

data/conf/nginx/includes/sogo_proxy_auth.conf

@@ -1,10 +1,8 @@
-if printf "%s\n" "${ALLOW_ADMIN_EMAIL_LOGIN}" | grep -E '^([yY][eE][sS]|[yY])+$' >/dev/null; then
-    echo 'auth_request /sogo-auth-verify;
+auth_request /sogo-auth-verify;
 auth_request_set $user $upstream_http_x_user;
 auth_request_set $auth $upstream_http_x_auth;
 auth_request_set $auth_type $upstream_http_x_auth_type;
 proxy_set_header x-webobjects-remote-user "$user";
 proxy_set_header Authorization "$auth";
 proxy_set_header x-webobjects-auth-type "$auth_type";
-'
-fi
+

data/conf/nginx/includes/ssl_client_auth.conf

@@ -0,0 +1,4 @@
+
+ssl_verify_client      optional;
+ssl_client_certificate /etc/nginx/conf.d/client_cas.crt;
+

data/conf/nginx/mailcow_auth.conf

@@ -0,0 +1,23 @@
+server {
+  listen 9082 ssl http2;
+
+  ssl_certificate /etc/ssl/mail/cert.pem;
+  ssl_certificate_key /etc/ssl/mail/key.pem;
+
+  index mailcowauth.php;
+  server_name _;
+  error_log  /var/log/nginx/error.log;
+  access_log /var/log/nginx/access.log;
+  root /mailcowauth;
+  client_max_body_size 10M;
+
+  location ~ \.php$ {
+    client_max_body_size 10M;
+    try_files $uri =404;
+    fastcgi_split_path_info ^(.+\.php)(/.+)$;
+    fastcgi_pass phpfpm:9001;
+    include fastcgi_params;
+    fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+    fastcgi_param PATH_INFO $fastcgi_path_info;
+  }
+}

data/conf/nginx/site.conf

@@ -1,6 +1,6 @@
-server_tokens off;
 proxy_cache_path /tmp levels=1:2 keys_zone=sogo:10m inactive=24h  max_size=1g;
-server_names_hash_bucket_size 64;
+server_names_hash_max_size 512;
+server_names_hash_bucket_size 128;
 
 map $http_x_forwarded_proto $client_req_scheme {
      default $scheme;

data/conf/nginx/templates/listen_ssl.template

@@ -1,2 +1,3 @@
-listen ${HTTPS_PORT} ssl http2;
-listen [::]:${HTTPS_PORT} ssl http2;
+listen ${HTTPS_PORT} ssl;
+listen [::]:${HTTPS_PORT} ssl;
+http2 on;

data/conf/nginx/templates/server_name.template

@@ -1 +0,0 @@
-server_name ${MAILCOW_HOSTNAME} autodiscover.* autoconfig.*;

data/conf/nginx/templates/server_name.template.sh

@@ -0,0 +1 @@
+echo "server_name ${MAILCOW_HOSTNAME} autodiscover.* autoconfig.* $(echo ${ADDITIONAL_SERVER_NAMES} | tr ',' ' ');"