From 5c0192e7204d83c957107aa744b92b125a3b8872 Mon Sep 17 00:00:00 2001 From: Sean Whalen <44679+seanthegeek@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:17:24 -0400 Subject: [PATCH] Match backfill guard on either domain or result subfield MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end upgrade testing (real parsedmarc 10.2.4 ingest, then a branch startup) surfaced that an exists query cannot see an empty string: a text field with no tokens is invisible to exists. The parsers we audited never store an auth result with an empty or missing domain — they drop such entries entirely, so the previous domain-only guard was sufficient for their data — but the storage shape of every historical parsedmarc version can't be audited, so the guard (and the documented manual command) now matches either the domain or the result subfield per protocol. Matching either costs nothing and cannot skip a document that has something to backfill. Verified by recomputing expected combined values from _source for all 2,299 documents on both engines: every document with stored auth results has exactly the recomputed pairs, and the legacy fo migration correctly did not fire on typeless indexes. Co-Authored-By: Claude Fable 5 --- docs/source/elasticsearch.md | 30 ++++++++++++++++++++++++++--- parsedmarc/elastic.py | 37 ++++++++++++++++++++++++++++++------ parsedmarc/opensearch.py | 37 ++++++++++++++++++++++++++++++------ 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/docs/source/elasticsearch.md b/docs/source/elasticsearch.md index 4408f79f..50bc984f 100644 --- a/docs/source/elasticsearch.md +++ b/docs/source/elasticsearch.md @@ -256,7 +256,11 @@ matches only documents that have at least one DKIM or SPF auth result and lack the corresponding combined field; documents with no auth results are skipped, because an `exists` query cannot see an empty array, and for search purposes an empty `dkim_results_combined` is identical to an -absent one. +absent one. Each result is matched on either its `domain` or its `result` +subfield as defense in depth: an empty string indexes no text tokens and +is invisible to `exists`, and the storage shape of every historical +parsedmarc version can't be audited, so matching either subfield ensures +no backfillable document is skipped. ```bash curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false" \ @@ -268,13 +272,33 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts= "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}} + ] + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}] } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}} + ] + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}] } } diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 2f5dccc8..5c7ebfc0 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -44,23 +44,48 @@ _SERVERLESS_REJECTED_SETTINGS = frozenset({"number_of_shards", "number_of_replic # Guard query for the dkim_results_combined/spf_results_combined backfill # (see ``migrate_indexes``). Matches only documents that have at least one # DKIM or SPF auth result and are missing the corresponding combined field. -# Empty arrays are invisible to ``exists``, so documents with zero DKIM/SPF -# results are correctly skipped, making this idempotent: once a document is -# backfilled it no longer matches, and re-running against an already -# up-to-date index counts 0. +# Empty arrays are invisible to ``exists``, so documents with zero +# DKIM/SPF results are correctly skipped (verified against real data; +# this also makes the query idempotent — a backfilled document no longer +# matches). Each result is matched on an OR of its ``domain``/``result`` +# subfields as defense in depth: the parsers we audited never store a +# result without both, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. _COMBINED_BACKFILL_QUERY: dict[str, Any] = { "bool": { "minimum_should_match": 1, "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}], } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}], } }, diff --git a/parsedmarc/opensearch.py b/parsedmarc/opensearch.py index 160b2b93..e353c549 100644 --- a/parsedmarc/opensearch.py +++ b/parsedmarc/opensearch.py @@ -37,23 +37,48 @@ class OpenSearchError(Exception): # Guard query for the dkim_results_combined/spf_results_combined backfill # (see ``migrate_indexes``). Matches only documents that have at least one # DKIM or SPF auth result and are missing the corresponding combined field. -# Empty arrays are invisible to ``exists``, so documents with zero DKIM/SPF -# results are correctly skipped, making this idempotent: once a document is -# backfilled it no longer matches, and re-running against an already -# up-to-date index counts 0. +# Empty arrays are invisible to ``exists``, so documents with zero +# DKIM/SPF results are correctly skipped (verified against real data; +# this also makes the query idempotent — a backfilled document no longer +# matches). Each result is matched on an OR of its ``domain``/``result`` +# subfields as defense in depth: the parsers we audited never store a +# result without both, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. _COMBINED_BACKFILL_QUERY: dict[str, Any] = { "bool": { "minimum_should_match": 1, "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}], } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}], } },