diff --git a/docs/source/elasticsearch.md b/docs/source/elasticsearch.md index 4408f79f..50bc984f 100644 --- a/docs/source/elasticsearch.md +++ b/docs/source/elasticsearch.md @@ -256,7 +256,11 @@ matches only documents that have at least one DKIM or SPF auth result and lack the corresponding combined field; documents with no auth results are skipped, because an `exists` query cannot see an empty array, and for search purposes an empty `dkim_results_combined` is identical to an -absent one. +absent one. Each result is matched on either its `domain` or its `result` +subfield as defense in depth: an empty string indexes no text tokens and +is invisible to `exists`, and the storage shape of every historical +parsedmarc version can't be audited, so matching either subfield ensures +no backfillable document is skipped. ```bash curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false" \ @@ -268,13 +272,33 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts= "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}} + ] + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}] } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}} + ] + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}] } } diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 2f5dccc8..5c7ebfc0 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -44,23 +44,48 @@ _SERVERLESS_REJECTED_SETTINGS = frozenset({"number_of_shards", "number_of_replic # Guard query for the dkim_results_combined/spf_results_combined backfill # (see ``migrate_indexes``). Matches only documents that have at least one # DKIM or SPF auth result and are missing the corresponding combined field. -# Empty arrays are invisible to ``exists``, so documents with zero DKIM/SPF -# results are correctly skipped, making this idempotent: once a document is -# backfilled it no longer matches, and re-running against an already -# up-to-date index counts 0. +# Empty arrays are invisible to ``exists``, so documents with zero +# DKIM/SPF results are correctly skipped (verified against real data; +# this also makes the query idempotent — a backfilled document no longer +# matches). Each result is matched on an OR of its ``domain``/``result`` +# subfields as defense in depth: the parsers we audited never store a +# result without both, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. _COMBINED_BACKFILL_QUERY: dict[str, Any] = { "bool": { "minimum_should_match": 1, "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}], } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}], } }, diff --git a/parsedmarc/opensearch.py b/parsedmarc/opensearch.py index 160b2b93..e353c549 100644 --- a/parsedmarc/opensearch.py +++ b/parsedmarc/opensearch.py @@ -37,23 +37,48 @@ class OpenSearchError(Exception): # Guard query for the dkim_results_combined/spf_results_combined backfill # (see ``migrate_indexes``). Matches only documents that have at least one # DKIM or SPF auth result and are missing the corresponding combined field. -# Empty arrays are invisible to ``exists``, so documents with zero DKIM/SPF -# results are correctly skipped, making this idempotent: once a document is -# backfilled it no longer matches, and re-running against an already -# up-to-date index counts 0. +# Empty arrays are invisible to ``exists``, so documents with zero +# DKIM/SPF results are correctly skipped (verified against real data; +# this also makes the query idempotent — a backfilled document no longer +# matches). Each result is matched on an OR of its ``domain``/``result`` +# subfields as defense in depth: the parsers we audited never store a +# result without both, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. _COMBINED_BACKFILL_QUERY: dict[str, Any] = { "bool": { "minimum_should_match": 1, "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}], } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}}, + ], + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}], } },