amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-08-15 11:53:16 +00:00
Code Issues Projects Releases Wiki Activity

Match backfill guard on either domain or result subfield

Browse Source 
End-to-end upgrade testing (real parsedmarc 10.2.4 ingest, then a
branch startup) surfaced that an exists query cannot see an empty
string: a text field with no tokens is invisible to exists. The
parsers we audited never store an auth result with an empty or
missing domain — they drop such entries entirely, so the previous
domain-only guard was sufficient for their data — but the storage
shape of every historical parsedmarc version can't be audited, so
the guard (and the documented manual command) now matches either
the domain or the result subfield per protocol. Matching either
costs nothing and cannot skip a document that has something to
backfill.

Verified by recomputing expected combined values from _source for
all 2,299 documents on both engines: every document with stored
auth results has exactly the recomputed pairs, and the legacy fo
migration correctly did not fire on typeless indexes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sean Whalen
2026-07-22 14:17:24 -04:00
co-authored by Claude Fable 5
parent bbee148d2a
commit 5c0192e720
3 changed files with 89 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/source/elasticsearch.md
+27 -3
View File
@@ -256,7 +256,11 @@ matches only documents that have at least one DKIM or SPF auth result and
lack the corresponding combined field; documents with no auth results are
skipped, because an `exists` query cannot see an empty array, and for
search purposes an empty `dkim_results_combined` is identical to an
absent one.
absent one. Each result is matched on either its `domain` or its `result`
subfield as defense in depth: an empty string indexes no text tokens and
is invisible to `exists`, and the storage shape of every historical
parsedmarc version can't be audited, so matching either subfield ensures
no backfillable document is skipped.
```bash
curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false" \
@@ -268,13 +272,33 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=
"should": [
{
"bool": {
"must": [{"exists": {"field": "dkim_results.domain"}}],
"must": [
{
"bool": {
"minimum_should_match": 1,
"should": [
{"exists": {"field": "dkim_results.domain"}},
{"exists": {"field": "dkim_results.result"}}
]
}
}
],
"must_not": [{"exists": {"field": "dkim_results_combined"}}]
}
},
{
"bool": {
"must": [{"exists": {"field": "spf_results.domain"}}],
"must": [
{
"bool": {
"minimum_should_match": 1,
"should": [
{"exists": {"field": "spf_results.domain"}},
{"exists": {"field": "spf_results.result"}}
]
}
}
],
"must_not": [{"exists": {"field": "spf_results_combined"}}]
}
}
parsedmarc/elastic.py
+31 -6
View File
@@ -44,23 +44,48 @@ _SERVERLESS_REJECTED_SETTINGS = frozenset({"number_of_shards", "number_of_replic
# Guard query for the dkim_results_combined/spf_results_combined backfill
# (see ``migrate_indexes``). Matches only documents that have at least one
# DKIM or SPF auth result and are missing the corresponding combined field.
# Empty arrays are invisible to ``exists``, so documents with zero DKIM/SPF
# results are correctly skipped, making this idempotent: once a document is
# backfilled it no longer matches, and re-running against an already
# up-to-date index counts 0.
# Empty arrays are invisible to ``exists``, so documents with zero
# DKIM/SPF results are correctly skipped (verified against real data;
# this also makes the query idempotent — a backfilled document no longer
# matches). Each result is matched on an OR of its ``domain``/``result``
# subfields as defense in depth: the parsers we audited never store a
# result without both, but an empty string indexes no text tokens and is
# invisible to ``exists``, and the storage shape of every historical
# parsedmarc version can't be audited — matching either subfield costs
# nothing and cannot skip a document that has something to backfill.
_COMBINED_BACKFILL_QUERY: dict[str, Any] = {
"bool": {
"minimum_should_match": 1,
"should": [
{
"bool": {
"must": [{"exists": {"field": "dkim_results.domain"}}],
"must": [
{
"bool": {
"minimum_should_match": 1,
"should": [
{"exists": {"field": "dkim_results.domain"}},
{"exists": {"field": "dkim_results.result"}},
],
}
}
],
"must_not": [{"exists": {"field": "dkim_results_combined"}}],
}
},
{
"bool": {
"must": [{"exists": {"field": "spf_results.domain"}}],
"must": [
{
"bool": {
"minimum_should_match": 1,
"should": [
{"exists": {"field": "spf_results.domain"}},
{"exists": {"field": "spf_results.result"}},
],
}
}
],
"must_not": [{"exists": {"field": "spf_results_combined"}}],
}
},
parsedmarc/opensearch.py
+31 -6
View File
@@ -37,23 +37,48 @@ class OpenSearchError(Exception):
# Guard query for the dkim_results_combined/spf_results_combined backfill
# (see ``migrate_indexes``). Matches only documents that have at least one
# DKIM or SPF auth result and are missing the corresponding combined field.
# Empty arrays are invisible to ``exists``, so documents with zero DKIM/SPF
# results are correctly skipped, making this idempotent: once a document is
# backfilled it no longer matches, and re-running against an already
# up-to-date index counts 0.
# Empty arrays are invisible to ``exists``, so documents with zero
# DKIM/SPF results are correctly skipped (verified against real data;
# this also makes the query idempotent — a backfilled document no longer
# matches). Each result is matched on an OR of its ``domain``/``result``
# subfields as defense in depth: the parsers we audited never store a
# result without both, but an empty string indexes no text tokens and is
# invisible to ``exists``, and the storage shape of every historical
# parsedmarc version can't be audited — matching either subfield costs
# nothing and cannot skip a document that has something to backfill.
_COMBINED_BACKFILL_QUERY: dict[str, Any] = {
"bool": {
"minimum_should_match": 1,
"should": [
{
"bool": {
"must": [{"exists": {"field": "dkim_results.domain"}}],
"must": [
{
"bool": {
"minimum_should_match": 1,
"should": [
{"exists": {"field": "dkim_results.domain"}},
{"exists": {"field": "dkim_results.result"}},
],
}
}
],
"must_not": [{"exists": {"field": "dkim_results_combined"}}],
}
},
{
"bool": {
"must": [{"exists": {"field": "spf_results.domain"}}],
"must": [
{
"bool": {
"minimum_should_match": 1,
"should": [
{"exists": {"field": "spf_results.domain"}},
{"exists": {"field": "spf_results.result"}},
],
}
}
],
"must_not": [{"exists": {"field": "spf_results_combined"}}],
}
},