mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-02-17 07:03:58 +00:00
8.19.0 (#622)
8.19.0 - Add multi-tenant support via an index-prefix domain mapping file - PSL overrides so that services like AWS are correctly identified - Additional improvements to report type detection - Fix webhook timeout parsing (PR #623) - Output to STDOUT when the new general config boolean `silent` is set to `False` (Close #614) - Additional services added to `base_reverse_dns_map.csv` --------- Co-authored-by: Sean Whalen <seanthegeek@users.noreply.github.com> Co-authored-by: Félix <felix.debloisbeaucage@gmail.com>
This commit is contained in:
Sean Whalen
GitHub
12
CHANGELOG.md
12
CHANGELOG.md
@@ -1,7 +1,17 @@
|
||||
Changelog
|
||||
=========
|
||||
|
||||
8.10.9
|
||||
8.19.0
|
||||
------
|
||||
|
||||
- Add multi-tenant support via an index-prefix domain mapping file
|
||||
- PSL overrides so that services like AWS are correctly identified
|
||||
- Additional improvements to report type detection
|
||||
- Fix webhook timeout parsing (PR #623)
|
||||
- Output to STDOUT when the new general config boolean `silent` is set to `False` (Close #614)
|
||||
- Additional services added to `base_reverse_dns_map.csv`
|
||||
|
||||
8.18.9
|
||||
------
|
||||
|
||||
- Complete fix for #687 and more robust report type detection
|
||||
|
||||
@@ -120,8 +120,10 @@ The full set of configuration options are:
|
||||
Elasticsearch, Splunk and/or S3
|
||||
- `save_smtp_tls` - bool: Save SMTP-STS report data to
|
||||
Elasticsearch, Splunk and/or S3
|
||||
- `index_prefix_domain_map` - bool: A path mapping of Opensearch/Elasticsearch index prefixes to domain names
|
||||
- `strip_attachment_payloads` - bool: Remove attachment
|
||||
payloads from results
|
||||
- `silent` - bool: Set this to `False` to output results to STDOUT
|
||||
- `output` - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options.
|
||||
- `aggregate_json_filename` - str: filename for the aggregate
|
||||
JSON output file
|
||||
@@ -445,6 +447,28 @@ PUT _cluster/settings
|
||||
Increasing this value increases resource usage.
|
||||
:::
|
||||
|
||||
## Multi-tenant support
|
||||
|
||||
Starting in `8.19.0`, ParseDMARC provides multi-tenant support by placing data into separate OpenSearch or Elasticsearch index prefixes. To set this up, create a YAML file that is formatted where each key is a tenant name, and the value is a list of domains related to that tenant, not including subdomains, like this:
|
||||
|
||||
```yaml
|
||||
example:
|
||||
- example.com
|
||||
- example.net
|
||||
- example.org
|
||||
|
||||
whalensolutions:
|
||||
- whalensolutions.com
|
||||
```
|
||||
|
||||
Save it to disk where the user running ParseDMARC can read it, then set `index_prefix_domain_map` to that filepath in the `[general]` section of the ParseDMARC configuration file and do not set an `index_prefix` option in the `[elasticsearch]` or `[opensearch]` sections.
|
||||
|
||||
When configured correctly, if ParseDMARC finds that a report is related to a domain in the mapping, the report will be saved in an index name that has the tenant name prefixed to it with a trailing underscore. Then, you can use the security features of Opensearch or the ELK stack to only grant users access to the indexes that they need.
|
||||
|
||||
:::{note}
|
||||
A domain cannot be used in multiple tenant lists. Only the first prefix list that contains the matching domain is used.
|
||||
:::
|
||||
|
||||
## Running parsedmarc as a systemd service
|
||||
|
||||
Use systemd to run `parsedmarc` as a service and process reports as
|
||||
|
||||
@@ -1200,12 +1200,14 @@ def parse_report_email(
|
||||
if "Subject" in msg_headers:
|
||||
subject = msg_headers["Subject"]
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_type()
|
||||
content_type = part.get_content_type().lower()
|
||||
payload = part.get_payload()
|
||||
if not isinstance(payload, list):
|
||||
payload = [payload]
|
||||
payload = payload[0].__str__()
|
||||
if content_type == "message/feedback-report":
|
||||
if content_type.startswith("multipart/"):
|
||||
continue
|
||||
elif content_type == "message/feedback-report":
|
||||
try:
|
||||
if "Feedback-Type" in payload:
|
||||
feedback_report = payload
|
||||
@@ -1216,13 +1218,12 @@ def parse_report_email(
|
||||
feedback_report = feedback_report.replace("\\n", "\n")
|
||||
except (ValueError, TypeError, binascii.Error):
|
||||
feedback_report = payload
|
||||
|
||||
elif content_type == "text/rfc822-headers":
|
||||
sample = payload
|
||||
elif content_type == "message/rfc822":
|
||||
sample = payload
|
||||
elif content_type == "application/tlsrpt+json":
|
||||
if "{" not in payload:
|
||||
if not payload.strip().startswith("{"):
|
||||
payload = str(b64decode(payload))
|
||||
smtp_tls_report = parse_smtp_tls_report_json(payload)
|
||||
return OrderedDict(
|
||||
@@ -1234,7 +1235,6 @@ def parse_report_email(
|
||||
return OrderedDict(
|
||||
[("report_type", "smtp_tls"), ("report", smtp_tls_report)]
|
||||
)
|
||||
|
||||
elif content_type == "text/plain":
|
||||
if "A message claiming to be from you has failed" in payload:
|
||||
try:
|
||||
@@ -1263,7 +1263,7 @@ def parse_report_email(
|
||||
payload = extract_report(payload)
|
||||
if isinstance(payload, bytes):
|
||||
payload = payload.decode("utf-8", errors="replace")
|
||||
if payload.startswith("{"):
|
||||
if payload.strip().startswith("{"):
|
||||
smtp_tls_report = parse_smtp_tls_report_json(payload)
|
||||
result = OrderedDict(
|
||||
[("report_type", "smtp_tls"), ("report", smtp_tls_report)]
|
||||
|
||||
@@ -9,6 +9,7 @@ from configparser import ConfigParser
|
||||
from glob import glob
|
||||
import logging
|
||||
import math
|
||||
import yaml
|
||||
from collections import OrderedDict
|
||||
import json
|
||||
from ssl import CERT_NONE, create_default_context
|
||||
@@ -46,7 +47,7 @@ from parsedmarc.mail import (
|
||||
from parsedmarc.mail.graph import AuthMethod
|
||||
|
||||
from parsedmarc.log import logger
|
||||
from parsedmarc.utils import is_mbox, get_reverse_dns
|
||||
from parsedmarc.utils import is_mbox, get_reverse_dns, get_base_domain
|
||||
from parsedmarc import SEEN_AGGREGATE_REPORT_IDS
|
||||
|
||||
http.client._MAXHEADERS = 200 # pylint:disable=protected-access
|
||||
@@ -101,6 +102,30 @@ def cli_parse(
|
||||
def _main():
|
||||
"""Called when the module is executed"""
|
||||
|
||||
def get_index_prefix(report):
|
||||
if index_prefix_domain_map is None:
|
||||
return None
|
||||
if "policy_published" in report:
|
||||
domain = report["policy_published"]["domain"]
|
||||
elif "reported_domain" in report:
|
||||
domain = report("reported_domain")
|
||||
elif "policies" in report:
|
||||
domain = report["policies"][0]["domain"]
|
||||
if domain:
|
||||
domain = get_base_domain(domain)
|
||||
for prefix in index_prefix_domain_map:
|
||||
if domain in index_prefix_domain_map[prefix]:
|
||||
prefix = (
|
||||
prefix.lower()
|
||||
.strip()
|
||||
.strip("_")
|
||||
.replace(" ", "_")
|
||||
.replace("-", "_")
|
||||
)
|
||||
prefix = f"{prefix}_"
|
||||
return prefix
|
||||
return None
|
||||
|
||||
def process_reports(reports_):
|
||||
indent_value = 2 if opts.prettify_json else None
|
||||
output_str = "{0}\n".format(
|
||||
@@ -129,7 +154,8 @@ def _main():
|
||||
elastic.save_aggregate_report_to_elasticsearch(
|
||||
report,
|
||||
index_suffix=opts.elasticsearch_index_suffix,
|
||||
index_prefix=opts.elasticsearch_index_prefix,
|
||||
index_prefix=opts.elasticsearch_index_prefix
|
||||
or get_index_prefix(report),
|
||||
monthly_indexes=opts.elasticsearch_monthly_indexes,
|
||||
number_of_shards=shards,
|
||||
number_of_replicas=replicas,
|
||||
@@ -150,7 +176,8 @@ def _main():
|
||||
opensearch.save_aggregate_report_to_opensearch(
|
||||
report,
|
||||
index_suffix=opts.opensearch_index_suffix,
|
||||
index_prefix=opts.opensearch_index_prefix,
|
||||
index_prefix=opts.opensearch_index_prefix
|
||||
or get_index_prefix(report),
|
||||
monthly_indexes=opts.opensearch_monthly_indexes,
|
||||
number_of_shards=shards,
|
||||
number_of_replicas=replicas,
|
||||
@@ -216,7 +243,8 @@ def _main():
|
||||
elastic.save_forensic_report_to_elasticsearch(
|
||||
report,
|
||||
index_suffix=opts.elasticsearch_index_suffix,
|
||||
index_prefix=opts.elasticsearch_index_prefix,
|
||||
index_prefix=opts.elasticsearch_index_prefix
|
||||
or get_index_prefix(report),
|
||||
monthly_indexes=opts.elasticsearch_monthly_indexes,
|
||||
number_of_shards=shards,
|
||||
number_of_replicas=replicas,
|
||||
@@ -235,7 +263,8 @@ def _main():
|
||||
opensearch.save_forensic_report_to_opensearch(
|
||||
report,
|
||||
index_suffix=opts.opensearch_index_suffix,
|
||||
index_prefix=opts.opensearch_index_prefix,
|
||||
index_prefix=opts.opensearch_index_prefix
|
||||
or get_index_prefix(report),
|
||||
monthly_indexes=opts.opensearch_monthly_indexes,
|
||||
number_of_shards=shards,
|
||||
number_of_replicas=replicas,
|
||||
@@ -299,7 +328,8 @@ def _main():
|
||||
elastic.save_smtp_tls_report_to_elasticsearch(
|
||||
report,
|
||||
index_suffix=opts.elasticsearch_index_suffix,
|
||||
index_prefix=opts.elasticsearch_index_prefix,
|
||||
index_prefix=opts.elasticsearch_index_prefix
|
||||
or get_index_prefix(report),
|
||||
monthly_indexes=opts.elasticsearch_monthly_indexes,
|
||||
number_of_shards=shards,
|
||||
number_of_replicas=replicas,
|
||||
@@ -318,7 +348,8 @@ def _main():
|
||||
opensearch.save_smtp_tls_report_to_opensearch(
|
||||
report,
|
||||
index_suffix=opts.opensearch_index_suffix,
|
||||
index_prefix=opts.opensearch_index_prefix,
|
||||
index_prefix=opts.opensearch_index_prefix
|
||||
or get_index_prefix(report),
|
||||
monthly_indexes=opts.opensearch_monthly_indexes,
|
||||
number_of_shards=shards,
|
||||
number_of_replicas=replicas,
|
||||
@@ -638,9 +669,16 @@ def _main():
|
||||
exit(-1)
|
||||
opts.silent = True
|
||||
config = ConfigParser()
|
||||
index_prefix_domain_map = None
|
||||
config.read(args.config_file)
|
||||
if "general" in config.sections():
|
||||
general_config = config["general"]
|
||||
if "silent" in general_config:
|
||||
if general_config["silent"].lower() == "false":
|
||||
opts.silent = False
|
||||
if "index_prefix_domain_map" in general_config:
|
||||
with open(general_config["index_prefix_domain_map"]) as f:
|
||||
index_prefix_domain_map = yaml.safe_load(f)
|
||||
if "offline" in general_config:
|
||||
opts.offline = general_config.getboolean("offline")
|
||||
if "strip_attachment_payloads" in general_config:
|
||||
@@ -1182,7 +1220,7 @@ def _main():
|
||||
if "smtp_tls_url" in webhook_config:
|
||||
opts.webhook_smtp_tls_url = webhook_config["smtp_tls_url"]
|
||||
if "timeout" in webhook_config:
|
||||
opts.webhook_timeout = webhook_config["timeout"]
|
||||
opts.webhook_timeout = webhook_config.getint("timeout")
|
||||
|
||||
logger.setLevel(logging.ERROR)
|
||||
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
__version__ = "8.18.9"
|
||||
__version__ = "8.19.0"
|
||||
USER_AGENT = f"parsedmarc/{__version__}"
|
||||
|
||||
@@ -427,7 +427,9 @@ def save_aggregate_report_to_elasticsearch(
|
||||
except Exception as error_:
|
||||
raise ElasticsearchError(
|
||||
"Elasticsearch's search for existing report \
|
||||
error: {}".format(error_.__str__())
|
||||
error: {}".format(
|
||||
error_.__str__()
|
||||
)
|
||||
)
|
||||
|
||||
if len(existing) > 0:
|
||||
@@ -740,7 +742,9 @@ def save_smtp_tls_report_to_elasticsearch(
|
||||
except Exception as error_:
|
||||
raise ElasticsearchError(
|
||||
"Elasticsearch's search for existing report \
|
||||
error: {}".format(error_.__str__())
|
||||
error: {}".format(
|
||||
error_.__str__()
|
||||
)
|
||||
)
|
||||
|
||||
if len(existing) > 0:
|
||||
|
||||
@@ -427,7 +427,9 @@ def save_aggregate_report_to_opensearch(
|
||||
except Exception as error_:
|
||||
raise OpenSearchError(
|
||||
"OpenSearch's search for existing report \
|
||||
error: {}".format(error_.__str__())
|
||||
error: {}".format(
|
||||
error_.__str__()
|
||||
)
|
||||
)
|
||||
|
||||
if len(existing) > 0:
|
||||
@@ -740,7 +742,9 @@ def save_smtp_tls_report_to_opensearch(
|
||||
except Exception as error_:
|
||||
raise OpenSearchError(
|
||||
"OpenSearch's search for existing report \
|
||||
error: {}".format(error_.__str__())
|
||||
error: {}".format(
|
||||
error_.__str__()
|
||||
)
|
||||
)
|
||||
|
||||
if len(existing) > 0:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ def _main():
|
||||
if domain in list_var:
|
||||
print(f"Error: {domain} is in {file_path} multiple times")
|
||||
exit(1)
|
||||
else:
|
||||
elif domain != "":
|
||||
list_var.append(domain)
|
||||
|
||||
load_list(known_unknown_list_file_path, known_unknown_domains)
|
||||
@@ -64,7 +64,7 @@ def _main():
|
||||
continue
|
||||
for psl_domain in psl_overrides:
|
||||
if domain.endswith(psl_domain):
|
||||
domain = psl_domain
|
||||
domain = psl_domain.strip(".").strip("-")
|
||||
break
|
||||
if domain not in known_domains and domain not in known_unknown_domains:
|
||||
print(f"New unknown domain found: {domain}")
|
||||
@@ -75,5 +75,6 @@ def _main():
|
||||
writer.writeheader()
|
||||
writer.writerows(output_rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main()
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
185.in-addr.arpa
|
||||
190.in-addr.arpa
|
||||
200.in-addr.arpa
|
||||
1jli.site
|
||||
26.107
|
||||
444qcuhilla.com
|
||||
4xr1.com
|
||||
9services.com
|
||||
a7e.ru
|
||||
a94434500-blog.com
|
||||
aams8.jp
|
||||
abv-10.top
|
||||
acemail.co.in
|
||||
activaicon.com
|
||||
adcritic.net
|
||||
adlucrumnewsletter.com
|
||||
admin.corpivensa.gob.ve
|
||||
@@ -18,8 +21,10 @@ ai270.net
|
||||
albagroup-eg.com
|
||||
alchemy.net
|
||||
alohabeachcamp.net
|
||||
alsiscad.com
|
||||
aluminumpipetubing.com
|
||||
americanstorageca.com
|
||||
amplusserver.info
|
||||
anchorfundhub.com
|
||||
anglishment.com
|
||||
anteldata.net.uy
|
||||
@@ -31,118 +36,186 @@ aosau.net
|
||||
arandomserver.com
|
||||
aransk.ru
|
||||
ardcs.cn
|
||||
armninl.met
|
||||
as29550.net
|
||||
asahachimaru.com
|
||||
aserv.co.za
|
||||
asmecam.it
|
||||
ateky.net.br
|
||||
aurelienvos.com
|
||||
automatech.lat
|
||||
avistaadvantage.com
|
||||
b8sales.com
|
||||
bahjs.com
|
||||
baliaura.com
|
||||
banaras.co
|
||||
bearandbullmarketnews.com
|
||||
bestinvestingtime.com
|
||||
bhjui.com
|
||||
biocorp.com
|
||||
biosophy.net
|
||||
bitter-echo.com
|
||||
bizhostingservices.com
|
||||
blguss.com
|
||||
bluenet.ch
|
||||
bluhosting.com
|
||||
bnasg.com
|
||||
bodiax.pp.ua
|
||||
bost-law.com
|
||||
brainity.com
|
||||
brazalnde.net
|
||||
brellatransplc.shop
|
||||
brnonet.cz
|
||||
broadwaycover.com
|
||||
brushinglegal.de
|
||||
brw.net
|
||||
btes.tv
|
||||
budgeteasehub.com
|
||||
buoytoys.com
|
||||
buyjapanese.jp
|
||||
c53dw7m24rj.com
|
||||
cahtelrandom.org
|
||||
casadelmarsamara.com
|
||||
cashflowmasterypro.com
|
||||
cavabeen.com
|
||||
cbti.net
|
||||
centralmalaysia.com
|
||||
chauffeurplan.co.uk
|
||||
checkpox.fun
|
||||
chegouseuvlache.org
|
||||
chinaxingyu.xyz
|
||||
christus.mx
|
||||
churchills.market
|
||||
ci-xyz.fit
|
||||
cisumrecords.com
|
||||
ckaik.cn
|
||||
clcktoact.com
|
||||
cli-eurosignal.cz
|
||||
cloud-admin.it
|
||||
cloud-edm.com
|
||||
cloudaccess.net
|
||||
cloudflare-email.org
|
||||
cloudhosting.rs
|
||||
cloudlogin.co
|
||||
cloudplatformpro.com
|
||||
cnode.io
|
||||
cntcloud.com
|
||||
code-it.net
|
||||
codefriend.top
|
||||
colombiaceropapel.org
|
||||
commerceinsurance.com
|
||||
comsharempc.com
|
||||
conexiona.com
|
||||
coolblaze.com
|
||||
coowo.com
|
||||
corpemail.net
|
||||
cp2-myorderbox.com
|
||||
cps.com.ar
|
||||
crnagora.net
|
||||
cross-d-bar-troutranch.com
|
||||
ctla.co.kr
|
||||
cumbalikonakhotel.com
|
||||
currencyexconverter.com
|
||||
daakbabu.com
|
||||
daikinmae.com
|
||||
dairyvalley.com.my
|
||||
dastans.ru
|
||||
datahost36.de
|
||||
ddii.network
|
||||
deep-sek.shop
|
||||
deetownsounds.com
|
||||
descarca-counter-strike.net
|
||||
detrot.xyz
|
||||
dettlaffinc.com
|
||||
dextoolse.net
|
||||
digestivedaily.com
|
||||
digi.net.my
|
||||
dinofelis.cn
|
||||
diwkyncbi.top
|
||||
dkginternet.com
|
||||
dnexpress.info
|
||||
dns-oid.com
|
||||
dnsindia.net
|
||||
domainserver.ne.jp
|
||||
domconfig.com
|
||||
doorsrv.com
|
||||
dreampox.fun
|
||||
dreamtechmedia.com
|
||||
ds.network
|
||||
dss-group.net
|
||||
dvj.theworkpc.com
|
||||
dwlcka.com
|
||||
dynamic-wiretel.in
|
||||
dyntcorp.com
|
||||
easternkingspei.com
|
||||
economiceagles.com
|
||||
egosimail.com
|
||||
eliotporterphotos.us
|
||||
emailgids.net
|
||||
emailperegrine.com
|
||||
entendercopilot.com
|
||||
entretothom.net
|
||||
epaycontrol.com
|
||||
epicinvestmentsreview.co
|
||||
epicinvestmentsreview.com
|
||||
epik.com
|
||||
epsilon-group.com
|
||||
erestaff.com
|
||||
euro-trade-gmbh.com
|
||||
example.com
|
||||
exposervers.com-new
|
||||
extendcp.co.uk
|
||||
eyecandyhosting.xyz
|
||||
fastwebnet.it
|
||||
fd9ing7wfn.com
|
||||
feipnghardware.com
|
||||
fetscorp.shop
|
||||
fewo-usedom.net
|
||||
fin-crime.com
|
||||
financeaimpoint.com
|
||||
financeupward.com
|
||||
firmflat.com
|
||||
flex-video.bnr.la
|
||||
flourishfusionlife.com
|
||||
formicidaehunt.net
|
||||
fosterheap.com
|
||||
fredi.shop
|
||||
frontiernet.net
|
||||
ftifb7tk3c.com
|
||||
gamersprotectionvpn.online
|
||||
gendns.com
|
||||
getgreencardsfast.com
|
||||
getthatroi.com
|
||||
gibbshosting.com
|
||||
gigidea.net
|
||||
giize.com
|
||||
ginous.eu.com
|
||||
gis.net
|
||||
gist-th.com
|
||||
globalglennpartners.com
|
||||
goldsboroughplace.com
|
||||
gophermedia.com
|
||||
gqlists.us.com
|
||||
gratzl.de
|
||||
greatestworldnews.com
|
||||
greennutritioncare.com
|
||||
gsbb.com
|
||||
gumbolimbo.net
|
||||
h-serv.co.uk
|
||||
haedefpartners.com
|
||||
halcyon-aboveboard.com
|
||||
hanzubon.org
|
||||
healthfuljourneyjoy.com
|
||||
hgnbroken.us.com
|
||||
highwey-diesel.com
|
||||
hirofactory.com
|
||||
hjd.asso.fr
|
||||
hongchenggco.pro
|
||||
hongkongtaxi.co
|
||||
hopsinthehanger.com
|
||||
hosted-by-worldstream.net
|
||||
hostelsucre.com
|
||||
hosting1337.com
|
||||
hostinghane.com
|
||||
hostinglotus.cloud
|
||||
hostingmichigan.com
|
||||
hostiran.name
|
||||
@@ -150,8 +223,11 @@ hostmnl.com
|
||||
hostname.localhost
|
||||
hostnetwork.com
|
||||
hosts.net.nz
|
||||
hostserv.eu
|
||||
hostwhitelabel.com
|
||||
hpms1.jp
|
||||
hunariojmk.net
|
||||
hunriokinmuim.net
|
||||
hypericine.com
|
||||
i-mecca.net
|
||||
iaasdns.com
|
||||
@@ -159,42 +235,88 @@ iam.net.ma
|
||||
iconmarketingguy.com
|
||||
idcfcloud.net
|
||||
idealconcept.live
|
||||
igmohji.com
|
||||
igppevents.org.uk
|
||||
ihglobaldns.com
|
||||
ilmessicano.com
|
||||
imjtmn.cn
|
||||
immenzaces.com
|
||||
in-addr-arpa
|
||||
in-addr.arpa
|
||||
indsalelimited.com
|
||||
indulgent-holistic.com
|
||||
industechint.org
|
||||
inshaaegypt.com
|
||||
intal.uz
|
||||
interfarma.kz
|
||||
intocpanel.com
|
||||
ip-147-135-108.us
|
||||
ip-178-33-109.eu
|
||||
ip-ptr.tech
|
||||
iswhatpercent.com
|
||||
itsidc.com
|
||||
itwebs.com
|
||||
iuon.net
|
||||
ivol.co
|
||||
jalanet.co.id
|
||||
jimishare.com
|
||||
jlccptt.net.cn
|
||||
jlenterprises.co.uk
|
||||
jmontalto.com
|
||||
joyomokei.com
|
||||
jumanra.org
|
||||
justlongshirts.com
|
||||
kahlaa.com
|
||||
kaw.theworkpc.com
|
||||
kbronet.com.tw
|
||||
kdnursing.org
|
||||
kielnet.net
|
||||
kihy.theworkpc.com
|
||||
kingschurchwirral.org
|
||||
kitchenaildbd.com
|
||||
klaomi.shop
|
||||
knkconsult.net
|
||||
kohshikai.com
|
||||
krhfund.org
|
||||
krillaglass.com
|
||||
lancorhomes.com
|
||||
landpedia.org
|
||||
lanzatuseo.es
|
||||
layerdns.cloud
|
||||
learninglinked.com
|
||||
legenditds.com
|
||||
levertechcentre.com
|
||||
lhost.no
|
||||
lideri.net.br
|
||||
lighthouse-media.com
|
||||
lightpath.net
|
||||
limogesporcelainboxes.com
|
||||
lindsaywalt.net
|
||||
linuxsunucum.com
|
||||
listertermoformadoa.com
|
||||
llsend.com
|
||||
local.net
|
||||
lohkal.com
|
||||
londionrtim.net
|
||||
lonestarmm.net
|
||||
longmarquis.com
|
||||
longwoodmgmt.com
|
||||
lse.kz
|
||||
lunvoy.com
|
||||
luxarpro.ru
|
||||
lwl-puehringer.at
|
||||
lynx.net.lb
|
||||
lyse.net
|
||||
m-sender.com.ua
|
||||
maggiolicloud.it
|
||||
magnetmail.net
|
||||
magnumgo.uz
|
||||
maia11.com
|
||||
mail-fire.com
|
||||
mailsentinel.net
|
||||
mailset.cn
|
||||
malardino.net
|
||||
managed-vps.net
|
||||
manhattanbulletpoint.com
|
||||
manpowerservices.com
|
||||
marketmysterycode.com
|
||||
@@ -204,11 +326,23 @@ matroguel.cam
|
||||
maximpactipo.com
|
||||
mechanicalwalk.store
|
||||
mediavobis.com
|
||||
meqlobal.com
|
||||
mgts.by
|
||||
migrans.net
|
||||
miixta.com
|
||||
milleniumsrv.com
|
||||
mindworksunlimited.com
|
||||
mirth-gale.com
|
||||
misorpresa.com
|
||||
mitomobile.com
|
||||
mitsubachi-kibako.net
|
||||
mjinn.com
|
||||
mkegs.shop
|
||||
mobius.fr
|
||||
model-ac.ink
|
||||
moderntradingnews.com
|
||||
monnaiegroup.com
|
||||
monopolizeright.com
|
||||
moonjaws.com
|
||||
morningnewscatcher.com
|
||||
motion4ever.net
|
||||
@@ -220,18 +354,27 @@ multifamilydesign.com
|
||||
mxserver.ro
|
||||
mxthunder.net
|
||||
my-ihor.ru
|
||||
mycloudmailbox.com
|
||||
myfriendforum.com
|
||||
myrewards.net
|
||||
mysagestore.com
|
||||
mysecurewebserver.com
|
||||
myshanet.net
|
||||
myvps.jp
|
||||
mywedsite.net
|
||||
mywic.eu
|
||||
name.tools
|
||||
nanshenqfurniture.com
|
||||
nask.pl
|
||||
navertise.net
|
||||
ncbb.kz
|
||||
ncport.ru
|
||||
ncsdi.ws
|
||||
nebdig.com
|
||||
neovet-base.ru
|
||||
netbri.com
|
||||
netcentertelecom.net.br
|
||||
neti.ee
|
||||
netkl.org
|
||||
newinvestingguide.com
|
||||
newwallstreetcode.com
|
||||
@@ -242,119 +385,210 @@ nieuwedagnetwerk.net
|
||||
nlscanme.com
|
||||
nmeuh.cn
|
||||
noisndametal.com
|
||||
nucleusemail.com
|
||||
nutriboostlife.com
|
||||
nwo.giize.com
|
||||
nwwhalewatchers.org
|
||||
ny.adsl
|
||||
nyt1.com
|
||||
offerslatedeals.com
|
||||
office365.us
|
||||
ogicom.net
|
||||
olivettilexikon.co.uk
|
||||
omegabrasil.inf.br
|
||||
onnet21.com
|
||||
onumubunumu.com
|
||||
oppt-ac.fit
|
||||
orbitel.net.co
|
||||
orfsurface.com
|
||||
orientalspot.com
|
||||
outsidences.com
|
||||
ovaltinalization.co
|
||||
overta.ru
|
||||
ox28vgrurc.com
|
||||
pamulang.net
|
||||
panaltyspot.space
|
||||
panolacountysheriffms.com
|
||||
passionatesmiles.com
|
||||
paulinelam.com
|
||||
pdi-corp.com
|
||||
peloquinbeck.com
|
||||
perimetercenter.net
|
||||
permanentscreen.com
|
||||
permasteellisagroup.com
|
||||
perumkijhyu.net
|
||||
pesnia.com.ua
|
||||
ph8ltwdi12o.com
|
||||
pharmada.com.de
|
||||
phdns3.es
|
||||
pigelixval1.com
|
||||
pipefittingsindia.com
|
||||
planethoster.net
|
||||
playamedia.io
|
||||
plesk.page
|
||||
pmnhost.net
|
||||
pokiloandhu.net
|
||||
pokupki5.ru
|
||||
polandi.net
|
||||
popiup.com
|
||||
ports.net
|
||||
posolstvostilya.com
|
||||
potia.net
|
||||
prima.com.ar
|
||||
prima.net.ar
|
||||
profsol.co.uk
|
||||
prohealthmotion.com
|
||||
promooffermarket.site
|
||||
proudserver.com
|
||||
proxado.com
|
||||
psnm.ru
|
||||
pvcwindowsprices.live
|
||||
qontenciplc.autos
|
||||
quakeclick.com
|
||||
quasarstate.store
|
||||
quatthonggiotico.com
|
||||
qxyxab44njd.com
|
||||
radianthealthrenaissance.com
|
||||
rapidns.com
|
||||
raxa.host
|
||||
reberte.com
|
||||
reethvikintl.com
|
||||
regruhosting.ru
|
||||
reliablepanel.com
|
||||
rgb365.eu
|
||||
riddlecamera.net
|
||||
riddletrends.com
|
||||
roccopugliese.com
|
||||
runnin-rebels.com
|
||||
rupar.puglia.it
|
||||
rwdhosting.ca
|
||||
s500host.com
|
||||
sageevents.co.ke
|
||||
sahacker-2020.com
|
||||
samsales.site
|
||||
sante-lorraine.fr
|
||||
saransk.ru
|
||||
satirogluet.com
|
||||
scioncontacts.com
|
||||
sdcc.my
|
||||
seaspraymta3.net
|
||||
secorp.mx
|
||||
securen.net
|
||||
securerelay.in
|
||||
securev.net
|
||||
seductiveeyes.com
|
||||
seizethedayconsulting.com
|
||||
serroplast.shop
|
||||
server290.com
|
||||
server342.com
|
||||
server3559.cc
|
||||
servershost.biz
|
||||
sfek.kz
|
||||
sgnetway.net
|
||||
shopfox.ca
|
||||
silvestrejaguar.sbs
|
||||
silvestreonca.sbs
|
||||
simplediagnostics.org
|
||||
siriuscloud.jp
|
||||
sisglobalresearch.com
|
||||
sixpacklink.net
|
||||
sjestyle.com
|
||||
smallvillages.com
|
||||
smartape-vps.com
|
||||
solusoftware.com
|
||||
sourcedns.com
|
||||
southcoastwebhosting12.com
|
||||
specialtvvs.com
|
||||
spiritualtechnologies.io
|
||||
sprout.org
|
||||
srv.cat
|
||||
stableserver.net
|
||||
statlerfa.co.uk
|
||||
stock-smtp.top
|
||||
stockepictigers.com
|
||||
stockexchangejournal.com
|
||||
subterranean-concave.com
|
||||
suksangroup.com
|
||||
swissbluetopaz.com
|
||||
switer.shop
|
||||
sysop4.com
|
||||
system.eu.com
|
||||
szhongbing.com
|
||||
t-jon.com
|
||||
tacaindo.net
|
||||
tacom.tj
|
||||
tankertelz.co
|
||||
tataidc.com
|
||||
teamveiw.com
|
||||
tecnoxia.net
|
||||
tel-xyz.fit
|
||||
tenkids.net
|
||||
terminavalley.com
|
||||
thaicloudsolutions.com
|
||||
thaikinghost.com
|
||||
thaimonster.com
|
||||
thegermainetruth.net
|
||||
thehandmaderose.com
|
||||
thepushcase.com
|
||||
ticdns.com
|
||||
tigo.bo
|
||||
toledofibra.net.br
|
||||
topdns.com
|
||||
totaal.net
|
||||
totalplay.net
|
||||
tqh.ro
|
||||
traderlearningcenter.com
|
||||
tradeukraine.site
|
||||
traveleza.com
|
||||
trwww.com
|
||||
tsuzakij.com
|
||||
tullostrucking.com
|
||||
turbinetrends.com
|
||||
twincitiesdistinctivehomes.com
|
||||
tylerfordonline.com
|
||||
uiyum.com
|
||||
ultragate.com
|
||||
uneedacollie.com
|
||||
unified.services
|
||||
unite.services
|
||||
urawasl.com
|
||||
us.servername.us
|
||||
vagebond.net
|
||||
varvia.de
|
||||
vbcploo.com
|
||||
vdc.vn
|
||||
vendimetry.com
|
||||
vibrantwellnesscorp.com
|
||||
virtualine.org
|
||||
visit.docotor
|
||||
viviotech.us
|
||||
vlflgl.com
|
||||
volganet.ru
|
||||
vrns.net
|
||||
vulterdi.edu
|
||||
vvondertex.com
|
||||
wallstreetsgossip.com
|
||||
wamego.net
|
||||
wanekoohost.com
|
||||
wealthexpertisepro.com
|
||||
web-login.eu
|
||||
weblinkinternational.com
|
||||
webnox.io
|
||||
websale.net
|
||||
welllivinghive.com
|
||||
westparkcom.com
|
||||
wetransfer-eu.com
|
||||
wheelch.me
|
||||
whoflew.com
|
||||
whpservers.com
|
||||
wisdomhard.com
|
||||
wisewealthcircle.com
|
||||
wisvis.com
|
||||
wodeniowa.com
|
||||
wordpresshosting.xyz
|
||||
wsiph2.com
|
||||
xnt.mx
|
||||
xodiax.com
|
||||
xpnuf.cn
|
||||
xsfati.us.com
|
||||
xspmail.jp
|
||||
@@ -362,5 +596,6 @@ yourciviccompass.com
|
||||
yourinvestworkbook.com
|
||||
yoursitesecure.net
|
||||
zerowebhosting.net
|
||||
zmml.uk
|
||||
znlc.jp
|
||||
ztomy.com
|
||||
|
||||
@@ -1,6 +1,23 @@
|
||||
akura.ne.jp
|
||||
amazonaws.com
|
||||
cloudaccess.net
|
||||
h-serv.co.uk
|
||||
linode.com
|
||||
plesk.page
|
||||
-applefibernet.com
|
||||
-c3.net.pl
|
||||
-celsiainternet.com
|
||||
-clientes-izzi.mx
|
||||
-clientes-zap-izzi.mx
|
||||
-imnet.com.br
|
||||
-mcnbd.com
|
||||
-smile.com.bd
|
||||
-tataidc.co.in
|
||||
-veloxfiber.com.br
|
||||
-wconect.com.br
|
||||
.amazonaws.com
|
||||
.cloudaccess.net
|
||||
.ddnsgeek.com
|
||||
.fastvps-server.com
|
||||
.in-addr-arpa
|
||||
.in-addr.arpa
|
||||
.kasserver.com
|
||||
.kinghost.net
|
||||
.linode.com
|
||||
.linodeusercontent.com
|
||||
.na4u.ru
|
||||
.sakura.ne.jp
|
||||
|
||||
@@ -44,6 +44,12 @@ parenthesis_regex = re.compile(r"\s*\(.*\)\s*")
|
||||
null_file = open(os.devnull, "w")
|
||||
mailparser_logger = logging.getLogger("mailparser")
|
||||
mailparser_logger.setLevel(logging.CRITICAL)
|
||||
psl = publicsuffixlist.PublicSuffixList()
|
||||
psl_overrides_path = str(files(parsedmarc.resources.maps).joinpath("psl_overrides.txt"))
|
||||
with open(psl_overrides_path) as f:
|
||||
psl_overrides = [line.rstrip() for line in f.readlines()]
|
||||
while "" in psl_overrides:
|
||||
psl_overrides.remove("")
|
||||
|
||||
|
||||
class EmailParserError(RuntimeError):
|
||||
@@ -78,7 +84,8 @@ def get_base_domain(domain):
|
||||
|
||||
.. note::
|
||||
Results are based on a list of public domain suffixes at
|
||||
https://publicsuffix.org/list/public_suffix_list.dat.
|
||||
https://publicsuffix.org/list/public_suffix_list.dat and overrides included in
|
||||
parsedmarc.resources.maps.psl_overrides.txt
|
||||
|
||||
Args:
|
||||
domain (str): A domain or subdomain
|
||||
@@ -87,8 +94,12 @@ def get_base_domain(domain):
|
||||
str: The base domain of the given domain
|
||||
|
||||
"""
|
||||
psl = publicsuffixlist.PublicSuffixList()
|
||||
return psl.privatesuffix(domain)
|
||||
domain = domain.lower()
|
||||
publicsuffix = psl.privatesuffix(domain)
|
||||
for override in psl_overrides:
|
||||
if domain.endswith(override):
|
||||
return override.strip(".").strip("-")
|
||||
return publicsuffix
|
||||
|
||||
|
||||
def query_dns(domain, record_type, cache=None, nameservers=None, timeout=2.0):
|
||||
|
||||
@@ -55,6 +55,7 @@ dependencies = [
|
||||
"tqdm>=4.31.1",
|
||||
"urllib3>=1.25.7",
|
||||
"xmltodict>=0.12.0",
|
||||
"PyYAML>=6.0.3"
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
Reference in New Issue
Block a user