From e588845f235f43cecf4ac85ee9a94220ceb179ba Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 10:30:25 -0500 Subject: [PATCH 01/14] 6.1.0 - Fix aggregate report email parsing regression Fix aggregate report email parsing regression in 6.0.3 (closes issues #57 and #58) --- CHANGELOG.md | 6 + parsedmarc/__init__.py | 5 +- parsedmarc/utils.py | 2 +- requirements.txt | 1 + ...ogle.com Report-ID- 949348866075514174.eml | 181 ++++++++++++++++++ samples/aggregate/twilight.eml | 111 +++++++++++ setup.py | 2 +- tests.py | 7 +- 8 files changed, 306 insertions(+), 9 deletions(-) create mode 100644 samples/aggregate/Report domain- borschow.com Submitter- google.com Report-ID- 949348866075514174.eml create mode 100644 samples/aggregate/twilight.eml diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c1d52..dc2317f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +6.1.0 +----- + +- Fix aggregate report email parsing regression introduced in 6.0.3 (closes issues #57 and #58) + + 6.0.3 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 13fe267..4494a75 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.0.3" +__version__ = "6.1.0" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' @@ -714,8 +714,7 @@ def parse_report_email(input_, nameservers=None, dns_timeout=2.0, sample = payload else: try: - if not payload.startswith(MAGIC_XML): - payload = b64decode(payload) + payload = b64decode(payload) if payload.startswith(MAGIC_ZIP) or \ payload.startswith(MAGIC_GZIP) or \ payload.startswith(MAGIC_XML): diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 119ecb9..8f040d3 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -26,7 +26,7 @@ import geoip2.errors import requests import publicsuffix -__version__ = "6.0.3" +__version__ = "6.1.0" USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc/{2}".format( platform.system(), diff --git a/requirements.txt b/requirements.txt index d47b87b..fbf2d96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +nose flake8 sphinx sphinx_rtd_theme diff --git a/samples/aggregate/Report domain- borschow.com Submitter- google.com Report-ID- 949348866075514174.eml b/samples/aggregate/Report domain- borschow.com Submitter- google.com Report-ID- 949348866075514174.eml new file mode 100644 index 0000000..64d0cdd --- /dev/null +++ b/samples/aggregate/Report domain- borschow.com Submitter- google.com Report-ID- 949348866075514174.eml @@ -0,0 +1,181 @@ +Received: from SN6PR04MB4480.namprd04.prod.outlook.com (2603:10b6:803:41::31) + by SN6PR04MB4477.namprd04.prod.outlook.com with HTTPS via + SN4PR0501CA0054.NAMPRD05.PROD.OUTLOOK.COM; Wed, 13 Feb 2019 10:48:13 +0000 +Received: from DM5PR04CA0035.namprd04.prod.outlook.com (2603:10b6:3:12b::21) + by SN6PR04MB4480.namprd04.prod.outlook.com (2603:10b6:805:a5::17) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.1622.16; Wed, 13 Feb + 2019 10:48:12 +0000 +Received: from BY2NAM03FT014.eop-NAM03.prod.protection.outlook.com + (2a01:111:f400:7e4a::207) by DM5PR04CA0035.outlook.office365.com + (2603:10b6:3:12b::21) with Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id 15.20.1622.16 via Frontend + Transport; Wed, 13 Feb 2019 10:48:12 +0000 +Authentication-Results: spf=softfail (sender IP is 199.230.200.198) + smtp.mailfrom=google.com; cardinalhealth.mail.onmicrosoft.com; dkim=fail + (signature did not verify) + header.d=google.com;cardinalhealth.mail.onmicrosoft.com; dmarc=fail + action=oreject header.from=google.com; +Received-SPF: SoftFail (protection.outlook.com: domain of transitioning + google.com discourages use of 199.230.200.198 as permitted sender) +Received: from SMTP10.cardinalhealth.com (199.230.200.198) by + BY2NAM03FT014.mail.protection.outlook.com (10.152.84.239) with Microsoft SMTP + Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id + 15.20.1580.10 via Frontend Transport; Wed, 13 Feb 2019 10:48:12 +0000 +Received: from WPOH0019EXHYB01.cardinalhealth.net (10.212.25.115) by + smtp10.cardinalhealth.com (10.208.183.149) with Microsoft SMTP Server (TLS) + id 14.3.361.1; Wed, 13 Feb 2019 05:47:36 -0500 +Received: from WPIL0210EXCAS23.cardinalhealth.net (161.244.3.66) by + WPOH0019EXHYB01.cardinalhealth.net (10.212.25.115) with Microsoft SMTP Server + (TLS) id 14.3.361.1; Wed, 13 Feb 2019 05:47:32 -0500 +Received: from smtp2.cardinal.com (198.89.161.108) by + WPIL0210EXCAS23.cardinalhealth.net (161.244.3.66) with Microsoft SMTP Server + (TLS) id 14.3.361.1; Wed, 13 Feb 2019 04:47:32 -0600 +Authentication-Results-Original: smtp2.cardinal.com; spf=Pass + smtp.mailfrom=noreply-dmarc-support@google.com +Received-SPF: Pass (smtp2.cardinal.com: domain of + noreply-dmarc-support@google.com designates 209.85.166.201 as + permitted sender) identity=mailfrom; + client-ip=209.85.166.201; receiver=smtp2.cardinal.com; + envelope-from="noreply-dmarc-support@google.com"; + x-sender="noreply-dmarc-support@google.com"; + x-conformance=spf_only; x-record-type="v=spf1" +X-SenderGrp: None +X-SBRS: 3.5 +X-ExtWarning: Yes +X-SLBL-Result: SAFE-LISTED +X-Amp-Result: UNKNOWN +X-Amp-Original-Verdict: FILE UNKNOWN +X-Amp-File-Uploaded: False +IronPort-PHdr: =?us-ascii?q?9a23=3AQPa/HRJPWXWEsohNPdmcpTZcNBhigK39O0su0rRi?= + =?us-ascii?q?jrtPdqq5+JG7Zh7Q4vJiiFKPVoLeuatJ?= +X-IPAS-Result: =?us-ascii?q?A0HNBQBz9WNch8mmVdFjHAEBAR8EAQEFAQEGEIE2AoJpgQM?= + =?us-ascii?q?nh0+FPIsFlRGCXYIygSQDGCoSCAEEGAMTgQKDOQECg1YhATQJDQEDAQEBAQEBA?= + =?us-ascii?q?QECAhABAQEKCwkIKSMIBIJ0BE06MAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEUAjM?= + =?us-ascii?q?XKQUYGwwBAThiMgEFAYNZAYIBngU8iikBAQGCCxOCeAEBBXSBFzuDPIEdBwgSg?= + =?us-ascii?q?luFFIF9gkSCFoFXgkyBJoF4AoF4NoR5GoEshjgJiViIBIhVgQsJgXaBRIEPgRu?= + =?us-ascii?q?BVosZGYFeAWOQLRmYRgGDVQ8hgSVugSFwXIESgU4JIxeBczSDIIp0ITCPbwEB?= +X-IronPort-Outbreak-Status: No, level 0, Unknown - Unknown +Received: from mail-it1-f201.google.com ([209.85.166.201]) by + smtp2.cardinal.com with ESMTP/TLS/AES128-GCM-SHA256; 13 Feb 2019 04:47:15 + -0600 +Received: by mail-it1-f201.google.com with SMTP id p21so3350239itb.8 + for ; Wed, 13 Feb 2019 02:47:15 -0800 + (PST) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=google.com; s=20161025; + h=mime-version:date:message-id:subject:from:to:content-disposition + :content-transfer-encoding; + bh=9auxxiIHA4359CcIJ8D48sw6ZXttCPhnCsgP3zpHWC4=; + b=QSiY0EPGhBY11lvmfniPOmA71xd6uAv17KYGbEtmOtocmGen1BQ96kZA2rNtAtZDUx + IfLoJgkzO31GmVXh9k0kBi+r8vR16zXebZHuBgfBesayykHOfSSWOTZtwSukaWV3RChV + PPMRPMksnrITfFHNo3u6xbcx6usplxn8XS8XyENgua30BBjweJXYMrQrRkrjgLv+JpoY + o6fVvAtcuSnwtm3fv9j1GsqSK05sw2aVFZkJLP9DvMfhW+bXJJ2rVp4MvVqlleua20Ro + Y0vbFMWtbvFZseSOc+AYGvv6oL9eBilXal26kS2BrRJQ+B4Yt4GFiRDbjF4QqVSTHOd4 + YDSw== +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20161025; + h=x-gm-message-state:mime-version:date:message-id:subject:from:to + :content-disposition:content-transfer-encoding; + bh=9auxxiIHA4359CcIJ8D48sw6ZXttCPhnCsgP3zpHWC4=; + b=lqkAygJJ/s8iZJI+AnwWegB47wiLE67qFfi26J+5Uu5lQuObEqK2KTlBZGwnd51c8R + A2y47YQ9RqBKWTibQH9pLBKT5ChyxP/+Vk8e3wK+IfU720wG8P2eDXn91rBSBNIXOTOn + McNwB/z6Ukurt8GFiy4aMvni0t3yWa4/xL0c5zFQJAgwm01jhxOjgOpnnqF0ppYatzf4 + 5maCMzhvRJ9FFsDhk8sE0pJjdaWS9ybDGKOteSYr1wlGSnydTkt2z6z38IO8fgVJwT3e + AUiqhNgNmDdyNI3Aom8dnfJHsyqjKC3iTXGxmqsMDVui4faHwOqMx2lgjuv7bbQFyv1K + b//w== +X-Gm-Message-State: AHQUAubgr+0/AsmLjETdSMNb9+rAZ5j0xlExGd75CusDHQJl4eIy2avN + XnhZqrYsvbPhKCNFyDE0KQ== +X-Google-Smtp-Source: AHgI3IZS0dawFR+Q0mnSaOenxA+M5W6V6z1dMorYgiX855zYf4aqZyS2Jjol+TCgY3f/lSsVDbuiefhqmZNtNA== +MIME-Version: 1.0 +X-Received: by 2002:a24:1c87:: with SMTP id c129mr1998419itc.11.1550054834929; + Wed, 13 Feb 2019 02:47:14 -0800 (PST) +Date: Tue, 12 Feb 2019 15:59:59 -0800 +Message-ID: <949348866075514174@google.com> +Subject: Report domain: borschow.com Submitter: google.com Report-ID: 949348866075514174 +From: +To: + name="google.com!borschow.com!1549929600!1550015999.zip" + filename="google.com!borschow.com!1549929600!1550015999.zip" +Return-Path: noreply-dmarc-support@google.com +X-OrganizationHeadersPreserved: WPIL0210EXCAS23.cardinalhealth.net +X-MS-Exchange-Organization-ExpirationStartTime: 13 Feb 2019 10:48:12.5214 + (UTC) +X-MS-Exchange-Organization-ExpirationStartTimeReason: OriginalSubmit +X-MS-Exchange-Organization-ExpirationInterval: 2:00:00:00.0000000 +X-MS-Exchange-Organization-ExpirationIntervalReason: OriginalSubmit +X-MS-Exchange-Organization-Network-Message-Id: + e9371fc9-cf12-4fcb-502a-08d691a0c038 +X-EOPAttributedMessage: 0 +X-MS-Exchange-Organization-MessageDirectionality: Originating +X-CrossPremisesHeadersPromoted: + BY2NAM03FT014.eop-NAM03.prod.protection.outlook.com +X-CrossPremisesHeadersFiltered: + BY2NAM03FT014.eop-NAM03.prod.protection.outlook.com +X-Forefront-Antispam-Report: + CIP:199.230.200.198;IPV:NLI;CTRY:US;EFV:NLI;SFV:SKN;SFS:;DIR:INB;SFP:;SCL:-1;SRVR:SN6PR04MB4480;H:SMTP10.cardinalhealth.com;FPR:;SPF:None;LANG:en; +X-Microsoft-Exchange-Diagnostics: + 1;BY2NAM03FT014;1:9gD0HAosLjXNiAwpOsmGDn2zgTJAIEDY0eKyvNoIEz6oio7FckNeMUQRNa3AUeC/Qil0Sn2rntyy6LjTvutGN6e2BsGQyNaiKEsKI3so3l0Kpb9i3CdYF21/lNVHrjKuxxEJ8F7WUBlR88jcBwDjl6x0CO2FBckAmUnBJOJv2zg= +X-MS-Exchange-Organization-AuthSource: WPIL0210EXCAS23.cardinalhealth.net +X-MS-Exchange-Organization-AuthAs: Anonymous +X-OriginatorOrg: cardinalhealth.onmicrosoft.com +X-MS-PublicTrafficType: Email +X-MS-Office365-Filtering-Correlation-Id: e9371fc9-cf12-4fcb-502a-08d691a0c038 +X-Microsoft-Antispam: + BCL:0;PCL:0;RULEID:(2390118)(7020095)(4652040)(8989299)(4534185)(4627221)(201703031133081)(8559020)(8990200)(5600110)(711020)(4605077)(4710068)(4711035)(2017052603328)(7153060)(49563074);SRVR:SN6PR04MB4480; +X-MS-TrafficTypeDiagnostic: SN6PR04MB4480: +X-MS-Exchange-Organization-SCL: -1 +X-Microsoft-Exchange-Diagnostics: + 1;SN6PR04MB4480;20:9lFp0O5yeS9rEVtgFCaVjrPpXZAA0REuLHin4MfFWihk274IOvh7BRRMQfNNBwtC3q0+vTeNPc37wIBQlwVq6T7j1bNe06DnEjGgP5GWNU7zOUjt6qeq21ebYk/RV2QcTM85ZcFxr8SF2bGAKcNkj7GQLnnogH5o1GotLtqwXOht4qNZxhp46eCrIt+yQJJHFJyHFoflM9+z2WAYSl6yY8Wauhp05LBIqjduLdEN6MmU7bpPrzOmBrc4nuOmA4s1D8A3KdzBCdx0wIEwBv1zq6RjAB43UrfhpwMmh07U60CU/0QPhSrEBfn19eZLv4eTRJozsA313tp/mPylGCegahxmClixw/ku/GENI6pWxTCz3Jz1x8YCMLqJS7M+UOg7IosLPbUr26Q2CtSduue1vhk6v8peX5c5di6b9WftMKup3YMESA0RrqNbS6HbBCmH+iHSUwWjTBVva4L0fGiG82SbFbkH/UH+ZoFzkFnMtDZwDK+9pK+oHS2k97XwVzzYrzFh98JBdnk6jI/L2MmHWldt75NqJjSifAf2P/PjlploUQ8CAHsZZSRx5cu5tLaptOzUq/NVXF23VavhTslxK5C0/ntAAZAAvhmR8v/FNFU=;23:m8XEp0VuraCd8j9yukaQaVZE+Ufz0qQ9v4369t8CewCI5dikl+UkdVfYaJsMrwHTAtH3it3YrgDXpo7py6m/RDwgDnLGxviIfy/puyEEN3Qq99TJUMn19W9S5U7VJ8DgMZsnEv6RVCrjf05bNshRyA== +X-Microsoft-Exchange-Diagnostics: + 1;SN6PR04MB4480;20:fJacS7QTNHPZGJt2KoBiyZLfHf3R2G+vFZ1EUyyFaqoQUdJU3WDLMmHMUqn36br0oZNxyMkV05SJMoFAz3mVO2hO/nsUX6SR7RMpr5XHYxLD+tPbbTTT2aGzo5IR+GOrJc5l3z4uX34x0WdoggvjUhi6DWaqwMn/OnkEBJ7ozYg= +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 13 Feb 2019 10:48:12.1984 + (UTC) +X-MS-Exchange-CrossTenant-Network-Message-Id: e9371fc9-cf12-4fcb-502a-08d691a0c038 +X-MS-Exchange-CrossTenant-Id: a86dc674-e6a2-4c25-b4ba-e36c31382530 +X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp: TenantId=a86dc674-e6a2-4c25-b4ba-e36c31382530;Ip=[199.230.200.198];Helo=[SMTP10.cardinalhealth.com] +X-MS-Exchange-CrossTenant-FromEntityHeader: HybridOnPrem +X-MS-Exchange-Transport-CrossTenantHeadersStamped: SN6PR04MB4480 +X-MS-Exchange-Transport-EndToEndLatency: 00:00:01.1339945 +X-MS-Exchange-Processed-By-BccFoldering: 15.20.1601.011 +X-Microsoft-Antispam-Mailbox-Delivery: + ucf:0;jmr:0;ex:0;auth:0;dest:I;ENG:(750119)(520011016)(944506303)(944626516); +X-Microsoft-Antispam-Message-Info: + Cqz80Cj8FVW4uTBbPiVvb9OH0/VEl4Uz5BiS+YWHErndckPxKMInYe6J09QImrgTO+t2bYNNpL66Km1sbVKa2o+iWj1pSIxAONbkfZuosS0y7Xbj/NEw1eqGtwavoj5WckV7MfJmBINAEBVg6UPnNhmW5rY8PTa27tmGQgYoE7wm5JLH0EH8ARuebjtTf8j+WuBg/BY6uaK7FLOdAUnnlvAtoHDKTnL/oH5E6GG59HTarZyC4wMl5sN2ibbMqZ80Aj4EczyuoWz1N2thodsW/4yzTk2w2dtHgwMdKgPI+4xALQC81kQAlgVyN9ukvBpJnRKrA4bvx+XhUUsMKsoXmpWdQCIoALqAfXheY/96JepEYN05Fqa7wzDRLkbejIfsfPq16asiX/kw8Dq2N/WG5LeQpC28iOkY4TB/GlI6CQPVd8rMY3DvzBYZPyCAcUdPhXC3nR+qxLea+zH/s1IRKaXolnF0r0zaiCki952hC6UwfdeK9F/nCA75kRb930QXJbmOS9emnEf+xqWhIhJuMdd8gV1NLSz6SDimeHRfDgXMTgNUjkk/BQQ0bCWAEQrRPxdvt/5PEiUZMZzMKZAqYh67j2RpV8FC/qJLjHPljagvtH4bUvGmpn/W9MArWgsUkk2skhNcLVletMwYbVXvJfJPr7K9Pnfpnd4p1ETHwjlzXaKcvlziIE29MYEXPUcg9rnk2t33YTM1NJHhgyiKebbrHC2/BU1O+BNrkZYQhqlkvsAu4JxBdyzld2sDz9CQdvwOSAwOkMpdlkaHV26Y0e6gPLkaprWVqXPr5KFXSUfuz2fvVUNM+FuHGV/fIFkcKdK4lw0MRufwhBz1gqudL/PSQuI8r9lQmuh7K3+gIprdWqOiYlYEELj+TMnSnZaFkbIX70rhAAkB7MoNfs/A38hIooGzxlRYzTrlPqwoIkOpyqQykDzXoNRODHwo7QJx +Content-type: multipart/mixed; + boundary="B_3632898004_720446853" + +> This message is in MIME format. Since your mail reader does not understand +this format, some or all of this message may not be legible. + +--B_3632898004_720446853 +Content-type: text/plain; + charset="UTF-8" +Content-transfer-encoding: 7bit + + + +--B_3632898004_720446853 +Content-type: application/zip; name="google.com!borschow.com!1549929600!1550015999.zip"; + x-mac-creator="4F50494D" +Content-ID: +Content-disposition: attachment; + filename="google.com!borschow.com!1549929600!1550015999.zip" +Content-transfer-encoding: base64 + + +UEsDBAoAAAAIAPhTTU5+28OP0QEAACgEAAAxAAAAZ29vZ2xlLmNvbSFib3JzY2hvdy5jb20h +MTU0OTkyOTYwMCExNTUwMDE1OTk5LnhtbI1TwXakIBC85yvmzX1EJ2rGfYTsab9g9+xjoFU2 +CjzATPL3iwHRnexhT2J1d3VXNeCX92k8vIGxQsnnY5HlxwNIpriQ/fPx188fp8vx8EIecAfA +r5S9kofDARvQyrh2Akc5dXTBPKpM30o6AemV6kfImJowSmDIgYmKkUjlGcaPE5+oYSc764Xu ++74s5MWad2doy5R0lLlWyE6RwTltvyEUS7OtFFFEpb2BQeeyrqtL7rm+1gfiKENw0pTNY3m5 +1HX+VFVFWTyVGG3RkO2VQmuo7KMWD12hF5IUVdk056bOfa+ArHGQ3EerPC+qpmn8JHIlQ3+z +pW57S7FWo2AfrZ6vo7ADpEGUN0eSqzKWDeoWDItgyKD8VUzEYBQOEbS6+8SWb4A0MfAbmMNI +R8RukF0xzRwpFnHL4XPYfw3m3WTKrDMadUsuWDUbBq3QpDln1WNWFHVW5GffIQXWVKZm6Zth +FA4rHPvBGx1n7xtfA4sZwmplhfPXN02+x3aZixWdv1Y+IbkSRXcxkKzZqbzr6le1asOCg3Si +E/75pLIBKAfTdkZNdyvaRyLVFwJMZze0Buw8uo1zN9b/7D9e7oUj6oo/Sdp2BdB9wyU5LBKj +7dH/AVBLAQIKAAoAAAAIAPhTTU5+28OP0QEAACgEAAAxAAAAAAAAAAAAAAAAAAAAAABnb29n +bGUuY29tIWJvcnNjaG93LmNvbSExNTQ5OTI5NjAwITE1NTAwMTU5OTkueG1sUEsFBgAAAAAB +AAEAXwAAACACAAAAAA== +--B_3632898004_720446853-- + diff --git a/samples/aggregate/twilight.eml b/samples/aggregate/twilight.eml new file mode 100644 index 0000000..d57f7fd --- /dev/null +++ b/samples/aggregate/twilight.eml @@ -0,0 +1,111 @@ +Return-Path: +Delivered-To: dmarc.reports@twlnet.com +Received: from mail.twlnet.com (unknown [10.0.0.1]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mail01.twlnet.com (Postfix) with ESMTPS id 8E73E1FAB7 + for ; Mon, 11 Feb 2019 11:23:41 +0100 (CET) +Received: from relay-twl-01.twlnet.com ([87.106.127.28]:58922) + by mail.twlnet.com with esmtps (TLSv1.2:DHE-RSA-AES256-GCM-SHA384:256) + (Exim 4.82_1-5b7a7c0-XX) + (envelope-from ) + id 1gt8kN-0003AY-1B + for dmarc.reports@twlnet.com; Mon, 11 Feb 2019 11:23:31 +0100 +Received: by relay-twl-01.twlnet.com (Postfix, from userid 112) + id 15771A0482; Mon, 11 Feb 2019 11:15:05 +0100 (CET) +X-CTCH-RefID: str=0001.0A0C0211.5C614D23.00C0,ss=1,re=0.000,recu=0.000,reip=0.000,cl=1,cld=1,fgs=0 +Authentication-Results: relay-twl-01.twlnet.com; + dkim=pass (2048-bit key; unprotected) header.d=google.com header.i=@google.com header.b=glE73hE7; + dkim-atps=neutral +X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on + relay-twl-01.twlnet.com +X-Spam-Level: +X-Spam-Status: No, score=-14.7 required=3.5 tests=DATE_IN_PAST_06_12, + DKIMWL_WL_MED,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF, + DKIM_VERIFIED,ENV_AND_HDR_SPF_MATCH,RCVD_IN_DNSWL_NONE,SPF_PASS, + TVD_SPACE_RATIO,USER_IN_DEF_DKIM_WL,USER_IN_DEF_SPF_WL autolearn=no + autolearn_force=no version=3.4.2 +X-Envelope-From: +DMARC-Filter: OpenDMARC Filter v1.3.1 relay-twl-01.twlnet.com 52C5FA0288 +Authentication-Results: relay-twl-01.twlnet.com; dmarc=pass header.from=google.com +Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.166.201; helo=mail-it1-f201.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc.reports@twlnet.com +Received: from mail-it1-f201.google.com (mail-it1-f201.google.com [209.85.166.201]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by relay-twl-01.twlnet.com (Postfix) with ESMTPS id 52C5FA0288 + for ; Mon, 11 Feb 2019 11:14:55 +0100 (CET) +Received: by mail-it1-f201.google.com with SMTP id j3so21119437itf.5 + for ; Mon, 11 Feb 2019 02:14:55 -0800 (PST) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=google.com; s=20161025; + h=mime-version:date:message-id:subject:from:to:content-disposition + :content-transfer-encoding; + bh=Y722jQXFp+v3E+J91/wVzAMn5Tm5ikRl6MINXaKKpB4=; + b=glE73hE7yBYYJTH5p90EOeEvrHwSWLFqkpJdh8P4osd43rrYzmQly2oAoyU2VtMUSl + lmE4b8m/PsRM7CKBEC1kFp+xpCosqMokDOSBvgQbAZL5EmPvwiXWMJMnaRGLU1eBI4Kw + f547ulUk0EilNUZTqrWfdY29+Usm57jONSGNhYFRGdmxe8L5DuAz7ggtYFFAqo6XD6f4 + uow/ZWCvXt47U8NPQwvt1DH4MWlS3vwvEVQmKrnuwkQq0ykciq1qzax891w3ex83xw1R + eCRQpT2tiu9K5HqF+yRCeL+bNakMmcFoqJo/5LaVufbsvLrkcU0+iRcgpm592ZjVmQXm + AILA== +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20161025; + h=x-gm-message-state:mime-version:date:message-id:subject:from:to + :content-disposition:content-transfer-encoding; + bh=Y722jQXFp+v3E+J91/wVzAMn5Tm5ikRl6MINXaKKpB4=; + b=knb52YcLt3kUw/tAlhlrRASh37+gYqvBnvJiG4L2YTsUX4R39tZtLKd5cI6SKqCvrD + rXjvaV9lelt9hPoxUWCRXMzX0WDFMUvwChzsMMigTbEEOwHXSU0jBz/l9k3j2oEAeTsc + 1B5FnnQ5DiOVoCNBNv7u60RBkDMrSGchav9oX+gjBkonOMt6lyL9hsvmj5qMkhZDi6M6 + 2LDViKpVvNtdTXikvxAhvrOLP/WDTDGOV+1IeqDLxNrMUGNw7nHeOd8VDybSxx01pncC + 5OgRyGlFzuRXX4CluLm5RDEEgBwIh71MiIYPlECekN1cXRU5QLEHAnHkemxxaeTxNShY + RPnQ== +X-Gm-Message-State: AHQUAuYMteZYtsvb1pPQWqTQXQH0dJpzc4utPCZ1xpCc+U7Kwhtkkhbs + tDGW/14HWdV7sCJ0ItVihA== +X-Google-Smtp-Source: AHgI3IbnGk4l6Au3/SoJ7ydpCJIZuJIsjo8K+dIi0LepThUGzwtExGj8yY6w9C85rVpZerhSO45zOFIgq7BAsA== +MIME-Version: 1.0 +X-Received: by 2002:a5e:c80b:: with SMTP id y11mr165512iol.20.1549880090723; + Mon, 11 Feb 2019 02:14:50 -0800 (PST) +Date: Sun, 10 Feb 2019 15:59:59 -0800 +Message-ID: <1627703331531660819@google.com> +Subject: Report domain: twlnet.com Submitter: google.com Report-ID: 1627703331531660819 +From: noreply-dmarc-support@google.com +To: dmarc.reports@twlnet.com +Content-type: multipart/mixed; boundary="----------=_1549880620-12208-0" + +This is a multi-part message in MIME format... + +------------=_1549880620-12208-0 +Content-Type: application/zip; + name="google.com!twlnet.com!1549756800!1549843199.zip" +Content-Disposition: attachment; + filename="google.com!twlnet.com!1549756800!1549843199.zip" +Content-Transfer-Encoding: base64 + +UEsDBAoAAAAIAKJQS04KMkMU6wEAAKIEAAAvAAAAZ29vZ2xlLmNvbSF0d2xu +ZXQuY29tITE1NDk3NTY4MDAhMTU0OTg0MzE5OS54bWylVE1zmzAQvedXML4b +IbAx7ihKT/0F7ZmRxYLVgKSRRJz8+wpLfMTpTA89Id7uvt33VkBe3oc+eQNj +hZLPO5xmuwQkV42Q3fPu188f+2qXvNAn0gI0F8Zf6VOSEANaGVcP4FjDHJsw +jyrT1ZINQDuluh5SrgaCFjDkwMBET6XyDP3HvhmY4Xs76onu+7Ys5MWad2dY +zZV0jLtayFbRq3PafkMolqZrKWKISXsDg/JDWR6rzHN9rQ/EUYZoKC7z0ykr +igIfC1yWWYXPBK3hkO6lQm2Y7KIYD12gE5Li4+F8OpZV5psFZI6DbO7R6lDg +s6ec3gMZ+sy2dNt6SrTqBf+o9Xjphb3CMojy7kjqbr0EF/yKUIiz5lUM1BIU +DhG0ur1j0zNAmhr4DdwRpCNiV8jOmOaO4knadLiP+rexvJlcmXlCo26LB1aN +hkMtNK1OKc7KFOenNK98hyUwp3I1St+MoHCY4dgP3lg/eteaOTBZIaxWVjh/ +e/2tkuCd2CCbvMkIzaw3YPUkSm5jYDFmo/Ghp1/TrIyIBqQTrfDfzlJ2BdaA +qVujhk/r2eKR6Es5YaO71gbs2LuV8WHaf6w+XuuJIoqKLxu90Pv9KkPzDFfY +r3UBFu3bnmTjyv/335jsb+KD3ik53CGC1t/NH1BLAQIKAAoAAAAIAKJQS04K +MkMU6wEAAKIEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIXR3bG5l +dC5jb20hMTU0OTc1NjgwMCExNTQ5ODQzMTk5LnhtbFBLBQYAAAAAAQABAF0A +AAA4AgAAAAA= + +------------=_1549880620-12208-0 +Content-Type: text/plain; charset="utf8" +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +MIME-Version: 1.0 +X-Mailer: MIME-tools 5.505 (Entity 5.505) + +This email was Virus checked by Sophos UTM 9. + +Twilight-Networks https://www.twlnet.com= + +------------=_1549880620-12208-0-- \ No newline at end of file diff --git a/setup.py b/setup.py index 4eae6fd..db6a25d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.0.3" +__version__ = "6.1.0" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" diff --git a/tests.py b/tests.py index f840aad..6d33a55 100644 --- a/tests.py +++ b/tests.py @@ -13,12 +13,11 @@ class Test(unittest.TestCase): sample_paths = glob("samples/aggregate/*") for sample_path in sample_paths: print("Testing {0}...\n".format(sample_path)) - parsed_report = parsedmarc.parse_aggregate_report_file( - sample_path) + parsed_report = parsedmarc.parse_report_file( + sample_path)["report"] print(json.dumps(parsed_report, ensure_ascii=False, indent=2)) print("\n") - print( - parsedmarc.parsed_aggregate_reports_to_csv(parsed_report)) + print(parsedmarc.parsed_aggregate_reports_to_csv(parsed_report)) def testForensicSamples(self): """Test sample forensic/ruf/failure DMARC reports""" From 603dd65da58c747d928ad65f7e437f29a11a429f Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 10:34:55 -0500 Subject: [PATCH 02/14] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc2317f..2053ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ 6.1.0 ----- -- Fix aggregate report email parsing regression introduced in 6.0.3 (closes issues #57 and #58) - +- Fix aggregate report email parsing regression introduced in 6.0.3 (closes issue #57) +- Fix Davmail support (closes issue #56) 6.0.3 ----- From b70d47e1b21124a3c965d1a119fbec7be42e815e Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 13:42:32 -0500 Subject: [PATCH 03/14] Update mailing list documentation --- docs/index.rst | 55 ++++++++++++++++++++++---------------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b7f09ff..ec9830b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1163,42 +1163,39 @@ What if a sender won't support DKIM/DMARC? What about mailing lists? ========================= -When you deploy DMARC on your domain, you might find that messages relayed by -mailing lists are failing DMARC. This has two causes: +What about mailing lists? -#. You are not DKIM signing your mail like you should be -#. The mailing list is altering emails in ways that are not DMARC compliant - before sending them +When you deploy DMARC on your domain, you might find that messages relayed by mailing lists are failing DMARC, most likely because the mailing list is spoofing your from address, and modifying the subject, footer, or other part of the message. -`Joe Nelson`_ does a fantastic job of explaining exactly what mailing lists -should and shouldn't do to be DMARC compliant. Rather than repeat his fine -work, here's a TL;DR: +To fix this, the mailing list administrator must configure the list to replace the from address of the message (also known as munging) with the address of the mailing list, so they no longer spoof email addresses with domains protected by DMARC. Configuration steps for comon mailing list platforms are listed below. -If you run a mailing list -------------------------- +Mailman 2 +---------- -**Do** +Navigate to Privacy Options> Sending Filters, and configure the settings below -- Retain headers from the original message -- Add `RFC 2369`_ List-Unsubscribe headers to outgoing messages, instead of - adding unsubscribe links to the body +================================== ========== +Setting Value +dmarc_moderation_action Munge From +dmarc_quarentine_moderation_action Yes +dmarc_none_moderation_action Yes +================================== ========== - :: +Mailman 3 +--------- - List-Unsubscribe: +In the DMARC Mitagations tab of the Settings page, configure the settings below -- Add `RFC 2919`_ List-Id headers instead of modifying the subject +============================== =============================== +Setting Value +DMARC mitigation action Replace From: with list address +DMARC Mitigate unconditionally No +============================== =============================== - :: +LISTSERV +-------- - List-Id: Example Mailing List - -**Do not** - -* Remove or modify any existing headers from the original message, including - From, Date, Subject, etc. -* Add to or remove content from the message body, **including traditional - disclaimers and unsubscribe footers** +`LISTSERV 16.0-2017a`_ and higher will rewrite the From header. Some additional steps are needed for Linux hosts. API === @@ -1271,8 +1268,4 @@ Indices and tables .. _XML files: https://github.com/domainaware/parsedmarc/tree/master/splunk -.. _Joe Nelson: https://begriffs.com/posts/2018-09-18-dmarc-mailing-list.html - -.. _RFC 2369: https://tools.ietf.org/html/rfc2369 - -.. _RFC 2919: https://tools.ietf.org/html/rfc2919 +.. _LISTSERV 16.0-2017a: https://www.lsoft.com/news/dmarc-issue1-2018.asp \ No newline at end of file From f5d645cebdb74c36f0301c27cd03e79b5f6a29e1 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 13:50:52 -0500 Subject: [PATCH 04/14] Fix table formatting --- docs/index.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ec9830b..d53356c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1174,23 +1174,23 @@ Mailman 2 Navigate to Privacy Options> Sending Filters, and configure the settings below -================================== ========== -Setting Value -dmarc_moderation_action Munge From -dmarc_quarentine_moderation_action Yes -dmarc_none_moderation_action Yes -================================== ========== +====================================== ========== +**Setting** **Value** +**dmarc_moderation_action** Munge From +**dmarc_quarentine_moderation_action** Yes +**dmarc_none_moderation_action** Yes +====================================== ========== Mailman 3 --------- In the DMARC Mitagations tab of the Settings page, configure the settings below -============================== =============================== -Setting Value -DMARC mitigation action Replace From: with list address -DMARC Mitigate unconditionally No -============================== =============================== +================================== =============================== +**Setting** **Value** +**DMARC mitigation action** Replace From: with list address +**DMARC Mitigate unconditionally** No +================================== =============================== LISTSERV -------- From e12c5637e1e6706047814b88906070d6ee793946 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 13:53:32 -0500 Subject: [PATCH 05/14] Update index.rst --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d53356c..f66f6bb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1163,8 +1163,6 @@ What if a sender won't support DKIM/DMARC? What about mailing lists? ========================= -What about mailing lists? - When you deploy DMARC on your domain, you might find that messages relayed by mailing lists are failing DMARC, most likely because the mailing list is spoofing your from address, and modifying the subject, footer, or other part of the message. To fix this, the mailing list administrator must configure the list to replace the from address of the message (also known as munging) with the address of the mailing list, so they no longer spoof email addresses with domains protected by DMARC. Configuration steps for comon mailing list platforms are listed below. From 93ecf4a262ee149ec0010b7070d53017b1c3ba7b Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 14:23:16 -0500 Subject: [PATCH 06/14] Fix docs typo --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index f66f6bb..dd71275 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1182,7 +1182,7 @@ Navigate to Privacy Options> Sending Filters, and configure the settings below Mailman 3 --------- -In the DMARC Mitagations tab of the Settings page, configure the settings below +In the DMARC Mitigations tab of the Settings page, configure the settings below ================================== =============================== **Setting** **Value** From d68294c58a3c86080721a96d3a8c904bc26dc871 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 15:04:47 -0500 Subject: [PATCH 07/14] Fix docs typo --- docs/index.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index dd71275..6d3b518 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1163,9 +1163,16 @@ What if a sender won't support DKIM/DMARC? What about mailing lists? ========================= -When you deploy DMARC on your domain, you might find that messages relayed by mailing lists are failing DMARC, most likely because the mailing list is spoofing your from address, and modifying the subject, footer, or other part of the message. +When you deploy DMARC on your domain, you might find that messages relayed by +mailing lists are failing DMARC, most likely because the mailing list is +spoofing your from address, and modifying the subject, footer, or other part +of the message. -To fix this, the mailing list administrator must configure the list to replace the from address of the message (also known as munging) with the address of the mailing list, so they no longer spoof email addresses with domains protected by DMARC. Configuration steps for comon mailing list platforms are listed below. +To fix this, the mailing list administrator must configure the list to replace +the from address of the message (also known as munging) with the address of +the mailing list, so they no longer spoof email addresses with domains +protected by DMARC. Configuration steps for common mailing list platforms are +listed below. Mailman 2 ---------- From 53510c1f78449a8cf6eec9804d703a73e1893622 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 19:15:52 -0500 Subject: [PATCH 08/14] Update example davmail config See also: Issue #57 --- docs/index.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 6d3b518..22fb6f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -653,6 +653,9 @@ Configure Davmail by creating a ``davmail.properties`` file # message size for performance reasons davmail.imapAlwaysApproxMsgSize=true + # Client connection timeout in seconds - default 300, 0 to disable + davmail.clientSoTimeout=0 + ############################################################# Run Davmail @@ -663,7 +666,8 @@ Run Davmail Because you are interacting with Davmail server over the loopback -(i.e. 127.0.0.1), add the following options to ``parsedmarc.ini`` config file: +(i.e. ``127.0.0.1``), add the following options to ``parsedmarc.ini`` +config file: .. code-block:: ini @@ -671,6 +675,7 @@ Because you are interacting with Davmail server over the loopback host=127.0.0.1 port=1143 ssl=False + watch = True Elasticsearch and Kibana ------------------------ From 71362b8d699689948b73ce4d9c5c881aa54931f1 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 19:30:42 -0500 Subject: [PATCH 09/14] Update example davmail config Make settings clearer See also: Issue #57 --- docs/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 22fb6f7..9397ec4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -626,17 +626,18 @@ Configure Davmail by creating a ``davmail.properties`` file # proxy exclude list davmail.noProxyFor= - # allow remote connection to DavMail + # block remote connection to DavMail davmail.allowRemote=false # bind server sockets to the loopback address davmail.bindAddress=127.0.0.1 # disable SSL for specified listeners - davmail.ssl.nosecureimap=false + davmail.ssl.nosecureimap=true # Send keepalive character during large folder and messages download davmail.enableKeepalive=true + # Message count limit on folder retrieval davmail.folderSizeLimit=0 From 22230af4d247fb012373641eb7f29794a0c6aaaa Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 19:44:08 -0500 Subject: [PATCH 10/14] Add a link to the Davmail FAQ Includes Modern Auth/multi-factor authentication instructions --- docs/index.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 9397ec4..aa0a79f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -578,7 +578,8 @@ Accessing an inbox using OWA/EWS Some organisations do not allow IMAP, and only support Exchange Web Services (EWS)/Outlook Web Access (OWA). In that case, Davmail will need to be set up -as a local EWS/OWA IMAP gateway. +as a local EWS/OWA IMAP gateway. It can even work where +` Modern Auth/multi-factor authentication`_ is required. To do this, download the latest ``davmail-version.zip`` from https://sourceforge.net/projects/davmail/files/ @@ -1263,6 +1264,8 @@ Indices and tables .. _Cloudflare's public resolvers: https://1.1.1.1/ +.. _ Modern Auth/multi-factor authentication: http://davmail.sourceforge.net/faq.html + .. _download the latest portable Linux version of pypy3: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux .. _Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html From fdf8ea292fee1a712879f4864b79381987011801 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 13 Feb 2019 19:50:42 -0500 Subject: [PATCH 11/14] Fix link --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index aa0a79f..4a8f81f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -579,7 +579,7 @@ Accessing an inbox using OWA/EWS Some organisations do not allow IMAP, and only support Exchange Web Services (EWS)/Outlook Web Access (OWA). In that case, Davmail will need to be set up as a local EWS/OWA IMAP gateway. It can even work where -` Modern Auth/multi-factor authentication`_ is required. +`Modern Auth/multi-factor authentication`_ is required. To do this, download the latest ``davmail-version.zip`` from https://sourceforge.net/projects/davmail/files/ @@ -1264,7 +1264,7 @@ Indices and tables .. _Cloudflare's public resolvers: https://1.1.1.1/ -.. _ Modern Auth/multi-factor authentication: http://davmail.sourceforge.net/faq.html +.. _Modern Auth/multi-factor authentication: http://davmail.sourceforge.net/faq.html .. _download the latest portable Linux version of pypy3: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux From 2eceac31009935cb5c1ed2965e304a1e93d78134 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 08:57:51 -0500 Subject: [PATCH 12/14] 6.1.1 --- CHANGELOG.md | 6 ++++++ parsedmarc/__init__.py | 5 +++-- parsedmarc/cli.py | 5 +++++ parsedmarc/utils.py | 2 +- setup.py | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2053ff1..7659d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +6.1.1 +----- + +- Fix aggregate report processing +- Check for the existence of a configuration file if a path is supplied + 6.1.0 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 4494a75..6c3425c 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.0" +__version__ = "6.1.1" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' @@ -725,6 +725,8 @@ def parse_report_email(input_, nameservers=None, dns_timeout=2.0, dns_timeout=dns_timeout) result = OrderedDict([("report_type", "aggregate"), ("report", aggregate_report)]) + return result + except (TypeError, ValueError, binascii.Error): pass @@ -765,7 +767,6 @@ def parse_report_email(input_, nameservers=None, dns_timeout=2.0, 'not a valid DMARC report'.format(subject) raise InvalidDMARCReport(error) - return result def parse_report_file(input_, nameservers=None, dns_timeout=2.0, diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 1b7ab27..14883a3 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -4,6 +4,7 @@ """A CLI for parsing DMARC reports""" from argparse import Namespace, ArgumentParser +import os from configparser import ConfigParser from glob import glob import logging @@ -182,6 +183,10 @@ def _main(): args = arg_parser.parse_args() if args.config_file: + abs_path = os.path.abspath(args.config_file) + if not os.path.exists(abs_path): + logger.error("A file does not exist at {0}".format(abs_path)) + exit(-1) opts.silent = True config = ConfigParser() config.read(args.config_file) diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 8f040d3..ffef046 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -26,7 +26,7 @@ import geoip2.errors import requests import publicsuffix -__version__ = "6.1.0" +__version__ = "6.1.1" USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc/{2}".format( platform.system(), diff --git a/setup.py b/setup.py index db6a25d..696a0b2 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.0" +__version__ = "6.1.1" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From f722907a9acebbbcc0a5eae4b146654c495f5c90 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 09:29:51 -0500 Subject: [PATCH 13/14] PEP 8 fix --- parsedmarc/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 6c3425c..b6d0731 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -768,7 +768,6 @@ def parse_report_email(input_, nameservers=None, dns_timeout=2.0, raise InvalidDMARCReport(error) - def parse_report_file(input_, nameservers=None, dns_timeout=2.0, strip_attachment_payloads=False): """Parses a DMARC aggregate or forensic file at the given path, a From 2ae500ba9cb2c8d08b5fa0b3dfe24390164cf58d Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 14:19:52 -0500 Subject: [PATCH 14/14] 6.1.1 release --- CHANGELOG.md | 2 ++ parsedmarc/utils.py | 4 ++-- requirements.txt | 25 ++++++++++++------------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7659d63..4d8e4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ - Fix aggregate report processing - Check for the existence of a configuration file if a path is supplied +- Replace `publicsuffix` with `publicsuffix2` +- Add minimum versions to requirements 6.1.0 ----- diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index ffef046..adcfd51 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -24,7 +24,7 @@ import dns.exception import geoip2.database import geoip2.errors import requests -import publicsuffix +import publicsuffix2 __version__ = "6.1.1" @@ -110,7 +110,7 @@ def get_base_domain(domain): logger.warning( "Failed to download an updated PSL {0}".format(error)) with open(psl_path, encoding="utf-8") as psl_file: - psl = publicsuffix.PublicSuffixList(psl_file) + psl = publicsuffix2.PublicSuffixList(psl_file) return psl.get_public_suffix(domain) diff --git a/requirements.txt b/requirements.txt index fbf2d96..8e0ff1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,17 +5,16 @@ sphinx_rtd_theme wheel rstcheck>=3.3.1 pygments -dnspython -expiringdict +dnspython>=1.16.0 +expiringdict>=1.1.4 urllib3>=1.21.1 -requests -publicsuffix -xmltodict -geoip2 -dnspython -imapclient -mail-parser -dateparser -elasticsearch -elasticsearch-dsl -kafka-python +requests>=2.2.16.0 +publicsuffix2 +xmltodict>=0.12.0 +geoip2>=2.9.0 +imapclient>=2.1.0 +mail-parser>=3.9.2 +dateparser>=0.7.1 +elasticsearch>=6.3.1 +elasticsearch-dsl>=0.0.12 +kafka-python>=1.4.4