diff --git a/build.sh b/build.sh index 4a1e5ffa..f2a8f7bc 100755 --- a/build.sh +++ b/build.sh @@ -17,7 +17,7 @@ touch build/html/.nojekyll if [ -d "./../parsedmarc-docs" ]; then cp -rf build/html/* ../../parsedmarc-docs/ fi -sort.sh +python3 sortmaps.py python3 tests.py rm -rf dist/ build/ -hatch build \ No newline at end of file +hatch build diff --git a/parsedmarc/resources/maps/base_reverse_dns_map.csv b/parsedmarc/resources/maps/base_reverse_dns_map.csv index 252246b1..07937853 100644 --- a/parsedmarc/resources/maps/base_reverse_dns_map.csv +++ b/parsedmarc/resources/maps/base_reverse_dns_map.csv @@ -44,6 +44,7 @@ airtel.in,Airtel,ISP airtelbroadband.in,Airtel,ISP ajusa.com,AJ-USA,Retail aknet.kg,AkNet,ISP +akura.ne.jp,Akura,Logistics albatros.uz,Albatros Health Care,Healthcare alembic.co.in,Alembic Pharmaceuticals,Healthcare alestra.net.mx,Alestra,ISP @@ -88,6 +89,7 @@ asl2.liguria.it,ASL2,Healthcare asmecal.it,ASMENET CALABRIA,Government assaytechnology.com,Assay Technology,Industrial assentportal.com,Assent,SaaS +assp.org,American Society of Safety Professionals,Healthcare atmailcloud.com,atmail,Email Provider ats.ca,ATS Healthcare,Healthcare atw.ne.jp,ATW,Web Host @@ -135,6 +137,7 @@ bluehost.com,Bluehost,Web Host bluemission.net,Blue Mission,Web Host bluesea.org,Blue Sea Foundation,Marketing bluetie.com,BlueTie,MSP +bnpparibas.fr,Banque BNP Paribas,Finance bohc.co.jp,Benefit One Inc.,Retail bostrad.com,Bostrad Supply Chain Ltd,Logistics bov.com,Bank of Valletta,Finance @@ -192,6 +195,7 @@ circleamedical.com,Circle A Medical,Healthcare cirrushosting.com,Cirrus Hosting,Web Host cisco.com,Cisco,Technology ckt.net,Craw-Kan Telephone Cooperative,ISP +clarix.com,Clarix,MSP claro.com.do,Claro,ISP claro.com.ec,Claro,ISP claro.net.do,Mail2World,Email Provider @@ -213,6 +217,7 @@ codetel.net.do,codetel.net.do,ISP coeficiente.net.mx,Coeficiente,ISP cogeco.ca,Cogeco,ISP cogeco.net,Cogoco,ISP +collectivhosting.com,Collectiv,Web Host colocrossing.com,ColoCrossing,ISP combell.com,Combell,Web Host comcast.net,Comcast,ISP @@ -220,6 +225,7 @@ comcastbusiness.net,Comcast Business,ISP comfori.com,Comfori,SaaS comline.com,West Coast Internet (WCI),ISP communitect.com,Solutionreach,SaaS +complemar.com,Complemar,Logistics compliancearchitects.net,Compliance Aarchitects,Healthcare compu-type.net,Compu-Type,MSP computronics.com,Computronics,Technology @@ -1322,6 +1328,7 @@ verizonbusiness.com,Verizon Business,ISP verointernet.com.br,Internet de Verdade,ISP versanet.de,1&1 Versatel,ISP vicc.co,Venco Imtiaz Contracting Co,Industrial +victorkaiser.com,Global Transport,Logistics videotron.ca,Videotron,ISP videotron.com,Videotron,ISP viecuri.nl,VieCuri,Healthcare @@ -1435,10 +1442,3 @@ zsttk.ru,TTK,ISP zyner.net,Zyner,Email Provider zyner.one,Zyner,Email Provider zyner.org,Zyner,Email Provider -akura.ne.jp,Akura,Logistics -assp.org,American Society of Safety Professionals,Healthcare -complemar.com,Complemar,Logistics -victorkaiser.com,Global Transport,Logistics -bnpparibas.fr,Banque BNP Paribas,Finance -clarix.com,Clarix,MSP -collectivhosting.com,Collectiv,Web Host diff --git a/parsedmarc/resources/maps/known_unknown_base_reverse_dns.txt b/parsedmarc/resources/maps/known_unknown_base_reverse_dns.txt index 5dc01807..59a3fb87 100644 --- a/parsedmarc/resources/maps/known_unknown_base_reverse_dns.txt +++ b/parsedmarc/resources/maps/known_unknown_base_reverse_dns.txt @@ -1,10 +1,15 @@ 200.in-addr.arpa +a7e.ru +a94434500-blog.com adlucrumnewsletter.com admin.corpivensa.gob.ve aerospacevitro.us.com albagroup-eg.com +anglishment.com anteldata.net.uy +antis.edu antonaoll.com +anviklass.org aosau.net arandomserver.com asmecam.it @@ -12,24 +17,30 @@ b8sales.com bestinvestingtime.com biocorp.com bisno1.co.jp +blguss.com +bluenet.ch bluhosting.com bodiax.pp.ua bost-law.com brnonet.cz brushinglegal.de +cavabeen.com christus.mx cloud-edm.com +cloudaccess.net cloudlogin.co cnode.io commerceinsurance.com coolblaze.com cps.com.ar +ctla.co.kr detrot.xyz digi.net.my dkginternet.com doorsrv.com dreamtechmedia.com ds.network +emailgids.net emailperegrine.com epsilon-group.com eyecandyhosting.xyz @@ -48,16 +59,21 @@ hostingmichigan.com hostname.localhost hostnetwork.com hostwhitelabel.com +i-mecca.net idcfcloud.net immenzaces.com +ip-147-135-108.us +itsidc.com ivol.co jalanet.co.id +jimishare.com kahlaa.com kbronet.com.tw kdnursing.org kitchenaildbd.com legenditds.com lighthouse-media.com +llsend.com lohkal.com lonestarmm.net magnetmail.net @@ -75,6 +91,7 @@ mysagestore.com ncport.ru nebdig.com neovet-base.ru +netkl.org nic.name nidix.net ogicom.net @@ -86,10 +103,12 @@ passionatesmiles.com planethoster.net pmnhost.net popiup.com +ports.net prima.com.ar prima.net.ar proudserver.com qontenciplc.autos +rapidns.com raxa.host sahacker-2020.com samsales.site @@ -123,22 +142,3 @@ xsfati.us.com xspmail.jp zerowebhosting.net znlc.jp -cavabeen.com -llsend.com -blguss.com -itsidc.com -anviklass.org -a7e.ru -antis.edu -ctla.co.kr -ip-147-135-108.us -cloudaccess.net -netkl.org -bluenet.ch -i-mecca.net -emailgids.net -jimishare.com -anglishment.com -ports.net -rapidns.com -a94434500-blog.com diff --git a/parsedmarc/resources/maps/psl_overrides.txt b/parsedmarc/resources/maps/psl_overrides.txt index 45789fa7..2fd7563b 100644 --- a/parsedmarc/resources/maps/psl_overrides.txt +++ b/parsedmarc/resources/maps/psl_overrides.txt @@ -1,6 +1,6 @@ +akura.ne.jp amazonaws.com cloudaccess.net -linode.comn h-serv.co.uk +linode.comn plesk.pages -akura.ne.jp diff --git a/sort.sh b/sort.sh deleted file mode 100755 index b8b92d47..00000000 --- a/sort.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -sort -o "parsedmarc/resources/maps/known_unknown_base_reverse_dns.txt" "parsedmarc/resources/maps/known_unknown_base_reverse_dns.txt" -sort -o "parsedmarc/resources/maps/public_suffix_overrides.txt" "parsedmarc/resources/maps/public_suffix_overrides.txt" -./sortmaps.py diff --git a/sortmaps.py b/sortmaps.py index 4653f643..35b599b5 100755 --- a/sortmaps.py +++ b/sortmaps.py @@ -23,8 +23,16 @@ def sort_csv(filepath, column=0): writer.writerow(header) writer.writerows(sorted_rows) -def sort_list_file(filepath, lowercase=True, strip=True, deduplicate=True, - remove_blank_lines=True, ending_newline=True, newline="\n"): + +def sort_list_file( + filepath, + lowercase=True, + strip=True, + deduplicate=True, + remove_blank_lines=True, + ending_newline=True, + newline="\n", +): with open(filepath, mode="r", newline=newline) as infile: lines = infile.readlines() for i in range(len(lines)): @@ -39,14 +47,13 @@ def sort_list_file(filepath, lowercase=True, strip=True, deduplicate=True, lines.remove("") lines = sorted(lines) if ending_newline: - if lines[-1] != newline: - lines.append(newline) + if lines[-1] != "": + lines.append("") with open(filepath, mode="w", newline=newline) as outfile: - outfile.write("\n".join(newline)) - outfile.write(newline) + outfile.write("\n".join(lines)) for csv_file in map_files: sort_csv(os.path.join(maps_dir, csv_file)) for list_file in list_files: - sort_list_file( os.path.join(maps_dir, list_file)) + sort_list_file(os.path.join(maps_dir, list_file))