remove v2 onions from CT log, and clean up CSV

ct-log.txt

@@ -1,62 +1,3 @@
-fyxifz6xwx3d67a2.onion
-k2hzvhcagvmx4us2.onion
-www.k2hzvhcagvmx4us2.onion
-voatibenjgi7dx33.onion
-*.voatibenjgi7dx33.onion
-*.www.voatibenjgi7dx33.onion
-try2checklm32oc3.onion
-www.try2checklm32oc3.onion
-pubapp7v22ykdou3.onion
-*.pubapp7v22ykdou3.onion
-itipwhiteopsdx64.onion
-xychelseaxqpywe4.onion
-www.xychelseaxqpywe4.onion
-tqyxkw5weugqkgk4.onion
-*.tqyxkw5weugqkgk4.onion
-qklykfiomrwjhdz4.onion
-*.qklykfiomrwjhdz4.onion
-*.cdnapisec.qklykfiomrwjhdz4.onion
-*.cfvod.qklykfiomrwjhdz4.onion
-*.analytics.qklykfiomrwjhdz4.onion
-*.stats.qklykfiomrwjhdz4.onion
-urdvoauhitx5icb5.onion
-*.urdvoauhitx5icb5.onion
-*.www.urdvoauhitx5icb5.onion
-jg2kqkrs2rapdiu5.onion
-jxtwbvj32chvkay5.onion
-*.jxtwbvj32chvkay5.onion
-sarajarxpvoenum6.onion
-vipebuzedutoa6p6.onion
-*.vipebuzedutoa6p6.onion
-www.vipebuzedutoa6p6.onion
-mkyivn5qd25lf7r6.onion
-*.mkyivn5qd25lf7r6.onion
-ppasset42kropoy6.onion
-*.ppasset42kropoy6.onion
-*.idelrea5soxeg7h7.onion
-*.www.idelrea5soxeg7h7.onion
-*.persgroepsttfzr7.onion
-*.idp.persgroepsttfzr7.onion
-*.dnr.persgroepsttfzr7.onion
-y6xjgkgwj47us5ca.onion
-www.y6xjgkgwj47us5ca.onion
-haiwnaestfknxbda.onion
-www.haiwnaestfknxbda.onion
-voacant5lzidz4ma.onion
-*.voacant5lzidz4ma.onion
-*.www.voacant5lzidz4ma.onion
-s5rhoqqosmcispfb.onion
-*.s5rhoqqosmcispfb.onion
-*.live.s5rhoqqosmcispfb.onion
-*.api.s5rhoqqosmcispfb.onion
-voachineksz65vsb.onion
-*.voachineksz65vsb.onion
-*.www.voachineksz65vsb.onion
-rferlcjay7gumqvb.onion
-*.rferlcjay7gumqvb.onion
-jcw5q6uyjioupxcc.onion
-www.jcw5q6uyjioupxcc.onion
-usatodayw7vu5egc.onion
 kushal76uaid62oup5774umh654scnu5dwzh4u2534qxhcbi4wbab3ad.onion
 ezdhgsy2aw7zg54z6dqsutrduhl22moami5zv2zt6urr6vub7gs6wfad.onion
 lukshelpqmximhkx4dd6374gqwwwur2fccw5kk7wfbdubyfip5eiugad.onion
@@ -72,9 +13,6 @@ h32njsj5f5dd4hbzxmupx76o7ktzybnrwp2c4eu7kirzhrvduq73ysad.onion
 ce2qbvaezhl33x5mv2y5nxfzcyorc7ylshkiubrppt7uooklv34uauad.onion
 duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion
 *.duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion
-kavkazr6vak4ftdd.onion
-*.kavkazr6vak4ftdd.onion
-*.www.kavkazr6vak4ftdd.onion
 archivev3qli37bju4rlh27glh24lljyezwxf4pokmrdbpefjlcrp5id.onion
 *.archivev3qli37bju4rlh27glh24lljyezwxf4pokmrdbpefjlcrp5id.onion
 *.us.archivev3qli37bju4rlh27glh24lljyezwxf4pokmrdbpefjlcrp5id.onion
@@ -92,8 +30,6 @@ hzwjmjimhr7bdmfv2doll4upibt5ojjmpo3pbp5ctwcg37n3hyk7qzid.onion
 *.hzwjmjimhr7bdmfv2doll4upibt5ojjmpo3pbp5ctwcg37n3hyk7qzid.onion
 6wvybf7ub3xk5ow66wt7os3aovbzoo2eei6vjirvhvvkmqg4alnezzid.onion
 www.6wvybf7ub3xk5ow66wt7os3aovbzoo2eei6vjirvhvvkmqg4alnezzid.onion
-tomritterbassljd.onion
-*.tomritterbassljd.onion
 kutj6adn66vdrpauox236wbc6g52ogg3pl4nfb5bfx7gzdgnqvhsl2qd.onion
 iiutnqstqrrygumyggv4t7vsrzf3dgd27rolrfkhkus4mg5i6yein4qd.onion
 27m3p2uv7igmj6kvd4ql3cct5h3sdwrsajovkkndeufumzyfhlfev4qd.onion
@@ -125,159 +61,3 @@ kuy2bbth3t7m46oidbhdq7os7ebnijgzdxhl3ttvpmijo7nqbopwizyd.onion
 projects.kuy2bbth3t7m46oidbhdq7os7ebnijgzdxhl3ttvpmijo7nqbopwizyd.onion
 *.projects.kuy2bbth3t7m46oidbhdq7os7ebnijgzdxhl3ttvpmijo7nqbopwizyd.onion
 protonmailrmez3lotccipshtkleegetolb73fuirgj7r4o4vfu7ozyd.onion
-graylady3jvrrxbe.onion
-*.graylady3jvrrxbe.onion
-*.stg.graylady3jvrrxbe.onion
-*.dev.graylady3jvrrxbe.onion
-deewamfupxj4cele.onion
-*.deewamfupxj4cele.onion
-*.gdb.deewamfupxj4cele.onion
-media.voltron.deewamfupxj4cele.onion
-*.media.voltron.deewamfupxj4cele.onion
-*.www.deewamfupxj4cele.onion
-privacyintyqcroe.onion
-*.privacyintyqcroe.onion
-www.privacyintyqcroe.onion
-marshouzr45aiykf.onion
-*.marshouzr45aiykf.onion
-*.www.marshouzr45aiykf.onion
-rfa62zl6z6owmtlf.onion
-*.rfa62zl6z6owmtlf.onion
-*.www.rfa62zl6z6owmtlf.onion
-erzs4cd7i5eox4cg.onion
-*.erzs4cd7i5eox4cg.onion
-*.tags.erzs4cd7i5eox4cg.onion
-sibrealdgxmk4gfg.onion
-*.sibrealdgxmk4gfg.onion
-*.www.sibrealdgxmk4gfg.onion
-nytimes3xbfgragh.onion
-*.nytimes3xbfgragh.onion
-*.prd.nytimes3xbfgragh.onion
-*.stg.nytimes3xbfgragh.onion
-*.blogs5.stg.nytimes3xbfgragh.onion
-*.api.stg.nytimes3xbfgragh.onion
-*.blogs.stg.nytimes3xbfgragh.onion
-*.api.nytimes3xbfgragh.onion
-*.blogs.nytimes3xbfgragh.onion
-*.stg.blogs.nytimes3xbfgragh.onion
-*.dev.blogs.nytimes3xbfgragh.onion
-www.bestsellers.nytimes3xbfgragh.onion
-*.dev.nytimes3xbfgragh.onion
-*.api.dev.nytimes3xbfgragh.onion
-*.newsdev.nytimes3xbfgragh.onion
-*.stg.newsdev.nytimes3xbfgragh.onion
-*.sbx.nytimes3xbfgragh.onion
-www.homedelivery.nytimes3xbfgragh.onion
-tess2mkorb55pl7i.onion
-www.tess2mkorb55pl7i.onion
-facebookcorewwwi.onion
-*.facebookcorewwwi.onion
-m.facebookcorewwwi.onion
-*.m.facebookcorewwwi.onion
-svobod65rq5ufpfj.onion
-*.svobod65rq5ufpfj.onion
-*.www.svobod65rq5ufpfj.onion
-bbpclp57ed3yhnqj.onion
-www.bbpclp57ed3yhnqj.onion
-golosctnxclzq4xj.onion
-*.golosctnxclzq4xj.onion
-*.www.golosctnxclzq4xj.onion
-brave5t5rjjg3s6k.onion
-*.brave5t5rjjg3s6k.onion
-*.s3.brave5t5rjjg3s6k.onion
-www.brave5t5rjjg3s6k.onion
-nytimes2tsqtnxek.onion
-blockchainbdgpzk.onion
-t3y3qp7zoduvisdl.onion
-www.t3y3qp7zoduvisdl.onion
-blockchatvqztbll.onion
-y27vf7g2ce5g3fnl.onion
-www.y27vf7g2ce5g3fnl.onion
-3g2upl4pq6kufc4m.onion
-www.3g2upl4pq6kufc4m.onion
-sgpsbq5hd3pxwl7m.onion
-www.sgpsbq5hd3pxwl7m.onion
-voatibetjrt52sem.onion
-*.voatibetjrt52sem.onion
-*.www.voatibetjrt52sem.onion
-azatliqmmvir2cxm.onion
-*.azatliqmmvir2cxm.onion
-*.www.azatliqmmvir2cxm.onion
-56m4joozvkzlx6ln.onion
-*.56m4joozvkzlx6ln.onion
-27p5nbsmdt5okqun.onion
-factogrepzbkwv3o.onion
-*.factogrepzbkwv3o.onion
-*.www.factogrepzbkwv3o.onion
-htikmirac7bzq75o.onion
-www.htikmirac7bzq75o.onion
-37nhmdszo2ltkkfo.onion
-*.37nhmdszo2ltkkfo.onion
-*.live.37nhmdszo2ltkkfo.onion
-*.api.37nhmdszo2ltkkfo.onion
-*.files.37nhmdszo2ltkkfo.onion
-*.test.37nhmdszo2ltkkfo.onion
-osf22p3lmweopgho.onion
-www.osf22p3lmweopgho.onion
-dwnewsvdyyiamwnp.onion
-*.dwnewsvdyyiamwnp.onion
-*.commons.dwnewsvdyyiamwnp.onion
-*.xczxxczkjivdfsop.onion
-pubdrop4dw6rk3aq.onion
-radiosvqf2b7qycq.onion
-*.radiosvqf2b7qycq.onion
-*.www.radiosvqf2b7qycq.onion
-cf6kvc4g63fnhkkq.onion
-*.cf6kvc4g63fnhkkq.onion
-fbcdn23dssr3jqnq.onion
-*.fbcdn23dssr3jqnq.onion
-xx.fbcdn23dssr3jqnq.onion
-*.xx.fbcdn23dssr3jqnq.onion
-xy.fbcdn23dssr3jqnq.onion
-*.xy.fbcdn23dssr3jqnq.onion
-xz.fbcdn23dssr3jqnq.onion
-*.xz.fbcdn23dssr3jqnq.onion
-wcoc6qfoyfmrqj2r.onion
-*.wcoc6qfoyfmrqj2r.onion
-coinpaymtstgtibr.onion
-orionyg556j7qidr.onion
-amcd7mdy354fajrr.onion
-cyphdbyhiddenbhs.onion
-*.cyphdbyhiddenbhs.onion
-im.cyphdbyhiddenbhs.onion
-video.cyphdbyhiddenbhs.onion
-audio.cyphdbyhiddenbhs.onion
-cyph.ws.cyphdbyhiddenbhs.onion
-voanewsktwx4bxhs.onion
-*.voanewsktwx4bxhs.onion
-ajarouh33jee5tos.onion
-greyrallymarqk3t.onion
-p5xhdvsjmnlmeo6v.onion
-*.p5xhdvsjmnlmeo6v.onion
-www.p5xhdvsjmnlmeo6v.onion
-propub3r6espa33w.onion
-*.propub3r6espa33w.onion
-protonirockerxow.onion
-fbsbx2q4mvcl63pw.onion
-*.fbsbx2q4mvcl63pw.onion
-nyttips4bmquxfzw.onion
-secrdrop5wyphb5x.onion
-www.secrdrop5wyphb5x.onion
-container656ovix.onion
-*.container656ovix.onion
-sgeffsn6tztqzlpx.onion
-5qlhz3jow7zhxtpx.onion
-k6jyto7zq7xxriux.onion
-www.k6jyto7zq7xxriux.onion
-btcforuml2pcfkyx.onion
-www.btcforuml2pcfkyx.onion
-bfnews3u2ox4m4ty.onion
-www.bfnews3u2ox4m4ty.onion
-bbcnewsv2vjtpsuy.onion
-*.bbcnewsv2vjtpsuy.onion
-*.live.bbcnewsv2vjtpsuy.onion
-*.bidi.live.bbcnewsv2vjtpsuy.onion
-*.bidi.bbcnewsv2vjtpsuy.onion
-*.api.bbcnewsv2vjtpsuy.onion
-*.iplayer.api.bbcnewsv2vjtpsuy.onion
-*.notifications.api.bbcnewsv2vjtpsuy.onion

get-ct-log.sh

@@ -5,7 +5,7 @@ tf=/tmp/ctget$$.txt
 
 curl "https://crt.sh/?q=%25.onion" |
     perl -nle 'next unless m!TD.*\.onion\b!; s!\s+!\n!go; s!</?TD>!\n!goi; s!<BR>!\n!goi; print' |
-    egrep '\.onion$' |
+    egrep '[2-7a-z]{56}\.onion$' |
     sort -u |
     rev |
     sort |