annotater

This commit is contained in:
Alec Muffett 2018-06-12 00:52:08 +01:00
parent 8a872e3f60
commit e9060a893c
3 changed files with 40 additions and 0 deletions

2
.gitignore vendored
View File

@ -1 +1,3 @@
*~
out/*
err/*

12
annotate.sh Normal file
View File

@ -0,0 +1,12 @@
#!/bin/sh
src=README.md
dst=${src}~~
for output in out/* ; do
test ! -s $output || continue
onion=`basename $output`
echo annotating: $onion
perl -pi "s/$/ :sos:/ if /$output/" <$src >$dst
mv $dst $src
done

26
checker.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/sh
doc=https://raw.githubusercontent.com/alecmuffett/onion-sites-that-dont-suck/master/README.md
tf=onions~txt~
for dir in out err ; do
test -d $dir || mkdir $dir || exit 1
done
if [ ! -s $tf ] ; then
tor-curl $doc |
perl -pe 's/\s+/\n/g' |
perl -pe 's!^(https?://.*?/).+$!$1!' |
sort -u |
egrep '^https?://.*\.onion/' > $tf
fi
for url in `randsort < $tf` ; do
host=`basename $url`
if [ ! -s out/$host ] ; then
echo polling: $host `date`
tor-curl $url >out/$host 2>err/$host
else
echo skipping: $host
fi
done