mirror of
https://github.com/alecmuffett/real-world-onion-sites.git
synced 2024-10-01 01:06:18 -04:00
annotater
This commit is contained in:
parent
8a872e3f60
commit
e9060a893c
2
.gitignore
vendored
2
.gitignore
vendored
@ -1 +1,3 @@
|
||||
*~
|
||||
out/*
|
||||
err/*
|
||||
|
12
annotate.sh
Normal file
12
annotate.sh
Normal file
@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
|
||||
src=README.md
|
||||
dst=${src}~~
|
||||
|
||||
for output in out/* ; do
|
||||
test ! -s $output || continue
|
||||
onion=`basename $output`
|
||||
echo annotating: $onion
|
||||
perl -pi "s/$/ :sos:/ if /$output/" <$src >$dst
|
||||
mv $dst $src
|
||||
done
|
26
checker.sh
Executable file
26
checker.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
|
||||
doc=https://raw.githubusercontent.com/alecmuffett/onion-sites-that-dont-suck/master/README.md
|
||||
tf=onions~txt~
|
||||
|
||||
for dir in out err ; do
|
||||
test -d $dir || mkdir $dir || exit 1
|
||||
done
|
||||
|
||||
if [ ! -s $tf ] ; then
|
||||
tor-curl $doc |
|
||||
perl -pe 's/\s+/\n/g' |
|
||||
perl -pe 's!^(https?://.*?/).+$!$1!' |
|
||||
sort -u |
|
||||
egrep '^https?://.*\.onion/' > $tf
|
||||
fi
|
||||
|
||||
for url in `randsort < $tf` ; do
|
||||
host=`basename $url`
|
||||
if [ ! -s out/$host ] ; then
|
||||
echo polling: $host `date`
|
||||
tor-curl $url >out/$host 2>err/$host
|
||||
else
|
||||
echo skipping: $host
|
||||
fi
|
||||
done
|
Loading…
Reference in New Issue
Block a user