annotater

.gitignore

@@ -1 +1,3 @@
 *~
+out/*
+err/*

annotate.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+src=README.md
+dst=${src}~~
+
+for output in out/* ; do
+    test ! -s $output || continue
+    onion=`basename $output`
+    echo annotating: $onion
+    perl -pi "s/$/ :sos:/ if /$output/" <$src >$dst
+    mv $dst $src
+done

checker.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+doc=https://raw.githubusercontent.com/alecmuffett/onion-sites-that-dont-suck/master/README.md
+tf=onions~txt~
+
+for dir in out err ; do
+    test -d $dir || mkdir $dir || exit 1
+done
+
+if [ ! -s $tf ] ; then
+    tor-curl $doc |
+        perl -pe 's/\s+/\n/g' |
+        perl -pe 's!^(https?://.*?/).+$!$1!' |
+        sort -u |
+        egrep '^https?://.*\.onion/' > $tf
+fi
+
+for url in `randsort < $tf` ; do
+    host=`basename $url`
+    if [ ! -s out/$host ] ; then
+        echo polling: $host `date`
+        tor-curl $url >out/$host 2>err/$host
+    else
+        echo skipping: $host
+    fi
+done