diff --git a/manual-check.sh b/manual-check.sh new file mode 100755 index 0000000..a40685c --- /dev/null +++ b/manual-check.sh @@ -0,0 +1,8 @@ +#!/bin/sh +SOCKS_PROXY='socks5h://127.0.0.1:9150/' +USER_AGENT='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0' +for url in "$@" ; do + echo ":::: $url ::::" + curl --head --user-agent "$USER_AGENT" --proxy "$SOCKS_PROXY" "$url" || exit $? +done +exit 0 diff --git a/rwos-db.py b/rwos-db.py index 8c003ab..ac0efc7 100755 --- a/rwos-db.py +++ b/rwos-db.py @@ -13,7 +13,7 @@ GLOBAL_DB = None # has to be a global because pickling :-( MASTER_CSV = 'master.csv' DB_FILENAME = 'fetch.sqlite3' SOCKS_PROXY = 'socks5h://127.0.0.1:9150/' -USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0" +USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0' BADNESS = 900 CURL_TIMEOUT = 120 RETRY_SLEEP = 60