commit: amend quote marks, and add manual-check.sh

This commit is contained in:
Alec Muffett 2019-11-21 12:06:53 +00:00
parent 649d5306cb
commit 7cf8b47ec2
2 changed files with 9 additions and 1 deletions

8
manual-check.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/sh
SOCKS_PROXY='socks5h://127.0.0.1:9150/'
USER_AGENT='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0'
for url in "$@" ; do
echo ":::: $url ::::"
curl --head --user-agent "$USER_AGENT" --proxy "$SOCKS_PROXY" "$url" || exit $?
done
exit 0

View File

@ -13,7 +13,7 @@ GLOBAL_DB = None # has to be a global because pickling :-(
MASTER_CSV = 'master.csv'
DB_FILENAME = 'fetch.sqlite3'
SOCKS_PROXY = 'socks5h://127.0.0.1:9150/'
USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0"
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0'
BADNESS = 900
CURL_TIMEOUT = 120
RETRY_SLEEP = 60