Git-Mirrors/real-world-onion-sites
1
0
Fork 0
mirror of https://github.com/alecmuffett/real-world-onion-sites.git synced 2024-10-01 01:06:18 -04:00
Code Issues Packages Projects Releases Wiki Activity

commit: raspbian python version amendments

Browse Source
This commit is contained in:
Alec Muffett 2019-11-16 14:52:12 +00:00
parent 800f46e278
commit 55bd63a5b3
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -14,4 +14,4 @@ db:
sqlite3 $(DB) sqlite3 $(DB)
db-nuke: clean db-nuke: clean
-rm $(DB) $(DB)-* -rm $(DB) $(DB)-* master.csv

7
rwos-db.py
View File

@ -18,7 +18,7 @@ BADNESS = 900
CURL_TIMEOUT = 120 CURL_TIMEOUT = 120
RETRY_SLEEP = 60 RETRY_SLEEP = 60
PLACEHOLDER = '-' PLACEHOLDER = '-'
POOL_WORKERS = 8 POOL_WORKERS = 10
YES = 'y' YES = 'y'
EMOJI_UNSET = ':question:' EMOJI_UNSET = ':question:'
@ -123,10 +123,11 @@ class URL:
def fetch1(self): def fetch1(self):
args = [ 'curl', '--head', '--user-agent', USER_AGENT, '--proxy', SOCKS_PROXY, self.url ] args = [ 'curl', '--head', '--user-agent', USER_AGENT, '--proxy', SOCKS_PROXY, self.url ]
time.sleep(1) # slight breathing space because MP
try: try:
p = subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) p = subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # todo: text=True
(out, err) = p.communicate(timeout=CURL_TIMEOUT) (out, err) = p.communicate(timeout=CURL_TIMEOUT)
hcode = extract_hcode(out) hcode = extract_hcode(str(out)) # str() not needed if text=True
if hcode == 200: err = PLACEHOLDER if hcode == 200: err = PLACEHOLDER
ecode = p.returncode ecode = p.returncode
except subprocess.TimeoutExpired as e: except subprocess.TimeoutExpired as e: