Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-28 17:14:17 -04:00
Code Issues Projects Releases Packages Wiki Activity

Catch connection errors here

Browse source
This commit is contained in:
Watchful1 2023-08-28 19:36:25 -07:00
parent 361f35a49e
commit 92ac7b7a35
1 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/combine/build_day.py
View file

@ -31,9 +31,13 @@ def query_pushshift(ids, bearer, object_type):
log.debug(f"pushshift query: {url}") log.debug(f"pushshift query: {url}")
response = None response = None
for i in range(4): for i in range(4):
try:
response = requests.get(url, headers={ response = requests.get(url, headers={
'User-Agent': "In script by /u/Watchful1", 'User-Agent': "In script by /u/Watchful1",
'Authorization': f"Bearer {bearer}"}, timeout=15) 'Authorization': f"Bearer {bearer}"}, timeout=15)
except ConnectionError:
time.sleep(2)
continue
if response.status_code == 200: if response.status_code == 200:
break break
if response.status_code == 403: if response.status_code == 403: