Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 14:50:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

More retries

Browse source
This commit is contained in:
Watchful1 2024-09-19 20:00:21 -07:00
parent 679728d7d1
commit 2fff2ad373
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
personal/combine/merge_and_backfill.py
View file

@ -83,13 +83,13 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}"
log.debug(f"pushshift query: {url}")
response = None
for i in range(4):
for i in range(10):
try:
response = requests.get(url, headers={
'User-Agent': "In script by /u/Watchful1",
'Authorization': f"Bearer {bearer}"}, timeout=15)
'Authorization': f"Bearer {bearer}"}, timeout=20)
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
time.sleep(4 * i)
time.sleep(i)
continue
if response is None:
continue
@ -100,7 +100,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}")
bearer = pushshift_token_function(bearer)
time.sleep(4 * i)
time.sleep(i)
if response is None:
log.warning(f"4 requests failed with no response")
log.warning(url)