Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-24 15:15:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

Longer sleeps

Browse source
This commit is contained in:
Watchful1 2024-09-16 19:33:35 -07:00
parent ed12d9c4ca
commit db2c739b6d
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/combine/merge_and_backfill.py
View file

@ -89,7 +89,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
'User-Agent': "In script by /u/Watchful1", 'User-Agent': "In script by /u/Watchful1",
'Authorization': f"Bearer {bearer}"}, timeout=15) 'Authorization': f"Bearer {bearer}"}, timeout=15)
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
time.sleep(2) time.sleep(4 * i)
continue continue
if response is None: if response is None:
continue continue
@ -100,7 +100,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
log.warning(url) log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}") log.warning(f"'Authorization': Bearer {bearer}")
bearer = pushshift_token_function(bearer) bearer = pushshift_token_function(bearer)
time.sleep(2) time.sleep(4 * i)
if response is None: if response is None:
log.warning(f"4 requests failed with no response") log.warning(f"4 requests failed with no response")
log.warning(url) log.warning(url)