mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-05 03:46:54 -04:00
Retry longer
This commit is contained in:
parent
fba4d6de99
commit
c959da912e
1 changed files with 9 additions and 5 deletions
|
@ -83,13 +83,16 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
|
|||
url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}"
|
||||
log.debug(f"pushshift query: {url}")
|
||||
response = None
|
||||
for i in range(10):
|
||||
attempts = 20
|
||||
sleep_per_attempt = 10
|
||||
for i in range(attempts):
|
||||
try:
|
||||
response = requests.get(url, headers={
|
||||
'User-Agent': "In script by /u/Watchful1",
|
||||
'Authorization': f"Bearer {bearer}"}, timeout=20)
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
|
||||
time.sleep(i)
|
||||
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}")
|
||||
time.sleep(i * sleep_per_attempt)
|
||||
continue
|
||||
if response is None:
|
||||
continue
|
||||
|
@ -100,15 +103,16 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
|
|||
log.warning(url)
|
||||
log.warning(f"'Authorization': Bearer {bearer}")
|
||||
bearer = pushshift_token_function(bearer)
|
||||
time.sleep(i)
|
||||
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}")
|
||||
time.sleep(i * sleep_per_attempt)
|
||||
if response is None:
|
||||
log.warning(f"4 requests failed with no response")
|
||||
log.warning(f"{attempts} requests failed with no response")
|
||||
log.warning(url)
|
||||
log.warning(f"'Authorization': Bearer {bearer}")
|
||||
discord_logging.flush_discord()
|
||||
sys.exit(1)
|
||||
if response.status_code != 200:
|
||||
log.warning(f"4 requests failed with status code {response.status_code}")
|
||||
log.warning(f"{attempts} requests failed with status code {response.status_code}")
|
||||
log.warning(url)
|
||||
log.warning(f"'Authorization': Bearer {bearer}")
|
||||
discord_logging.flush_discord()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue