Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-22 22:30:49 -04:00
Code Issues Projects Releases Packages Wiki Activity

More retries

Browse source
This commit is contained in:
Watchful1 2024-11-10 11:30:02 -08:00
parent 254942b6ce
commit 314ed1f1c9
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
personal/combine/merge_and_backfill.py
View file

@ -83,7 +83,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}"
log.debug(f"pushshift query: {url}")
response = None
total_attempts = 50
total_attempts = 100
current_attempt = 0
sleep_per_attempt = 10
for current_attempt in range(total_attempts):
@ -109,19 +109,19 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
log.info(f"Pushshift failed, sleeping {current_attempt * sleep_per_attempt} : status {response.status_code}")
time.sleep(current_attempt * sleep_per_attempt)
if response is None:
log.warning(f"{current_attempt} requests failed with no response")
log.warning(f"{current_attempt + 1} requests failed with no response")
log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}")
discord_logging.flush_discord()
sys.exit(1)
if response.status_code != 200:
log.warning(f"{current_attempt} requests failed with status code {response.status_code}")
log.warning(f"{current_attempt + 1} requests failed with status code {response.status_code}")
log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}")
discord_logging.flush_discord()
sys.exit(1)
if current_attempt > 0:
log.info(f"Pushshift call succeeded after {current_attempt} retries")
log.info(f"Pushshift call succeeded after {current_attempt + 1} retries")
return response.json()['data'], bearer