Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-06 04:14:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

More logging

Browse source
This commit is contained in:
Watchful1 2024-11-08 17:52:18 -08:00
parent 162fd42558
commit 08e5e80e02
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
personal/combine/merge_and_backfill.py
View file

@ -92,11 +92,13 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
'Authorization': f"Bearer {bearer}"}, timeout=20)
if i > 0:
log.info(f"Pushshift call succeeded after {i} retries")
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}")
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as err:
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt} : {err}")
time.sleep(i * sleep_per_attempt)
continue
if response is None:
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt} : no response")
time.sleep(i * sleep_per_attempt)
continue
if response.status_code == 200:
break
@ -105,7 +107,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}")
bearer = pushshift_token_function(bearer)
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}")
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt} : status {response.status_code}")
time.sleep(i * sleep_per_attempt)
if response is None:
log.warning(f"{attempts} requests failed with no response")