Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-24 15:15:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

Better logging

Browse source
This commit is contained in:
Watchful1 2024-11-08 17:50:16 -08:00
parent c959da912e
commit 162fd42558
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
personal/combine/merge_and_backfill.py
View file

@ -90,6 +90,8 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
response = requests.get(url, headers={ response = requests.get(url, headers={
'User-Agent': "In script by /u/Watchful1", 'User-Agent': "In script by /u/Watchful1",
'Authorization': f"Bearer {bearer}"}, timeout=20) 'Authorization': f"Bearer {bearer}"}, timeout=20)
if i > 0:
log.info(f"Pushshift call succeeded after {i} retries")
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}") log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}")
time.sleep(i * sleep_per_attempt) time.sleep(i * sleep_per_attempt)