Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-25 15:45:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

More logging

Browse source
This commit is contained in:
Watchful1 2024-11-08 17:52:18 -08:00
parent 162fd42558
commit 08e5e80e02
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
personal/combine/merge_and_backfill.py
View file

@ -92,11 +92,13 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
'Authorization': f"Bearer {bearer}"}, timeout=20) 'Authorization': f"Bearer {bearer}"}, timeout=20)
if i > 0: if i > 0:
log.info(f"Pushshift call succeeded after {i} retries") log.info(f"Pushshift call succeeded after {i} retries")
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as err:
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}") log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt} : {err}")
time.sleep(i * sleep_per_attempt) time.sleep(i * sleep_per_attempt)
continue continue
if response is None: if response is None:
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt} : no response")
time.sleep(i * sleep_per_attempt)
continue continue
if response.status_code == 200: if response.status_code == 200:
break break
@ -105,7 +107,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
log.warning(url) log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}") log.warning(f"'Authorization': Bearer {bearer}")
bearer = pushshift_token_function(bearer) bearer = pushshift_token_function(bearer)
log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}") log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt} : status {response.status_code}")
time.sleep(i * sleep_per_attempt) time.sleep(i * sleep_per_attempt)
if response is None: if response is None:
log.warning(f"{attempts} requests failed with no response") log.warning(f"{attempts} requests failed with no response")