Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-24 15:15:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

Bah

Browse source
This commit is contained in:
Watchful1 2024-11-09 23:14:53 -08:00
parent 04f5790d4f
commit 254942b6ce
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/combine/merge_and_backfill.py
View file

@ -93,11 +93,11 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
'Authorization': f"Bearer {bearer}"}, timeout=20) 'Authorization': f"Bearer {bearer}"}, timeout=20)
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as err: except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as err:
log.info(f"Pushshift failed, sleeping {current_attempt * sleep_per_attempt} : {err}") log.info(f"Pushshift failed, sleeping {current_attempt * sleep_per_attempt} : {err}")
time.sleep(i * sleep_per_attempt) time.sleep(current_attempt * sleep_per_attempt)
continue continue
if response is None: if response is None:
log.info(f"Pushshift failed, sleeping {current_attempt * sleep_per_attempt} : no response") log.info(f"Pushshift failed, sleeping {current_attempt * sleep_per_attempt} : no response")
time.sleep(i * sleep_per_attempt) time.sleep(current_attempt * sleep_per_attempt)
continue continue
if response.status_code == 200: if response.status_code == 200:
break break