Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-29 09:28:41 -04:00
Code Issues Projects Releases Packages Wiki Activity

More logging here

Browse source
This commit is contained in:
Watchful1 2023-09-18 18:07:14 -07:00
parent 4dab643fff
commit ab2a94c241
1 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
personal/combine/build_day.py
View file

@ -39,9 +39,11 @@ def load_pushshift_token():
def re_auth_pushshift(old_token):
response = requests.post(f"https://auth.pushshift.io/refresh?access_token={old_token}")
url = f"https://auth.pushshift.io/refresh?access_token={old_token}"
log.warning(f"Reauth request: {url}")
response = requests.post(url)
result = response.json()
log.warning(f"Reauth responce: {str(result)}")
log.warning(f"Reauth response: {str(result)}")
new_token = result['access_token']
log.warning(f"New pushshift token: {new_token}")
save_pushshift_token(new_token)
@ -67,7 +69,9 @@ def query_pushshift(ids, bearer, object_type):
if response.status_code == 200:
break
if response.status_code == 403:
log.warning(f"Pushshift unauthorized, trying reauth")
log.warning(f"Pushshift 403, trying reauth")
log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}")
bearer = re_auth_pushshift(bearer)
time.sleep(2)
if response.status_code != 200:
@ -235,6 +239,7 @@ if __name__ == "__main__":
)
if args.pushshift is not None:
log.warning(f"Saving pushshift token: {args.pushshift}")
save_pushshift_token(args.pushshift)
while start_date <= end_date: