Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-31 18:38:44 -04:00
Code Issues Projects Releases Packages Wiki Activity

More logging here

Browse source
This commit is contained in:
Watchful1 2023-09-18 18:07:14 -07:00
parent 4dab643fff
commit ab2a94c241
1 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
personal/combine/build_day.py
View file

@ -39,9 +39,11 @@ def load_pushshift_token():
def re_auth_pushshift(old_token): def re_auth_pushshift(old_token):
response = requests.post(f"https://auth.pushshift.io/refresh?access_token={old_token}") url = f"https://auth.pushshift.io/refresh?access_token={old_token}"
log.warning(f"Reauth request: {url}")
response = requests.post(url)
result = response.json() result = response.json()
log.warning(f"Reauth responce: {str(result)}") log.warning(f"Reauth response: {str(result)}")
new_token = result['access_token'] new_token = result['access_token']
log.warning(f"New pushshift token: {new_token}") log.warning(f"New pushshift token: {new_token}")
save_pushshift_token(new_token) save_pushshift_token(new_token)
@ -67,7 +69,9 @@ def query_pushshift(ids, bearer, object_type):
if response.status_code == 200: if response.status_code == 200:
break break
if response.status_code == 403: if response.status_code == 403:
log.warning(f"Pushshift unauthorized, trying reauth") log.warning(f"Pushshift 403, trying reauth")
log.warning(url)
log.warning(f"'Authorization': Bearer {bearer}")
bearer = re_auth_pushshift(bearer) bearer = re_auth_pushshift(bearer)
time.sleep(2) time.sleep(2)
if response.status_code != 200: if response.status_code != 200:
@ -235,6 +239,7 @@ if __name__ == "__main__":
) )
if args.pushshift is not None: if args.pushshift is not None:
log.warning(f"Saving pushshift token: {args.pushshift}")
save_pushshift_token(args.pushshift) save_pushshift_token(args.pushshift)
while start_date <= end_date: while start_date <= end_date: