Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-22 22:30:49 -04:00
Code Issues Projects Releases Packages Wiki Activity

Way longer

Browse source
This commit is contained in:
Watchful1 2025-03-09 13:18:58 -07:00
parent 7347fae28a
commit 2cc3e3f68e
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
personal/combine/merge_and_backfill.py
View file

@ -129,13 +129,16 @@ def query_reddit(ids, reddit, object_type):
id_prefix = 't1_' if object_type == ObjectType.COMMENT else 't3_' id_prefix = 't1_' if object_type == ObjectType.COMMENT else 't3_'
id_string = f"{id_prefix}{(f',{id_prefix}'.join(ids))}" id_string = f"{id_prefix}{(f',{id_prefix}'.join(ids))}"
response = None response = None
for i in range(10): for i in range(100):
try: try:
response = reddit.request(method="GET", path=endpoints.API_PATH["info"], params={"id": id_string}) response = reddit.request(method="GET", path=endpoints.API_PATH["info"], params={"id": id_string})
break break
except (prawcore.exceptions.ServerError, prawcore.exceptions.RequestException) as err: except (prawcore.exceptions.ServerError, prawcore.exceptions.RequestException) as err:
log.info(f"No response from reddit api: {err} : {id_string}") log.info(f"No response from reddit api for {object_type}, sleeping {i * 5} seconds: {err} : {id_string}")
time.sleep(i * 5) time.sleep(i * 5)
if response is None:
log.warning(f"Reddit api failed, aborting")
raise IOError
return response['data']['children'] return response['data']['children']