From 2fff2ad373de6a9d8ee1d49342412c9c3043ecd6 Mon Sep 17 00:00:00 2001 From: Watchful1 Date: Thu, 19 Sep 2024 20:00:21 -0700 Subject: [PATCH] More retries --- personal/combine/merge_and_backfill.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/personal/combine/merge_and_backfill.py b/personal/combine/merge_and_backfill.py index 80424d8..4617bcb 100644 --- a/personal/combine/merge_and_backfill.py +++ b/personal/combine/merge_and_backfill.py @@ -83,13 +83,13 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function): url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}" log.debug(f"pushshift query: {url}") response = None - for i in range(4): + for i in range(10): try: response = requests.get(url, headers={ 'User-Agent': "In script by /u/Watchful1", - 'Authorization': f"Bearer {bearer}"}, timeout=15) + 'Authorization': f"Bearer {bearer}"}, timeout=20) except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): - time.sleep(4 * i) + time.sleep(i) continue if response is None: continue @@ -100,7 +100,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function): log.warning(url) log.warning(f"'Authorization': Bearer {bearer}") bearer = pushshift_token_function(bearer) - time.sleep(4 * i) + time.sleep(i) if response is None: log.warning(f"4 requests failed with no response") log.warning(url)