From c959da912e3a9e5dcfd70c61ffd046c457f75abc Mon Sep 17 00:00:00 2001 From: Watchful1 Date: Fri, 8 Nov 2024 17:47:58 -0800 Subject: [PATCH] Retry longer --- personal/combine/merge_and_backfill.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/personal/combine/merge_and_backfill.py b/personal/combine/merge_and_backfill.py index 4617bcb..e8e8f88 100644 --- a/personal/combine/merge_and_backfill.py +++ b/personal/combine/merge_and_backfill.py @@ -83,13 +83,16 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function): url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}" log.debug(f"pushshift query: {url}") response = None - for i in range(10): + attempts = 20 + sleep_per_attempt = 10 + for i in range(attempts): try: response = requests.get(url, headers={ 'User-Agent': "In script by /u/Watchful1", 'Authorization': f"Bearer {bearer}"}, timeout=20) except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): - time.sleep(i) + log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}") + time.sleep(i * sleep_per_attempt) continue if response is None: continue @@ -100,15 +103,16 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function): log.warning(url) log.warning(f"'Authorization': Bearer {bearer}") bearer = pushshift_token_function(bearer) - time.sleep(i) + log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}") + time.sleep(i * sleep_per_attempt) if response is None: - log.warning(f"4 requests failed with no response") + log.warning(f"{attempts} requests failed with no response") log.warning(url) log.warning(f"'Authorization': Bearer {bearer}") discord_logging.flush_discord() sys.exit(1) if response.status_code != 200: - log.warning(f"4 requests failed with status code {response.status_code}") + log.warning(f"{attempts} requests failed with status code {response.status_code}") log.warning(url) log.warning(f"'Authorization': Bearer {bearer}") discord_logging.flush_discord()