From 162fd425586afc4e891c38b6188d3008edc0d766 Mon Sep 17 00:00:00 2001 From: Watchful1 Date: Fri, 8 Nov 2024 17:50:16 -0800 Subject: [PATCH] Better logging --- personal/combine/merge_and_backfill.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/personal/combine/merge_and_backfill.py b/personal/combine/merge_and_backfill.py index e8e8f88..a327567 100644 --- a/personal/combine/merge_and_backfill.py +++ b/personal/combine/merge_and_backfill.py @@ -90,6 +90,8 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function): response = requests.get(url, headers={ 'User-Agent': "In script by /u/Watchful1", 'Authorization': f"Bearer {bearer}"}, timeout=20) + if i > 0: + log.info(f"Pushshift call succeeded after {i} retries") except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): log.info(f"Pushshift failed, sleeping {i * sleep_per_attempt}") time.sleep(i * sleep_per_attempt)