From 91159f05092bff18ba7438e1ba639c674227adec Mon Sep 17 00:00:00 2001 From: Watchful1 Date: Wed, 13 Sep 2023 10:05:32 -0700 Subject: [PATCH] Add merge field --- personal/combine/build_day.py | 2 +- personal/combine/merge.py | 2 +- personal/transform/split_by_minutes.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/personal/combine/build_day.py b/personal/combine/build_day.py index 0fe21f8..7e03057 100644 --- a/personal/combine/build_day.py +++ b/personal/combine/build_day.py @@ -66,7 +66,7 @@ def query_reddit(ids, reddit, object_type): try: response = reddit.request(method="GET", path=endpoints.API_PATH["info"], params={"id": id_string}) break - except prawcore.exceptions.ServerError: + except (prawcore.exceptions.ServerError, prawcore.exceptions.RequestException): time.sleep(2) return response['data']['children'] diff --git a/personal/combine/merge.py b/personal/combine/merge.py index 2e67a20..8ee0e42 100644 --- a/personal/combine/merge.py +++ b/personal/combine/merge.py @@ -155,7 +155,7 @@ field_actions = { "contest_mode": FieldAction.OVERWRITE, "created": FieldAction.OVERWRITE_IF_NONE, "created_utc": FieldAction.OVERWRITE_IF_NONE, - "crosspost_parent": FieldAction.ALLOW, + "crosspost_parent": FieldAction.OVERWRITE_NOT_NONE, "crosspost_parent_list": FieldAction.OVERWRITE_NOT_NONE, "discussion_type": FieldAction.OVERWRITE_NOT_NONE, "distinguished": FieldAction.OVERWRITE, diff --git a/personal/transform/split_by_minutes.py b/personal/transform/split_by_minutes.py index c814a73..e7b1702 100644 --- a/personal/transform/split_by_minutes.py +++ b/personal/transform/split_by_minutes.py @@ -12,8 +12,8 @@ NEWLINE_ENCODED = "\n".encode('utf-8') if __name__ == "__main__": - input_file = r"\\MYCLOUDPR4100\Public\RS_2023-07.zst" - output_folder = r"\\MYCLOUDPR4100\Public\ingest\download" + input_file = r"\\MYCLOUDPR4100\Public\RS_2023-04.zst" + output_folder = r"\\MYCLOUDPR4100\Public\ingest\download2" file_type = "comments" if "RC" in input_file else "submissions" previous_minute, output_handle, created_utc = None, None, None