mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-03 02:46:40 -04:00
Add merge field
This commit is contained in:
parent
9595fa7e40
commit
91159f0509
3 changed files with 4 additions and 4 deletions
|
@ -66,7 +66,7 @@ def query_reddit(ids, reddit, object_type):
|
|||
try:
|
||||
response = reddit.request(method="GET", path=endpoints.API_PATH["info"], params={"id": id_string})
|
||||
break
|
||||
except prawcore.exceptions.ServerError:
|
||||
except (prawcore.exceptions.ServerError, prawcore.exceptions.RequestException):
|
||||
time.sleep(2)
|
||||
return response['data']['children']
|
||||
|
||||
|
|
|
@ -155,7 +155,7 @@ field_actions = {
|
|||
"contest_mode": FieldAction.OVERWRITE,
|
||||
"created": FieldAction.OVERWRITE_IF_NONE,
|
||||
"created_utc": FieldAction.OVERWRITE_IF_NONE,
|
||||
"crosspost_parent": FieldAction.ALLOW,
|
||||
"crosspost_parent": FieldAction.OVERWRITE_NOT_NONE,
|
||||
"crosspost_parent_list": FieldAction.OVERWRITE_NOT_NONE,
|
||||
"discussion_type": FieldAction.OVERWRITE_NOT_NONE,
|
||||
"distinguished": FieldAction.OVERWRITE,
|
||||
|
|
|
@ -12,8 +12,8 @@ NEWLINE_ENCODED = "\n".encode('utf-8')
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
input_file = r"\\MYCLOUDPR4100\Public\RS_2023-07.zst"
|
||||
output_folder = r"\\MYCLOUDPR4100\Public\ingest\download"
|
||||
input_file = r"\\MYCLOUDPR4100\Public\RS_2023-04.zst"
|
||||
output_folder = r"\\MYCLOUDPR4100\Public\ingest\download2"
|
||||
file_type = "comments" if "RC" in input_file else "submissions"
|
||||
|
||||
previous_minute, output_handle, created_utc = None, None, None
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue