Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-04 19:36:44 -04:00
Code Issues Projects Releases Packages Wiki Activity

Subreddit can change

Browse source
This commit is contained in:
Watchful1 2024-05-18 16:44:09 -07:00
parent 746d0a8330
commit a0e1817d20
2 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/combine/merge.py
View file

@ -103,7 +103,7 @@ field_actions = {
"spam": FieldAction.DELETE,
"stickied": FieldAction.OVERWRITE,
"subreddit": FieldAction.OVERWRITE_NOT_NONE,
"subreddit_id": FieldAction.ALLOW,
"subreddit_id": FieldAction.OVERWRITE_NOT_NONE,
"subreddit_name_prefixed": FieldAction.OVERWRITE_NOT_NONE,
"subreddit_type": FieldAction.DONT_OVERWRITE,
"top_awarded_type": FieldAction.ALLOW_EMPTY,
@ -263,7 +263,7 @@ field_actions = {
"stickied": FieldAction.OVERWRITE,
"subcaption": FieldAction.OVERWRITE,
"subreddit": FieldAction.OVERWRITE_NOT_NONE,
"subreddit_id": FieldAction.ALLOW,
"subreddit_id": FieldAction.OVERWRITE_NOT_NONE,
"subreddit_name_prefixed": FieldAction.OVERWRITE_NOT_NONE,
"subreddit_subscribers": FieldAction.OVERWRITE_IF_NONE,
"subreddit_type": FieldAction.DONT_OVERWRITE,

2
personal/process_month.py
View file

@ -212,6 +212,7 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
# log.info(f"{file_type} {stage}: {status}")
except Exception as err:
queue.put((file_type, "error", str(err)))
discord_logging.flush_discord()
# for stage, status in type_stages.items():
# log.info(f"{file_type} {stage}: {status}")
@ -258,4 +259,5 @@ if __name__ == "__main__":
log.error(f"Error in {file_type}: {status}")
stages[file_type][stage] = status
save_status(status_file, stages, month)
discord_logging.flush_discord()
#log.info(f"workers {workers.ready()} : queue {queue.empty()}")