Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-25 23:55:18 -04:00
Code Issues Projects Releases Packages Wiki Activity

rte_mode

Browse source
This commit is contained in:
Watchful1 2023-09-19 17:32:45 -07:00
parent 1dfdea9458
commit 2f3c3c8f88
3 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
personal/combine/build_day.py
View file

@ -188,7 +188,8 @@ def build_day(day_to_process, input_folders, output_folder, object_type, reddit)
discord_logging.flush_discord() discord_logging.flush_discord()
if unmatched_field: if unmatched_field:
log.info(f"Unmatched field, aborting") log.warning(f"Unmatched field, aborting")
discord_logging.flush_discord()
sys.exit(1) sys.exit(1)
minute_iterator += timedelta(minutes=1) minute_iterator += timedelta(minutes=1)

1
personal/combine/merge.py
View file

@ -93,6 +93,7 @@ field_actions = {
"report_reasons": FieldAction.SPECIAL_NO_OVERWRITE, "report_reasons": FieldAction.SPECIAL_NO_OVERWRITE,
"retrieved_on": FieldAction.SPECIAL, "retrieved_on": FieldAction.SPECIAL,
"retrieved_utc": FieldAction.SPECIAL, "retrieved_utc": FieldAction.SPECIAL,
"rte_mode": FieldAction.OVERWRITE_NOT_NONE,
"saved": FieldAction.SPECIAL_NO_OVERWRITE, "saved": FieldAction.SPECIAL_NO_OVERWRITE,
"score": FieldAction.OVERWRITE_NOT_NONE, "score": FieldAction.OVERWRITE_NOT_NONE,
"score_hidden": FieldAction.OVERWRITE, "score_hidden": FieldAction.OVERWRITE,

2
personal/transform/split_by_minutes.py
View file

@ -12,7 +12,7 @@ NEWLINE_ENCODED = "\n".encode('utf-8')
if __name__ == "__main__": if __name__ == "__main__":
input_file = r"\\MYCLOUDPR4100\Public\RC_2023-06.zst" input_file = r"\\MYCLOUDPR4100\Public\RC_2023-08.zst"
output_folder = r"\\MYCLOUDPR4100\Public\ingest\download" output_folder = r"\\MYCLOUDPR4100\Public\ingest\download"
file_type = "comments" if "RC" in input_file else "submissions" file_type = "comments" if "RC" in input_file else "submissions"