Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 06:40:47 -04:00
Code Issues Projects Releases Packages Wiki Activity

Save pushshift token, write to discord

Browse source
This commit is contained in:
Watchful1 2023-09-13 21:48:02 -07:00
parent e119573d1c
commit 34c0f93fe1
2 changed files with 35 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

5
personal/transform/split_by_minutes.py
View file

@ -12,10 +12,11 @@ NEWLINE_ENCODED = "\n".encode('utf-8')
if __name__ == "__main__":
input_file = r"\\MYCLOUDPR4100\Public\RS_2023-04.zst"
output_folder = r"\\MYCLOUDPR4100\Public\ingest\download2"
input_file = r"\\MYCLOUDPR4100\Public\RS_2023-05.zst"
output_folder = r"\\MYCLOUDPR4100\Public\ingest\download"
file_type = "comments" if "RC" in input_file else "submissions"
log.info(f"Input: {input_file} - Output: {output_folder}")
previous_minute, output_handle, created_utc = None, None, None
count_objects, count_minute = 0, 0
for obj in utils.read_obj_zst(input_file):