Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-04 11:26:41 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add B to blocks files

Browse source
This commit is contained in:
Watchful1 2024-09-08 16:26:42 -07:00
parent 5e1642cea0
commit 51de1f2ac2
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
personal/process_month.py
View file

@ -136,7 +136,11 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
# log.info(f"{file_type} {stage}: {status}")
file_prefix = "RC" if file_type == "comment" else "RS"
if not type_stages["split"]:
split_file = os.path.join(base_folder, "reddit", "blocks", f"{file_prefix}_20{month}.zst")
original_split_file = os.path.join(base_folder, "reddit", "blocks", f"{file_prefix}_20{month}.zst")
split_file = os.path.join(base_folder, "reddit", "blocks", f"{file_prefix}B_20{month}.zst")
if os.path.exists(original_split_file):
os.rename(original_split_file, split_file)
if not os.path.exists(split_file):
log.info(f"{file_type}: File {split_file} doesn't exist, checking for blocks")
split_file = os.path.join(base_folder, "reddit", "blocks", f"{file_prefix}_20{month}.zst_blocks")