Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-24 15:15:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

Cleanup, this needs to be a date

Browse source
This commit is contained in:
Watchful1 2024-05-19 19:40:38 -07:00
parent a0e1817d20
commit 8169e18cdf
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
personal/process_month.py
View file

@ -176,7 +176,7 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
log.info(f"{file_type}: Reading from: {input_folder[0]} : {input_folder[1]}") log.info(f"{file_type}: Reading from: {input_folder[0]} : {input_folder[1]}")
combined_folder = os.path.join(base_folder, "ingest", "combined") combined_folder = os.path.join(base_folder, "ingest", "combined")
log.info(f"{file_type}: Writing to: {combined_folder}") log.info(f"{file_type}: Writing to: {combined_folder}")
while start_date <= end_date: while start_date < end_date:
build_day( build_day(
start_date, start_date,
input_folders, input_folders,
@ -192,12 +192,13 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
if not type_stages["build"]: if not type_stages["build"]:
log.info(f"{file_type}: Starting {file_type} build") log.info(f"{file_type}: Starting {file_type} build")
start_date = datetime.strptime(month, "%y-%m")
input_folder = os.path.join(base_folder, "ingest", "combined") input_folder = os.path.join(base_folder, "ingest", "combined")
log.info(f"{file_type}: Reading from: {input_folder}") log.info(f"{file_type}: Reading from: {input_folder}")
log.info(f"{file_type}: Writing to: {base_folder}") log.info(f"{file_type}: Writing to: {base_folder}")
build_month.build_month( build_month.build_month(
month, start_date,
input_folder, input_folder,
base_folder, base_folder,
file_type, file_type,