mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-23 14:50:35 -04:00
Cleanup, this needs to be a date
This commit is contained in:
parent
a0e1817d20
commit
8169e18cdf
1 changed files with 3 additions and 2 deletions
|
@ -176,7 +176,7 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
|
|||
log.info(f"{file_type}: Reading from: {input_folder[0]} : {input_folder[1]}")
|
||||
combined_folder = os.path.join(base_folder, "ingest", "combined")
|
||||
log.info(f"{file_type}: Writing to: {combined_folder}")
|
||||
while start_date <= end_date:
|
||||
while start_date < end_date:
|
||||
build_day(
|
||||
start_date,
|
||||
input_folders,
|
||||
|
@ -192,12 +192,13 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
|
|||
|
||||
if not type_stages["build"]:
|
||||
log.info(f"{file_type}: Starting {file_type} build")
|
||||
start_date = datetime.strptime(month, "%y-%m")
|
||||
|
||||
input_folder = os.path.join(base_folder, "ingest", "combined")
|
||||
log.info(f"{file_type}: Reading from: {input_folder}")
|
||||
log.info(f"{file_type}: Writing to: {base_folder}")
|
||||
build_month.build_month(
|
||||
month,
|
||||
start_date,
|
||||
input_folder,
|
||||
base_folder,
|
||||
file_type,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue