Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-03 19:06:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

We can't support minutes for the end here

Browse source
This commit is contained in:
Watchful1 2023-08-27 20:10:21 -07:00
parent 9ad5d9c06f
commit 9376b45f44
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/combine/build_day.py
View file

@ -140,7 +140,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Combine the ingest and rescan files, clean and do pushshift lookups as needed")
parser.add_argument("--type", help="The object type, either comments or submissions", required=True)
parser.add_argument("--start_date", help="The start of the date range to process, format YY-MM-DD_HH-MM", required=True)
parser.add_argument("--end_date", help="The end of the date range to process, format YY-MM-DD_HH-MM. If not provided, the script processes to the end of the day")
parser.add_argument("--end_date", help="The end of the date range to process, format YY-MM-DD. If not provided, the script processes to the end of the day")
parser.add_argument('--input', help='Input folder', required=True)
parser.add_argument('--output', help='Output folder', required=True)
parser.add_argument('--pushshift', help='The pushshift token', required=True)
@ -158,7 +158,7 @@ if __name__ == "__main__":
start_date = datetime.strptime(args.start_date, '%y-%m-%d_%H-%M')
end_date = end_of_day(start_date)
if args.end_date is not None:
end_date = datetime.strptime(args.end_date, '%y-%m-%d_%H-%M')
end_date = datetime.strptime(args.end_date, '%y-%m-%d')
for input_folder, ingest_type in input_folders:
log.info(f"Input folder: {input_folder}")