Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 23:00:40 -04:00
Code Issues Projects Releases Packages Wiki Activity

Here too

Browse source
This commit is contained in:
Watchful1 2024-01-07 20:29:55 -08:00
parent d57293d404
commit 199a967888
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
personal/combine/build_month.py
View file

@ -84,7 +84,10 @@ if __name__ == "__main__":
count_objects = 0
count_bytes = 0
minute_iterator = month
end_time = month.replace(month=month.month + 1)
if month.month == 12:
end_time = month.replace(year=month.year + 1, month=1)
else:
end_time = month.replace(month=month.month + 1)
while minute_iterator < end_time:
minute_file_path = os.path.join(args.input, args.type, minute_iterator.strftime('%y-%m-%d'), f"{prefix}_{minute_iterator.strftime('%y-%m-%d_%H-%M')}.zst")
for obj, line, _ in utils.read_obj_zst_meta(minute_file_path):