Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-03 10:56:40 -04:00
Code Issues Projects Releases Packages Wiki Activity

Move this in here

Browse source
This commit is contained in:
Watchful1 2024-05-19 19:44:22 -07:00
parent 8169e18cdf
commit c5bf0ef593
1 changed files with 8 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

17
personal/combine/build_month.py
View file

@ -28,6 +28,14 @@ reg = re.compile(r"\d\d-\d\d-\d\d_\d\d-\d\d")
def build_month(month, input_folder, output_folder, file_type, compression_level):
if args.type == "comment":
prefix = "RC"
elif args.type == "submission":
prefix = "RS"
else:
log.error(f"Invalid type: {args.type}")
sys.exit(2)
total_objects = 0
total_bytes = 0
minute_iterator = month
@ -99,15 +107,6 @@ if __name__ == "__main__":
log.info(f"Month: {args.month}")
log.info(f"Compression level: {level}")
prefix = None
if args.type == "comments":
prefix = "RC"
elif args.type == "submissions":
prefix = "RS"
else:
log.error(f"Invalid type: {args.type}")
sys.exit(2)
build_month(
month,
args.input,