Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 23:00:40 -04:00
Code Issues Projects Releases Packages Wiki Activity

Needs to be an int

Browse source
This commit is contained in:
Watchful1 2024-05-20 11:02:17 -07:00
parent e79bb322cc
commit 22a9fc0504
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/process_month.py
View file

@ -245,6 +245,8 @@ if __name__ == "__main__":
sys.exit(0) sys.exit(0)
month = args.month month = args.month
log.info(f"Processing {month}") log.info(f"Processing {month}")
level = int(args.level)
log.info(f"Compression level: {level}")
multiprocessing.set_start_method('spawn') multiprocessing.set_start_method('spawn')
queue = multiprocessing.Manager().Queue() queue = multiprocessing.Manager().Queue()
@ -252,7 +254,7 @@ if __name__ == "__main__":
with multiprocessing.Pool(processes=2, initializer=init, initargs=(p_lock,)) as pool: with multiprocessing.Pool(processes=2, initializer=init, initargs=(p_lock,)) as pool:
arguments = [] arguments = []
for file_type, type_stages in stages.items(): for file_type, type_stages in stages.items():
arguments.append((queue, args.folder, month, file_type, type_stages, "Watchful12", args.level, ignore_ids)) arguments.append((queue, args.folder, month, file_type, type_stages, "Watchful12", level, ignore_ids))
workers = pool.starmap_async(process, arguments, chunksize=1, error_callback=log.info) workers = pool.starmap_async(process, arguments, chunksize=1, error_callback=log.info)
while not workers.ready() or not queue.empty(): while not workers.ready() or not queue.empty():
file_type, stage, status = queue.get() file_type, stage, status = queue.get()