Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-24 15:15:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

Wait here too

Browse source
This commit is contained in:
Watchful1 2025-07-22 20:58:08 -07:00
parent d7751783a9
commit 15d88a138f
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
personal/diagnostic/count_subreddits_multiprocess.py
View file

@ -238,7 +238,7 @@ if __name__ == '__main__':
# start the workers
with multiprocessing.Pool(processes=min(args.processes, len(files_to_process))) as pool:
workers = pool.starmap_async(process_file, [(file, queue, args.field) for file in files_to_process], chunksize=1, error_callback=log.info)
while not workers.ready():
while not workers.ready() or not queue.empty():
# loop until the workers are all done, pulling in status messages as they are sent
file_update = queue.get()
if file_update.error_message is not None: