diff --git a/personal/count_subreddits_multiprocess.py b/personal/count_subreddits_multiprocess.py index 0f37d2f..51996de 100644 --- a/personal/count_subreddits_multiprocess.py +++ b/personal/count_subreddits_multiprocess.py @@ -233,7 +233,7 @@ if __name__ == '__main__': log.info(f"Processing file: {file.input_path}") # start the workers with multiprocessing.Pool(processes=min(args.processes, len(files_to_process))) as pool: - workers = pool.starmap_async(process_file, [(file, queue, args.field) for file in files_to_process], error_callback=log.info) + workers = pool.starmap_async(process_file, [(file, queue, args.field) for file in files_to_process], chunksize=1, error_callback=log.info) while not workers.ready(): # loop until the workers are all done, pulling in status messages as they are sent file_update = queue.get() diff --git a/scripts/combine_folder_multiprocess.py b/scripts/combine_folder_multiprocess.py index e4e49c2..affd154 100644 --- a/scripts/combine_folder_multiprocess.py +++ b/scripts/combine_folder_multiprocess.py @@ -295,7 +295,7 @@ if __name__ == '__main__': log.info(f"Processing file: {file.input_path}") # start the workers with multiprocessing.Pool(processes=min(args.processes, len(files_to_process))) as pool: - workers = pool.starmap_async(process_file, [(file, queue, args.field, value, values, args.case_sensitive) for file in files_to_process], error_callback=log.info) + workers = pool.starmap_async(process_file, [(file, queue, args.field, value, values, args.case_sensitive) for file in files_to_process], chunksize=1, error_callback=log.info) while not workers.ready(): # loop until the workers are all done, pulling in status messages as they are sent file_update = queue.get()