Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 14:50:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

Wait here too

Browse source
This commit is contained in:
Watchful1 2025-07-22 20:58:08 -07:00
parent d7751783a9
commit 15d88a138f
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
personal/diagnostic/count_subreddits_multiprocess.py
View file

@ -238,7 +238,7 @@ if __name__ == '__main__':
# start the workers
with multiprocessing.Pool(processes=min(args.processes, len(files_to_process))) as pool:
workers = pool.starmap_async(process_file, [(file, queue, args.field) for file in files_to_process], chunksize=1, error_callback=log.info)
while not workers.ready():
while not workers.ready() or not queue.empty():
# loop until the workers are all done, pulling in status messages as they are sent
file_update = queue.get()
if file_update.error_message is not None:

2
personal/diagnostic/test_file.py
View file

@ -8,7 +8,7 @@ log = discord_logging.init_logging()
if __name__ == "__main__":
input_path = r"\\MYCLOUDPR4100\Public\reddit\submissions\RS_2025-05.zst"
input_path = r"\\MYCLOUDPR4100\Public\reddit\comments\RC_2025-06.zst"
input_file_paths = []
if os.path.isdir(input_path):