Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 16:15:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

Short script to sort the subreddit counts

Browse source
This commit is contained in:
Watchful1 2023-01-24 20:51:39 -08:00
parent 2358bf555b
commit 52d65e3c8d
2 changed files with 22 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
personal/count_subreddits_multiprocess.py
View file

@ -127,7 +127,6 @@ def read_lines_zst(file_name):
# information back to the parent via a queue
def process_file(file, queue, field):
output_file = None
log.debug(f"Starting file: {file.input_path} : {file.file_size:,}")
try:
for line, file_bytes_processed in read_lines_zst(file.input_path):
try:
@ -149,7 +148,6 @@ def process_file(file, queue, field):
file.complete = True
file.bytes_processed = file.file_size
log.debug(f"Finished file: {file.input_path} : {file.file_size:,}")
except Exception as err:
file.error_message = str(err)
queue.put(file)