Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 16:15:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

Missed one

Browse source
This commit is contained in:
Watchful1 2025-02-12 19:23:59 -08:00
parent b312694821
commit d292e18204
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
personal/diagnostic/test_files_multiprocess.py
View file

@ -77,7 +77,7 @@ def load_file_list(status_json):
if os.path.exists(status_json): if os.path.exists(status_json):
with open(status_json, 'r') as status_json_file: with open(status_json, 'r') as status_json_file:
output_dict = json.load(status_json_file) output_dict = json.load(status_json_file)
input_files = [] input_files = {}
for simple_file in output_dict["files"]: for simple_file in output_dict["files"]:
input_files[simple_file[0]] = FileConfig(simple_file[0], simple_file[1], simple_file[2], simple_file[3], simple_file[4]) input_files[simple_file[0]] = FileConfig(simple_file[0], simple_file[1], simple_file[2], simple_file[3], simple_file[4])
return input_files, output_dict["type"] return input_files, output_dict["type"]
@ -222,7 +222,7 @@ if __name__ == '__main__':
files_processed = 0 files_processed = 0
files_errored = 0 files_errored = 0
i = 0 i = 0
for file in input_files: for file in input_files.values():
total_lines_processed += file.lines_processed total_lines_processed += file.lines_processed
total_bytes_processed += file.bytes_processed total_bytes_processed += file.bytes_processed
total_lines_errored += file.error_lines total_lines_errored += file.error_lines
@ -254,7 +254,7 @@ if __name__ == '__main__':
count_complete = 0 count_complete = 0
count_incomplete = 0 count_incomplete = 0
# build a list of output files to combine # build a list of output files to combine
for file in input_files: for file in input_files.values():
if not file.complete: if not file.complete:
if file.error_message is not None: if file.error_message is not None:
log.info(f"File {file.input_path} errored {file.error_message}") log.info(f"File {file.input_path} errored {file.error_message}")