Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 16:15:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

Cleaner logging here

Browse source
This commit is contained in:
Watchful1 2025-02-08 14:51:19 -08:00
parent 85c44168e0
commit 693e4c5b3d
1 changed files with 36 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

5
personal/diagnostic/test_files_multiprocess.py
View file

@ -199,6 +199,7 @@ if __name__ == '__main__':
log.info(f"Processed {files_processed} of {len(input_files)} files with {(total_bytes_processed / (2**30)):.2f} of {(total_bytes / (2**30)):.2f} gigabytes") log.info(f"Processed {files_processed} of {len(input_files)} files with {(total_bytes_processed / (2**30)):.2f} of {(total_bytes / (2**30)):.2f} gigabytes")
start_time = time.time() start_time = time.time()
last_log_time = None
if len(files_to_process): if len(files_to_process):
progress_queue = Queue(40) progress_queue = Queue(40)
progress_queue.put([start_time, total_lines_processed, total_bytes_processed]) progress_queue.put([start_time, total_lines_processed, total_bytes_processed])
@ -213,6 +214,8 @@ if __name__ == '__main__':
file_update = queue.get() file_update = queue.get()
if file_update.error_message is not None: if file_update.error_message is not None:
log.warning(f"File failed {file_update.input_path}: {file_update.error_message}") log.warning(f"File failed {file_update.input_path}: {file_update.error_message}")
current_time = time.time()
if last_log_time is None or (current_time - last_log_time) > 5 or queue.empty():
# I'm going to assume that the list of files is short enough that it's no # I'm going to assume that the list of files is short enough that it's no
# big deal to just iterate each time since that saves a bunch of work # big deal to just iterate each time since that saves a bunch of work
total_lines_processed = 0 total_lines_processed = 0
@ -233,7 +236,6 @@ if __name__ == '__main__':
i += 1 i += 1
if file_update.complete or file_update.error_message is not None: if file_update.complete or file_update.error_message is not None:
save_file_list(input_files, status_json, script_type) save_file_list(input_files, status_json, script_type)
current_time = time.time()
progress_queue.put([current_time, total_lines_processed, total_bytes_processed]) progress_queue.put([current_time, total_lines_processed, total_bytes_processed])
first_time, first_lines, first_bytes = progress_queue.peek() first_time, first_lines, first_bytes = progress_queue.peek()
@ -249,6 +251,7 @@ if __name__ == '__main__':
f"{(total_bytes_processed / (2**30)):.2f} gb at {(bytes_per_second / (2**20)):,.0f} mb/s, {(total_bytes_processed / total_bytes) * 100:.0f}% : " f"{(total_bytes_processed / (2**30)):.2f} gb at {(bytes_per_second / (2**20)):,.0f} mb/s, {(total_bytes_processed / total_bytes) * 100:.0f}% : "
f"{files_processed}({files_errored})/{len(input_files)} files : " f"{files_processed}({files_errored})/{len(input_files)} files : "
f"{(str(days_left) + 'd ' if days_left > 0 else '')}{hours_left - (days_left * 24)}:{minutes_left - (hours_left * 60):02}:{seconds_left - (minutes_left * 60):02} remaining") f"{(str(days_left) + 'd ' if days_left > 0 else '')}{hours_left - (days_left * 24)}:{minutes_left - (hours_left * 60):02}:{seconds_left - (minutes_left * 60):02} remaining")
last_log_time = time.time()
log.info(f"{total_lines_processed:,}, {total_lines_errored} errored : {(total_bytes_processed / (2**30)):.2f} gb, {(total_bytes_processed / total_bytes) * 100:.0f}% : {files_processed}/{len(input_files)}") log.info(f"{total_lines_processed:,}, {total_lines_errored} errored : {(total_bytes_processed / (2**30)):.2f} gb, {(total_bytes_processed / total_bytes) * 100:.0f}% : {files_processed}/{len(input_files)}")