Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-12-17 09:14:04 -05:00
Code Issues Projects Releases Packages Wiki Activity

Apply file_filter when combining output files in combine_folder_multiprocess

Browse source
This commit is contained in:
László Kustra 2025-11-12 16:50:50 +01:00
parent a9b75717e5
commit ad8b31724d
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
scripts/combine_folder_multiprocess.py
View file

@ -489,7 +489,9 @@ if __name__ == '__main__':
count_intermediate_files = 0
# build a list of output files to combine
for file in sorted(input_files, key=lambda item: os.path.split(item.output_path)[1]):
if not file.complete:
if re.search(args.file_filter, os.path.basename(file.input_path)) is None:
log.info(f"File {file.input_path} skipped based on file_filter")
elif not file.complete:
if file.error_message is not None:
log.info(f"File {file.input_path} errored {file.error_message}")
else: