Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-12-17 09:14:04 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge ad8b31724d into a9b75717e5

Browse source
This commit is contained in:
László Zsolt Kustra 2025-11-12 17:07:30 +01:00 • committed by GitHub
commit 52f6db949e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
scripts/combine_folder_multiprocess.py
View file

@ -489,7 +489,9 @@ if __name__ == '__main__':
count_intermediate_files = 0
# build a list of output files to combine
for file in sorted(input_files, key=lambda item: os.path.split(item.output_path)[1]):
if not file.complete:
if re.search(args.file_filter, os.path.basename(file.input_path)) is None:
log.info(f"File {file.input_path} skipped based on file_filter")
elif not file.complete:
if file.error_message is not None:
log.info(f"File {file.input_path} errored {file.error_message}")
else: