Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-04 03:16:40 -04:00
Code Issues Projects Releases Packages Wiki Activity

Full redesign of multiprocess

Browse source
This commit is contained in:
Watchful1 2023-03-08 17:48:59 -08:00
parent 31ad7179dc
commit f7286b7572
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/combine_folder_multiprocess.py
View file

@ -284,7 +284,7 @@ if __name__ == '__main__':
parser.add_argument("--value", help="When deciding what lines to keep, compare the field to this value. Supports a comma separated list. This is case sensitive", default="pushshift")
parser.add_argument("--value_list", help="A file of newline separated values to use. Overrides the value param if it is set", default=None)
parser.add_argument("--processes", help="Number of processes to use", default=10, type=int)
parser.add_argument("--file_filter", help="Regex filenames have to match to be processed", default="^RC_|RS_")
parser.add_argument("--file_filter", help="Regex filenames have to match to be processed", default="^RC_|^RS_")
parser.add_argument("--split_intermediate", help="Split the intermediate files by the first letter of the matched field, use if the filter will result in a large number of separate files", action="store_true")
parser.add_argument(
"--error_rate", help=