Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 23:00:40 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add mongo libraries

Browse source
This commit is contained in:
Watchful1 2021-11-20 19:05:10 -08:00
parent 50be918a1c
commit e4e8ad480c
3 changed files with 183 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/combine_folder_multiprocess.py
View file

@ -174,7 +174,7 @@ if __name__ == '__main__':
parser.add_argument("output", help="The output folder to store temporary files in and write the output to")
parser.add_argument("--name", help="What to name the output file", default="pushshift")
parser.add_argument("--field", help="When deciding what lines to keep, use this field for comparisons", default="subreddit")
parser.add_argument("--value", help="When deciding what lines to keep, compare the field to this value. Supports a comma separated list", default="pushshift")
parser.add_argument("--value", help="When deciding what lines to keep, compare the field to this value. Supports a comma separated list. This is case sensitive", default="pushshift")
parser.add_argument("--processes", help="Number of processes to use", default=10, type=int)
parser.add_argument(
"--error_rate", help=