Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-04 03:16:40 -04:00
Code Issues Projects Releases Packages Wiki Activity

Support empty filter

Browse source
This commit is contained in:
Watchful1 2023-08-21 21:45:58 -07:00
parent f7146593a0
commit 78c1814a60
1 changed files with 15 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

29
scripts/filter_file.py
View file

@ -7,7 +7,7 @@ from datetime import datetime
import logging.handlers
# put the path to the input file
input_file = r"\\MYCLOUDPR4100\Public\reddit\submissions\RS_2023-02.zst"
input_file = r"\\MYCLOUDPR4100\Public\reddit\subreddits\redditdev_submissions.zst"
# put the name or path to the output file. The file extension from below will be added automatically
output_file = r"\\MYCLOUDPR4100\Public\output"
# the format to output in, pick from the following options
@ -233,19 +233,20 @@ if __name__ == "__main__":
if created > to_date:
continue
field_value = obj[field].lower()
matched = False
for value in values:
if exact_match:
if value == field_value:
matched = True
break
else:
if value in field_value:
matched = True
break
if not matched:
continue
if field is not None:
field_value = obj[field].lower()
matched = False
for value in values:
if exact_match:
if value == field_value:
matched = True
break
else:
if value in field_value:
matched = True
break
if not matched:
continue
matched_lines += 1
if output_format == "zst":