Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-25 15:45:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Remove filter here

Browse source
This commit is contained in:
Watchful1 2023-01-24 09:40:41 -08:00
parent cae4434c33
commit 3415c7880e
1 changed files with 4 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

1
scripts/to_csv.py
View file

@ -56,7 +56,6 @@ if __name__ == "__main__":
for line, file_bytes_processed in read_lines_zst(input_file_path): for line, file_bytes_processed in read_lines_zst(input_file_path):
try: try:
obj = json.loads(line) obj = json.loads(line)
if "social dilemma" in obj['body'].lower():
output_obj = [] output_obj = []
for field in fields: for field in fields:
output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode()) output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode())