Remove filter here

This commit is contained in:
Watchful1 2023-01-24 09:40:41 -08:00
parent cae4434c33
commit 3415c7880e

View file

@ -56,11 +56,10 @@ if __name__ == "__main__":
for line, file_bytes_processed in read_lines_zst(input_file_path):
try:
obj = json.loads(line)
if "social dilemma" in obj['body'].lower():
output_obj = []
for field in fields:
output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode())
writer.writerow(output_obj)
output_obj = []
for field in fields:
output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode())
writer.writerow(output_obj)
created = datetime.utcfromtimestamp(int(obj['created_utc']))
except json.JSONDecodeError as err: