From 3415c7880e79ad3a727020afe98c970cb4319a99 Mon Sep 17 00:00:00 2001 From: Watchful1 Date: Tue, 24 Jan 2023 09:40:41 -0800 Subject: [PATCH] Remove filter here --- scripts/to_csv.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/to_csv.py b/scripts/to_csv.py index 9daa4bf..94830ed 100644 --- a/scripts/to_csv.py +++ b/scripts/to_csv.py @@ -56,11 +56,10 @@ if __name__ == "__main__": for line, file_bytes_processed in read_lines_zst(input_file_path): try: obj = json.loads(line) - if "social dilemma" in obj['body'].lower(): - output_obj = [] - for field in fields: - output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode()) - writer.writerow(output_obj) + output_obj = [] + for field in fields: + output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode()) + writer.writerow(output_obj) created = datetime.utcfromtimestamp(int(obj['created_utc'])) except json.JSONDecodeError as err: