mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-23 14:50:35 -04:00
Remove filter here
This commit is contained in:
parent
cae4434c33
commit
3415c7880e
1 changed files with 4 additions and 5 deletions
|
@ -56,11 +56,10 @@ if __name__ == "__main__":
|
|||
for line, file_bytes_processed in read_lines_zst(input_file_path):
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
if "social dilemma" in obj['body'].lower():
|
||||
output_obj = []
|
||||
for field in fields:
|
||||
output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode())
|
||||
writer.writerow(output_obj)
|
||||
output_obj = []
|
||||
for field in fields:
|
||||
output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode())
|
||||
writer.writerow(output_obj)
|
||||
|
||||
created = datetime.utcfromtimestamp(int(obj['created_utc']))
|
||||
except json.JSONDecodeError as err:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue