Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-25 15:45:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Clean up

Browse source
This commit is contained in:
Watchful1 2023-08-26 20:18:38 -07:00
parent 4f56c141fd
commit cf4962fd4c
3 changed files with 10 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

5
personal/combine/classes.py
View file

@ -11,6 +11,7 @@ from collections import defaultdict
log = discord_logging.get_logger()
import utils
import merge
NEWLINE_ENCODED = "\n".encode('utf-8')
@ -258,12 +259,12 @@ class ObjectDict:
created_minute = created_utc.replace(second=0, microsecond=0)
if obj['id'] in self.by_id:
existing_obj = self.by_id[obj['id']]
unmatched_field = utils.merge_fields(existing_obj, obj, self.obj_type)
unmatched_field = merge.merge_fields(existing_obj, obj, self.obj_type)
self.counts[created_minute][ingest_type][False] += 1
return unmatched_field
if created_utc < self.min_datetime or created_utc > self.max_datetime:
return False
unmatched_field = utils.parse_fields(obj, self.obj_type)
unmatched_field = merge.parse_fields(obj, self.obj_type)
self.by_id[obj['id']] = obj
self.by_minute[created_minute].add(obj)
self.counts[created_minute][ingest_type][True] += 1