Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 08:05:27 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add split blocks by minute

Browse source
This commit is contained in:
Watchful1 2023-11-02 21:04:03 -07:00
parent 4ecf22aaee
commit f35762e203
7 changed files with 318 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

4
personal/diagnostic/test_file.py
View file

@ -8,7 +8,7 @@ log = discord_logging.init_logging()
if __name__ == "__main__":
input_path = r"\\MYCLOUDPR4100\Public\ingest\combined\comments\RC_23-07-10.zst"
input_path = r"\\MYCLOUDPR4100\Public\reddit\comments\RC_2023-09.zst"
input_file_paths = []
if os.path.isdir(input_path):
@ -32,7 +32,7 @@ if __name__ == "__main__":
for obj, line, file_bytes_processed in utils.read_obj_zst_meta(file_path):
new_timestamp = int(obj['created_utc'])
created = datetime.utcfromtimestamp(new_timestamp)
if previous_timestamp is not None and previous_timestamp - (60 * 60) > new_timestamp:
if previous_timestamp is not None and previous_timestamp - (2) > new_timestamp:
log.warning(f"Out of order timestamps {datetime.utcfromtimestamp(previous_timestamp).strftime('%Y-%m-%d %H:%M:%S')} - 4 hours > {created.strftime('%Y-%m-%d %H:%M:%S')}")
previous_timestamp = new_timestamp
file_lines += 1