Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-27 00:25:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

Update frame sizes

Browse source
This commit is contained in:
Watchful1 2022-07-17 15:56:06 -07:00
parent 3fa63048e3
commit c4d652d0cf
3 changed files with 40 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

11
personal/test_file.py
View file

@ -8,7 +8,7 @@ log = discord_logging.init_logging()
if __name__ == "__main__":
input_path = r"\\MYCLOUDPR4100\Public\reddit\requests\jeanyp"
input_path = r"\\MYCLOUDPR4100\Public\reddit\requests\wallstreetbets_comments.zst"
input_file_paths = []
if os.path.isdir(input_path):
@ -27,11 +27,16 @@ if __name__ == "__main__":
file_lines = 0
file_bytes_processed = 0
created = None
previous_timestamp = None
inserts = []
for obj, line, file_bytes_processed in utils.read_obj_zst_meta(file_path):
created = datetime.utcfromtimestamp(int(obj['created_utc']))
new_timestamp = int(obj['created_utc'])
created = datetime.utcfromtimestamp(new_timestamp)
if previous_timestamp is not None and previous_timestamp - (60 * 60 * 4) > new_timestamp:
log.warning(f"Out of order timestamps {datetime.utcfromtimestamp(previous_timestamp).strftime('%Y-%m-%d %H:%M:%S')} - 4 hours > {created.strftime('%Y-%m-%d %H:%M:%S')}")
previous_timestamp = new_timestamp
file_lines += 1
if file_lines % 100000 == 0:
if file_lines % 10000 == 0:
log.info(f"{files_processed}/{len(input_file_paths)}: {file_name} : {created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {(file_bytes_processed / file_size) * 100:.0f}%")
log.info(f"{files_processed}/{len(input_file_paths)}: {file_name} : {created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : 100%")