Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 14:50:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

Reorganize

Browse source
This commit is contained in:
Watchful1 2023-08-26 16:52:58 -07:00
parent 3700b21b81
commit 4f1d70d34a
19 changed files with 0 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

30
personal/compression/extract_file.py Normal file
View file

@ -0,0 +1,30 @@
import utils
import discord_logging
import os
import sys
from datetime import datetime
log = discord_logging.init_logging()
if __name__ == "__main__":
input_file_path = r"\\MYCLOUDPR4100\Public\reddit_final\curiousdrive_submissions.zst"
output_file_path = r"\\MYCLOUDPR4100\Public\reddit_final\curiousdrive_submissions.txt"
file_size = os.stat(input_file_path).st_size
file_lines = 0
file_bytes_processed = 0
created = None
inserts = []
output_file = open(output_file_path, 'w')
for obj, line, file_bytes_processed in utils.read_obj_zst_meta(input_file_path):
created = datetime.utcfromtimestamp(int(obj['created_utc']))
file_lines += 1
output_file.write(line)
output_file.write("\n")
if file_lines % 100000 == 0:
log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {(file_bytes_processed / file_size) * 100:.0f}%")
log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : 100%")
output_file.close()