Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 16:15:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add csv script

Browse source
This commit is contained in:
Watchful1 2022-02-14 16:04:27 -08:00
parent c08f5f212f
commit 461028b401
6 changed files with 169 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

36
personal/test_file.py
View file

@ -8,18 +8,30 @@ log = discord_logging.init_logging()
if __name__ == "__main__":
file_path = r"\\MYCLOUDPR4100\Public\reddit\submissions\RS_2011-01.zst"
file_size = os.stat(file_path).st_size
input_path = r"\\MYCLOUDPR4100\Public\reddit\requests\jeanyp"
file_lines = 0
file_bytes_processed = 0
created = None
inserts = []
for obj, line, file_bytes_processed in utils.read_obj_zst_meta(file_path):
created = datetime.utcfromtimestamp(int(obj['created_utc']))
file_lines += 1
if file_lines % 100000 == 0:
log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {(file_bytes_processed / file_size) * 100:.0f}%")
input_file_paths = []
if os.path.isdir(input_path):
for subdir, dirs, files in os.walk(input_path):
files.sort()
for file_name in files:
if file_name.endswith(".zst"):
input_file_paths.append(os.path.join(subdir, file_name))
else:
input_file_paths.append(input_path)
log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : 100%")
files_processed = 0
for file_path in input_file_paths:
file_name = os.path.basename(file_path)
file_size = os.stat(file_path).st_size
file_lines = 0
file_bytes_processed = 0
created = None
inserts = []
for obj, line, file_bytes_processed in utils.read_obj_zst_meta(file_path):
created = datetime.utcfromtimestamp(int(obj['created_utc']))
file_lines += 1
if file_lines % 100000 == 0:
log.info(f"{files_processed}/{len(input_file_paths)}: {file_name} : {created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {(file_bytes_processed / file_size) * 100:.0f}%")
log.info(f"{files_processed}/{len(input_file_paths)}: {file_name} : {created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : 100%")