Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 14:50:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add path

Browse source
This commit is contained in:
Watchful1 2023-11-12 16:25:07 -08:00
parent 4e140e218b
commit 82ce8c88dd
1 changed files with 3 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

21
personal/compression/recompress_file.py
View file

@ -3,7 +3,9 @@ import zstandard
import utils
import discord_logging
import time
import os
import sys
sys.path.append('personal')
log = discord_logging.init_logging()
@ -17,23 +19,6 @@ if __name__ == '__main__':
log.info(f"Input file {args.input}")
log.info(f"Output file {args.output}")
# files = []
# total_size = 0
# for file_name in os.listdir(args.input):
# file_path = os.path.join(args.input, file_name)
# if file_name.endswith(".zst") and os.path.isfile(file_path):
# file_size = os.stat(file_path).st_size
# total_size += file_size
# files.append((file_name, file_size))
# if len(files) % 1000 == 0:
# log.info(f"Loaded {len(files)} files")
# log.info(f"Loaded {len(files)} files of total size {total_size:,}")
#
# level = int(args.level)
# log.info(f"Writing files out to {args.output} at ratio {level}")
# if not os.path.exists(args.output):
# os.makedirs(args.output)
total_objects = 0
total_bytes = 0
for obj, line, _ in utils.read_obj_zst_meta(args.input):