mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-04 11:26:41 -04:00
34 lines
1.2 KiB
Python
34 lines
1.2 KiB
Python
import utils
|
|
import discord_logging
|
|
import os
|
|
from datetime import datetime
|
|
|
|
log = discord_logging.init_logging()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
subreddits = {}
|
|
field = 'subreddit'
|
|
object_type = "submissions"
|
|
folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}"
|
|
if not os.path.exists(folder):
|
|
os.makedirs(folder)
|
|
input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}.zst"
|
|
input_file_size = os.stat(input_file).st_size
|
|
total_lines = 0
|
|
for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file):
|
|
if comment[field] not in subreddits:
|
|
subreddits[comment[field]] = {'writer': utils.OutputZst(os.path.join(folder, comment[field] + f"_{object_type}.zst")), 'lines': 0}
|
|
subreddit = subreddits[comment[field]]
|
|
subreddit['writer'].write(line)
|
|
subreddit['writer'].write("\n")
|
|
subreddit['lines'] += 1
|
|
total_lines += 1
|
|
if total_lines % 100000 == 0:
|
|
log.info(f"{total_lines:,} lines, {(file_bytes_processed / input_file_size) * 100:.0f}%")
|
|
|
|
log.info(f"{total_lines:,} lines, 100%")
|
|
|
|
for name, subreddit in subreddits.items():
|
|
log.info(f"r/{name}: {subreddit['lines']:,} lines")
|
|
subreddit['writer'].close()
|