Reorganize

personal/transform/split_by_subreddit.py

@@ -0,0 +1,34 @@
+import utils
+import discord_logging
+import os
+from datetime import datetime
+
+log = discord_logging.init_logging()
+
+
+if __name__ == "__main__":
+	subreddits = {}
+	field = 'subreddit'
+	object_type = "submissions"
+	folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}"
+	if not os.path.exists(folder):
+		os.makedirs(folder)
+	input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}.zst"
+	input_file_size = os.stat(input_file).st_size
+	total_lines = 0
+	for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file):
+		if comment[field] not in subreddits:
+			subreddits[comment[field]] = {'writer': utils.OutputZst(os.path.join(folder, comment[field] + f"_{object_type}.zst")), 'lines': 0}
+		subreddit = subreddits[comment[field]]
+		subreddit['writer'].write(line)
+		subreddit['writer'].write("\n")
+		subreddit['lines'] += 1
+		total_lines += 1
+		if total_lines % 100000 == 0:
+			log.info(f"{total_lines:,} lines, {(file_bytes_processed / input_file_size) * 100:.0f}%")
+
+	log.info(f"{total_lines:,} lines, 100%")
+
+	for name, subreddit in subreddits.items():
+		log.info(f"r/{name}: {subreddit['lines']:,} lines")
+		subreddit['writer'].close()