Reorganize

personal/transform/copy_listed_files.py

@@ -0,0 +1,50 @@
+import shutil
+import os
+import logging.handlers
+import re
+
+log = logging.getLogger("bot")
+log.setLevel(logging.DEBUG)
+log.addHandler(logging.StreamHandler())
+
+if __name__ == '__main__':
+	input_folder = r"\\MYCLOUDPR4100\Public\pushshift_output"
+	output_folder = r"\\MYCLOUDPR4100\Public\request"
+	subs = ['PoliticalDiscussion', 'worldnews', 'science']
+	overwrite = False
+
+	lower_subs = set()
+	for sub in subs:
+		lower_subs.add(sub.lower())
+
+	matched_subs = set()
+	total_size = 0
+	for file_name in os.listdir(input_folder):
+		file_path = os.path.join(input_folder, file_name)
+		if file_name.endswith(".zst") and os.path.isfile(file_path):
+			match = re.match(r"(\w+)(?:_(?:comments|submissions).zst)", file_name)
+			if match:
+				sub_cased = match.group(1)
+				if sub_cased.lower() in lower_subs:
+					matched_subs.add(sub_cased)
+					file_size = os.stat(file_path).st_size
+					total_size += file_size
+					log.info(f"Copying {file_name} : {(file_size / (2**20)):,.0f} mb : {(total_size / (2**20)):,.0f} mb")
+					output_path = os.path.join(output_folder, file_name)
+					if overwrite or not os.path.exists(output_path):
+						shutil.copy(file_path, output_path)
+
+	log.info(f"Copied {len(matched_subs)}/{len(subs)} subs of total size {(total_size / (2**20)):,.0f} mb")
+	if len(matched_subs) != len(lower_subs):
+		lower_matched_subs = [sub.lower() for sub in matched_subs]
+		for sub in lower_subs:
+			if sub not in lower_matched_subs:
+				log.info(f"Missing r/{sub}")
+
+	sorted_case_subs = sorted(matched_subs)
+	bldr = ['torrenttools create -a "https://academictorrents.com/announce.php" -c "Comments and submissions from r/']
+	bldr.append(', r/'.join(sorted_case_subs))
+	bldr.append(' through the end of 2022"  --include ".*')
+	bldr.append('.*zst" --include ".*'.join(sorted_case_subs))
+	bldr.append('.*zst" -o username.torrent reddit')
+	log.info(''.join(bldr))

personal/transform/split_by_subreddit.py

@@ -0,0 +1,34 @@
+import utils
+import discord_logging
+import os
+from datetime import datetime
+
+log = discord_logging.init_logging()
+
+
+if __name__ == "__main__":
+	subreddits = {}
+	field = 'subreddit'
+	object_type = "submissions"
+	folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}"
+	if not os.path.exists(folder):
+		os.makedirs(folder)
+	input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}.zst"
+	input_file_size = os.stat(input_file).st_size
+	total_lines = 0
+	for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file):
+		if comment[field] not in subreddits:
+			subreddits[comment[field]] = {'writer': utils.OutputZst(os.path.join(folder, comment[field] + f"_{object_type}.zst")), 'lines': 0}
+		subreddit = subreddits[comment[field]]
+		subreddit['writer'].write(line)
+		subreddit['writer'].write("\n")
+		subreddit['lines'] += 1
+		total_lines += 1
+		if total_lines % 100000 == 0:
+			log.info(f"{total_lines:,} lines, {(file_bytes_processed / input_file_size) * 100:.0f}%")
+
+	log.info(f"{total_lines:,} lines, 100%")
+
+	for name, subreddit in subreddits.items():
+		log.info(f"r/{name}: {subreddit['lines']:,} lines")
+		subreddit['writer'].close()