mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-23 14:50:35 -04:00
Reorganize
This commit is contained in:
parent
3700b21b81
commit
4f1d70d34a
19 changed files with 0 additions and 105 deletions
50
personal/transform/copy_listed_files.py
Normal file
50
personal/transform/copy_listed_files.py
Normal file
|
@ -0,0 +1,50 @@
|
|||
import shutil
|
||||
import os
|
||||
import logging.handlers
|
||||
import re
|
||||
|
||||
log = logging.getLogger("bot")
|
||||
log.setLevel(logging.DEBUG)
|
||||
log.addHandler(logging.StreamHandler())
|
||||
|
||||
if __name__ == '__main__':
|
||||
input_folder = r"\\MYCLOUDPR4100\Public\pushshift_output"
|
||||
output_folder = r"\\MYCLOUDPR4100\Public\request"
|
||||
subs = ['PoliticalDiscussion', 'worldnews', 'science']
|
||||
overwrite = False
|
||||
|
||||
lower_subs = set()
|
||||
for sub in subs:
|
||||
lower_subs.add(sub.lower())
|
||||
|
||||
matched_subs = set()
|
||||
total_size = 0
|
||||
for file_name in os.listdir(input_folder):
|
||||
file_path = os.path.join(input_folder, file_name)
|
||||
if file_name.endswith(".zst") and os.path.isfile(file_path):
|
||||
match = re.match(r"(\w+)(?:_(?:comments|submissions).zst)", file_name)
|
||||
if match:
|
||||
sub_cased = match.group(1)
|
||||
if sub_cased.lower() in lower_subs:
|
||||
matched_subs.add(sub_cased)
|
||||
file_size = os.stat(file_path).st_size
|
||||
total_size += file_size
|
||||
log.info(f"Copying {file_name} : {(file_size / (2**20)):,.0f} mb : {(total_size / (2**20)):,.0f} mb")
|
||||
output_path = os.path.join(output_folder, file_name)
|
||||
if overwrite or not os.path.exists(output_path):
|
||||
shutil.copy(file_path, output_path)
|
||||
|
||||
log.info(f"Copied {len(matched_subs)}/{len(subs)} subs of total size {(total_size / (2**20)):,.0f} mb")
|
||||
if len(matched_subs) != len(lower_subs):
|
||||
lower_matched_subs = [sub.lower() for sub in matched_subs]
|
||||
for sub in lower_subs:
|
||||
if sub not in lower_matched_subs:
|
||||
log.info(f"Missing r/{sub}")
|
||||
|
||||
sorted_case_subs = sorted(matched_subs)
|
||||
bldr = ['torrenttools create -a "https://academictorrents.com/announce.php" -c "Comments and submissions from r/']
|
||||
bldr.append(', r/'.join(sorted_case_subs))
|
||||
bldr.append(' through the end of 2022" --include ".*')
|
||||
bldr.append('.*zst" --include ".*'.join(sorted_case_subs))
|
||||
bldr.append('.*zst" -o username.torrent reddit')
|
||||
log.info(''.join(bldr))
|
34
personal/transform/split_by_subreddit.py
Normal file
34
personal/transform/split_by_subreddit.py
Normal file
|
@ -0,0 +1,34 @@
|
|||
import utils
|
||||
import discord_logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
log = discord_logging.init_logging()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
subreddits = {}
|
||||
field = 'subreddit'
|
||||
object_type = "submissions"
|
||||
folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}"
|
||||
if not os.path.exists(folder):
|
||||
os.makedirs(folder)
|
||||
input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\multisub_{object_type}.zst"
|
||||
input_file_size = os.stat(input_file).st_size
|
||||
total_lines = 0
|
||||
for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file):
|
||||
if comment[field] not in subreddits:
|
||||
subreddits[comment[field]] = {'writer': utils.OutputZst(os.path.join(folder, comment[field] + f"_{object_type}.zst")), 'lines': 0}
|
||||
subreddit = subreddits[comment[field]]
|
||||
subreddit['writer'].write(line)
|
||||
subreddit['writer'].write("\n")
|
||||
subreddit['lines'] += 1
|
||||
total_lines += 1
|
||||
if total_lines % 100000 == 0:
|
||||
log.info(f"{total_lines:,} lines, {(file_bytes_processed / input_file_size) * 100:.0f}%")
|
||||
|
||||
log.info(f"{total_lines:,} lines, 100%")
|
||||
|
||||
for name, subreddit in subreddits.items():
|
||||
log.info(f"r/{name}: {subreddit['lines']:,} lines")
|
||||
subreddit['writer'].close()
|
Loading…
Add table
Add a link
Reference in a new issue