Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 16:15:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add csv script

Browse source
This commit is contained in:
Watchful1 2022-02-14 16:04:27 -08:00
parent c08f5f212f
commit 461028b401
6 changed files with 169 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

6
personal/split_by_subreddit.py
View file

@ -8,11 +8,11 @@ log = discord_logging.init_logging()
if __name__ == "__main__":
subreddits = {}
object_type = "submissions"
folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\{object_type}"
object_type = "comments"
folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\ratmanreturns265_{object_type}"
if not os.path.exists(folder):
os.makedirs(folder)
input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\relationships_{object_type}.zst"
input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\ratmanreturns265_{object_type}.zst"
input_file_size = os.stat(input_file).st_size
total_lines = 0
for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file):