Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 14:50:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

Fix logging

Browse source
This commit is contained in:
Watchful1 2024-11-07 21:38:53 -08:00
parent 2fff2ad373
commit fba4d6de99
3 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

10
personal/process_month.py
View file

@ -17,6 +17,10 @@ import discord_logging
import multiprocessing import multiprocessing
log = discord_logging.init_logging() log = discord_logging.init_logging()
discord_logging.init_discord_logging(
section_name="Watchful12",
log_level=logging.WARNING,
)
multiprocessing_logging.install_mp_handler(log) multiprocessing_logging.install_mp_handler(log)
import utils import utils
@ -244,11 +248,7 @@ if __name__ == "__main__":
start_id, end_id = id_range.split("-") start_id, end_id = id_range.split("-")
ignore_ids.append((utils.base36decode(start_id), utils.base36decode(end_id))) ignore_ids.append((utils.base36decode(start_id), utils.base36decode(end_id)))
discord_logging.init_discord_logging( log.warning(f"Processing {args.month}")
section_name="Watchful12",
log_level=logging.WARNING,
)
log.warning("test")
discord_logging.flush_discord() discord_logging.flush_discord()
status_file = "process.json" status_file = "process.json"

8
scripts/find_overlapping_users.py
View file

@ -12,11 +12,13 @@ import json
# change the subreddits to the list of subreddits, one per line. The case must exactly match, ie, for r/AskReddit, put "AskReddit" # change the subreddits to the list of subreddits, one per line. The case must exactly match, ie, for r/AskReddit, put "AskReddit"
# the files in the folder must match the format from the torrent, subreddit_type.zst, like AskReddit_comments.zst # the files in the folder must match the format from the torrent, subreddit_type.zst, like AskReddit_comments.zst
# the script will look for both comments and submissions files for each subreddit # the script will look for both comments and submissions files for each subreddit
folder = r"C:\Users\user\Downloads" folder = r"\\MYCLOUDPR4100\Public\reddit\subreddits23"
subreddits = [ subreddits = [
"demolitionranch", "phillies",
"EASportsFC",
"eagles",
"politics", "politics",
"pittsburgh", "BucksCountyPA",
] ]
ignored_users = {'[deleted]', 'automoderator'} ignored_users = {'[deleted]', 'automoderator'}
# this is a list of users to ignore when doing the comparison. Most popular bots post in many subreddits and aren't the person you're looking for # this is a list of users to ignore when doing the comparison. Most popular bots post in many subreddits and aren't the person you're looking for

2
scripts/ignored.txt
View file

@ -129,3 +129,5 @@ I_Love_You-BOT
imdad_bot imdad_bot
metric_units metric_units
YoUaReSoHiLaRiOuS YoUaReSoHiLaRiOuS
HIPPAbot
VettedBot