Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-24 15:15:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

More logging

Browse source
This commit is contained in:
Watchful1 2025-03-09 11:34:38 -07:00
parent 15d574e198
commit 3c97f8ffcb
2 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
personal/combine/merge_and_backfill.py
View file

@ -133,7 +133,8 @@ def query_reddit(ids, reddit, object_type):
try: try:
response = reddit.request(method="GET", path=endpoints.API_PATH["info"], params={"id": id_string}) response = reddit.request(method="GET", path=endpoints.API_PATH["info"], params={"id": id_string})
break break
except (prawcore.exceptions.ServerError, prawcore.exceptions.RequestException): except (prawcore.exceptions.ServerError, prawcore.exceptions.RequestException) as err:
log.info(f"No response from reddit api: {err} : {id_string}")
time.sleep(2) time.sleep(2)
return response['data']['children'] return response['data']['children']

3
scripts/find_overlapping_users.py
View file

@ -24,7 +24,7 @@ ignored_users_file = "ignored.txt"
min_comments_per_sub = 1 min_comments_per_sub = 1
output_file_name = "users.txt" output_file_name = "users.txt"
require_first_subreddit = False # if true, print users that occur in the first subreddit and any one of the following ones. Otherwise just find the most overlap between all subs require_first_subreddit = False # if true, print users that occur in the first subreddit and any one of the following ones. Otherwise just find the most overlap between all subs
from_date = datetime.strptime("2005-01-01", "%Y-%m-%d") from_date = datetime.strptime("2024-11-01", "%Y-%m-%d")
to_date = datetime.strptime("2030-12-31", "%Y-%m-%d") to_date = datetime.strptime("2030-12-31", "%Y-%m-%d")
@ -119,6 +119,7 @@ if __name__ == "__main__":
log.info(f"Finding users from the first subreddit that are in any of the other subreddits") log.info(f"Finding users from the first subreddit that are in any of the other subreddits")
log.info(f"Minimum comments per subreddit set to {min_comments_per_sub}") log.info(f"Minimum comments per subreddit set to {min_comments_per_sub}")
log.info(f"Outputting to {output_file_name}") log.info(f"Outputting to {output_file_name}")
log.info(f"From date {from_date.strftime('%Y-%m-%d')} to date {to_date.strftime('%Y-%m-%d')}")
if os.path.exists(ignored_users_file): if os.path.exists(ignored_users_file):
with open(ignored_users_file) as fh: with open(ignored_users_file) as fh: