Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 06:40:47 -04:00
Code Issues Projects Releases Packages Wiki Activity

Typo

Browse source
This commit is contained in:
Watchful1 2024-11-09 23:13:00 -08:00
parent 86fa32fa9e
commit 04f5790d4f
2 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
personal/combine/merge_and_backfill.py
View file

@ -121,7 +121,7 @@ def query_pushshift(ids, bearer, object_type, pushshift_token_function):
discord_logging.flush_discord() discord_logging.flush_discord()
sys.exit(1) sys.exit(1)
if current_attempt > 0: if current_attempt > 0:
log.info(f"Pushshift call succeeded after {i} retries") log.info(f"Pushshift call succeeded after {current_attempt} retries")
return response.json()['data'], bearer return response.json()['data'], bearer

3
personal/process_month.py
View file

@ -10,6 +10,7 @@ import time
import logging.handlers import logging.handlers
import requests import requests
import praw import praw
import traceback
from datetime import datetime, timedelta from datetime import datetime, timedelta
import multiprocessing_logging import multiprocessing_logging
@ -228,6 +229,8 @@ def process(queue, base_folder, month, file_type, type_stages, reddit_username,
# for stage, status in type_stages.items(): # for stage, status in type_stages.items():
# log.info(f"{file_type} {stage}: {status}") # log.info(f"{file_type} {stage}: {status}")
except Exception as err: except Exception as err:
log.warning(f"Error in {type}: {err}")
log.warning(traceback.format_exc())
queue.put((file_type, "error", str(err))) queue.put((file_type, "error", str(err)))
discord_logging.flush_discord() discord_logging.flush_discord()
# for stage, status in type_stages.items(): # for stage, status in type_stages.items():