Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-26 08:05:27 -04:00
Code Issues Projects Releases Packages Wiki Activity

Another couple merges

Browse source
This commit is contained in:
Watchful1 2023-09-18 12:04:24 -07:00
parent a8d56ad156
commit 4dab643fff
1 changed files with 12 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

21
personal/combine/merge.py
View file

@ -72,7 +72,7 @@ field_actions = {
"id": FieldAction.ALLOW, "id": FieldAction.ALLOW,
"ignore_reports": FieldAction.DELETE, "ignore_reports": FieldAction.DELETE,
"is_submitter": FieldAction.DONT_OVERWRITE, "is_submitter": FieldAction.DONT_OVERWRITE,
"likes": FieldAction.ALLOW_EMPTY, "likes": FieldAction.OVERWRITE_NOT_NONE,
"link_id": FieldAction.ALLOW, "link_id": FieldAction.ALLOW,
"locked": FieldAction.OVERWRITE, "locked": FieldAction.OVERWRITE,
"media_metadata": FieldAction.OVERWRITE, "media_metadata": FieldAction.OVERWRITE,
@ -106,7 +106,7 @@ field_actions = {
"top_awarded_type": FieldAction.ALLOW_EMPTY, "top_awarded_type": FieldAction.ALLOW_EMPTY,
"total_awards_received": FieldAction.OVERWRITE_NOT_NONE, "total_awards_received": FieldAction.OVERWRITE_NOT_NONE,
"treatment_tags": FieldAction.OVERWRITE_NOT_NONE, "treatment_tags": FieldAction.OVERWRITE_NOT_NONE,
"unrepliable_reason": FieldAction.ALLOW_EMPTY, "unrepliable_reason": FieldAction.OVERWRITE_NOT_NONE,
"ups": FieldAction.OVERWRITE_NOT_NONE, "ups": FieldAction.OVERWRITE_NOT_NONE,
"user_reports": FieldAction.SPECIAL_NO_OVERWRITE, "user_reports": FieldAction.SPECIAL_NO_OVERWRITE,
"user_reports_dismissed": FieldAction.SPECIAL_NO_OVERWRITE, "user_reports_dismissed": FieldAction.SPECIAL_NO_OVERWRITE,
@ -182,9 +182,9 @@ field_actions = {
"impression_id": FieldAction.ALLOW_EMPTY, "impression_id": FieldAction.ALLOW_EMPTY,
"impression_id_str": FieldAction.ALLOW_EMPTY, "impression_id_str": FieldAction.ALLOW_EMPTY,
"is_blank": FieldAction.ALLOW_EMPTY, "is_blank": FieldAction.ALLOW_EMPTY,
"is_created_from_ads_ui": FieldAction.ALLOW, "is_created_from_ads_ui": FieldAction.OVERWRITE_NOT_NONE,
"is_crosspostable": FieldAction.OVERWRITE, "is_crosspostable": FieldAction.OVERWRITE,
"is_gallery": FieldAction.ALLOW, "is_gallery": FieldAction.OVERWRITE_NOT_NONE,
"is_meta": FieldAction.OVERWRITE, "is_meta": FieldAction.OVERWRITE,
"is_original_content": FieldAction.OVERWRITE, "is_original_content": FieldAction.OVERWRITE,
"is_reddit_media_domain": FieldAction.OVERWRITE, "is_reddit_media_domain": FieldAction.OVERWRITE,
@ -192,7 +192,7 @@ field_actions = {
"is_self": FieldAction.DONT_OVERWRITE, "is_self": FieldAction.DONT_OVERWRITE,
"is_survey_ad": FieldAction.ALLOW_EMPTY, "is_survey_ad": FieldAction.ALLOW_EMPTY,
"is_video": FieldAction.OVERWRITE, "is_video": FieldAction.OVERWRITE,
"likes": FieldAction.ALLOW_EMPTY, "likes": FieldAction.OVERWRITE_NOT_NONE,
"link_flair_background_color": FieldAction.OVERWRITE_NOT_NONE, "link_flair_background_color": FieldAction.OVERWRITE_NOT_NONE,
"link_flair_css_class": FieldAction.OVERWRITE_NOT_NONE, "link_flair_css_class": FieldAction.OVERWRITE_NOT_NONE,
"link_flair_richtext": FieldAction.OVERWRITE_NOT_NONE, "link_flair_richtext": FieldAction.OVERWRITE_NOT_NONE,
@ -225,6 +225,7 @@ field_actions = {
"poll_data": FieldAction.OVERWRITE_NOT_NONE, "poll_data": FieldAction.OVERWRITE_NOT_NONE,
"post_hint": FieldAction.OVERWRITE, "post_hint": FieldAction.OVERWRITE,
"preview": FieldAction.OVERWRITE_NOT_NONE, "preview": FieldAction.OVERWRITE_NOT_NONE,
"previous_selftext": FieldAction.ALLOW,
"priority_id": FieldAction.ALLOW_EMPTY, "priority_id": FieldAction.ALLOW_EMPTY,
"product_ids": FieldAction.ALLOW_EMPTY, "product_ids": FieldAction.ALLOW_EMPTY,
"promo_layout": FieldAction.OVERWRITE, "promo_layout": FieldAction.OVERWRITE,
@ -241,6 +242,7 @@ field_actions = {
"report_reasons": FieldAction.SPECIAL_NO_OVERWRITE, "report_reasons": FieldAction.SPECIAL_NO_OVERWRITE,
"retrieved_on": FieldAction.SPECIAL, "retrieved_on": FieldAction.SPECIAL,
"retrieved_utc": FieldAction.SPECIAL, "retrieved_utc": FieldAction.SPECIAL,
"rte_mode": FieldAction.OVERWRITE_NOT_NONE,
"saved": FieldAction.SPECIAL_NO_OVERWRITE, "saved": FieldAction.SPECIAL_NO_OVERWRITE,
"score": FieldAction.OVERWRITE_NOT_NONE, "score": FieldAction.OVERWRITE_NOT_NONE,
"secure_media": FieldAction.OVERWRITE_NOT_NONE, "secure_media": FieldAction.OVERWRITE_NOT_NONE,
@ -271,6 +273,7 @@ field_actions = {
"total_awards_received": FieldAction.OVERWRITE_NOT_NONE, "total_awards_received": FieldAction.OVERWRITE_NOT_NONE,
"treatment_tags": FieldAction.OVERWRITE_NOT_NONE, "treatment_tags": FieldAction.OVERWRITE_NOT_NONE,
"tournament_data": FieldAction.OVERWRITE_NOT_NONE, "tournament_data": FieldAction.OVERWRITE_NOT_NONE,
"unrepliable_reason": FieldAction.OVERWRITE_NOT_NONE,
"updated_on": FieldAction.SPECIAL, "updated_on": FieldAction.SPECIAL,
"updated_utc": FieldAction.SPECIAL, "updated_utc": FieldAction.SPECIAL,
"ups": FieldAction.OVERWRITE_NOT_NONE, "ups": FieldAction.OVERWRITE_NOT_NONE,
@ -318,10 +321,10 @@ def merge_fields(existing_obj, new_obj, obj_type):
original_value = existing_obj.get(key) original_value = existing_obj.get(key)
if new_value != original_value: if new_value != original_value:
if isinstance(new_value, str) and unencode_regex.search(new_value): # if isinstance(new_value, str) and unencode_regex.search(new_value):
new_value_no_encode = unencode_regex.sub(replace, new_value) # new_value_no_encode = unencode_regex.sub(replace, new_value)
if new_value_no_encode == original_value: # if new_value_no_encode == original_value:
continue # continue
if action == FieldAction.OVERWRITE: if action == FieldAction.OVERWRITE:
existing_obj[key] = new_value existing_obj[key] = new_value
elif action == FieldAction.OVERWRITE_NOT_NONE: elif action == FieldAction.OVERWRITE_NOT_NONE: