import spacy
from collections import Counter
import re as regex
import os
from saxonche import PySaxonProcessor
#### Loads all of the necessary variables and functions.
nlp = spacy.load("en_core_web_lg")
#########################################################################################
# ebb: After reading the NLP output, we know spaCy is making some mistakes.
# So, here let's try adding an EntityRuler to customize spaCy's classification. We need
# to configure this BEFORE we send the tokens off to nlp() for processing.
##########################################################################################
# Create the EntityRuler and set it so the ner comes after, so OUR rules take precedence
# Sources:
# W. J. B. Mattingly: https://ner.pythonhumanities.com/02_01_spaCy_Entity_Ruler.html
# spaCy documentation on NER Entity Ruler: https://spacy.io/usage/rule-based-matching#entityruler
config = {"spans_key": None, "annotate_ents": True, "overwrite": True, "validate": True}
ruler = nlp.add_pipe("span_ruler", before="ner", config=config)
# 2023-04-07: ebb: NOTE: before="ner" setting seems to allow the spaCy NER rules to prevail over these patterns where
# there is a conflict.
# after="ner" means that the spaCy ner is TOTALLY OVERWRITTEN and invalidated by our patterns.
# Notes: Mattingly has this: ruler = nlp.add_pipe("entity_ruler", after="ner", config={"validate": True})
# But this only works when spaCy doesn't recognize a word / phrase as a named entity of any kind.
# If it recognizes a named entity but tags it wrong, we correct it with the span_ruler, not the entity_ruler
patterns = [
{"label": "NULL", "pattern": [{"TEXT" : {"REGEX": "^-\w+?"}}]},
{"label": "NULL", "pattern": [{"TEXT" : {"REGEX": "^\w$"}}]},
{"label": "GPE", "pattern": [{"TEXT" : {"REGEX": "Babylon(ia)?"}}]},
{"label": "NULL", "pattern": "di"},
{"label": "ORG", "pattern": "Falangist"},
{"label": "NORP", "pattern": "Dropa"},
{"label": "GPE", "pattern": "Nazareth"},
{"label": "NULL", "pattern": "Bab"},
]
ruler.add_patterns(patterns)
workingDir = os.getcwd()
CollPath = os.path.join(workingDir, '../regexConsp')
outputPath = os.path.join(workingDir, 'cultureTaggedOutput/')
# Everything in original conspiracy directory.
insideDir = os.listdir(CollPath)
print(insideDir)
# Copies files in case they do not exist
def copyTextFiles(file):
content = []
# Reads the contents of file, and saves each line of file into the content array.
with open(CollPath + "/" + file, 'r', encoding='utf8') as inFile:
for line in inFile:
content.append(line)
print(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~ copying " + file + " ~~~~~~~~~~~~~~~~~~~~~~~~~~~ ")
inFile.close()
# With the contents copied, a loop will go through the array and write it all in a new file in output folder.
with open(outputPath + "/" + file, 'w', encoding='utf8') as f:
for line in content:
f.write(str(line))
# Function runs through the tokens of given file. Entities are stored in array, then returned. Called by regexFile().
def entitycollector(tokens):
# creates a new file that includes all of the found entities.
with open('output.txt', 'w') as f:
entities = {}
# goes through each entity in the token list.
for ent in sorted(tokens.ents):
entityInfo = [ent.text, ent.label_, spacy.explain(ent.label_)]
stringify = str(entityInfo)
f.write(stringify)
f.write('\n')
entities[ent.text] = ent.label_
# return all entities with its label and text.
return entities
# Function runs regex through given file.
def regexFile(file):
fileDir = os.path.join(outputPath, file)
with PySaxonProcessor(license=False) as proc:
# grabs the original xml file and stores it in a variable for later. this some xquery bs
xml = open(fileDir, encoding='utf-8').read()
xp = proc.new_xpath_processor()
node = proc.parse_xml(xml_text=xml)
xp.set_context(xdm_item=node)
# xquery goes through original text, and stores it all in a single string.
xpath = xp.evaluate('//p ! normalize-space() => string-join()')
string = str(xpath)
# regex goes through the text and deletes anything that is not a letter or space.
cleanedText = regex.sub(r'[^A-z ]+', ' ', string)
# gets the tokens of the clean text.
tokens = nlp(cleanedText)
wrappedText = xml
# grabs all the entities in file and stores it in a list/array.
dictEntities = entitycollector(tokens)
# if anything exists in the list, the following code will run.
if dictEntities:
# it will check through each entity in the list and see its entity type. it is looking for "PERSON" tokens
# in this instance, which includes of nouns and names.
for entity, value in dictEntities.items():
if value == "LOC" or value=="ORG" or value=="GPE" or value=="NORP":
# key_template variable is the elements we wrap around found instances.
key_template = "" + entity + ""
# loops through wrappedText until all entities are wrapped.
wrappedText = wrappedText.replace(entity, key_template)
# Saves newly wrapped elements and then writes it into new file.
with open(fileDir, 'w', encoding='utf8') as f:
f.write(wrappedText)
print("WRAPPING " + entity)
# This part of the code does not run. It is a WIP.
## It tries to find weird or invalid elements/tags and fix them.
def checkTags(file):
content = []
fileDir = os.path.join(outputPath, file)
with open(fileDir, 'r', encoding='utf8') as inFile:
for line in inFile:
content.append(line)
# With the contents copied, a loop will go through the array and write it all in a new file in output folder.
with open(fileDir, 'w', encoding='utf8') as f:
for line in content:
# match = regex.search(r"([^<>]*?)]+?>([^<>]+?)([^<>]*?)", line)
# if match:
print("broken line found, fixing...")
# newLine = regex.sub(r"([^<>]*?)]+?>([^<>]+?)([^<>]*?)", r"\1\2\3",line)
newLine = regex.sub(r"(.*?)(.+?)(.*?)", r"\1\2\3", line)
newLine = regex.sub(r"(.*?)(.+?)(.*?)", r"\1\2\3", newLine)
newLine = regex.sub(r"(.*?)(.+?)(.*?)", r"\1\2\3", newLine)
newLine = regex.sub(r"((cia)(l>)", r"\1\2\3", newLine)
newLine = regex.sub(r"(<)(di)(v>)", r"\1\2\3", newLine)
newLine = regex.sub(r"((ORG)('>)", r"\1\2\3", newLine)
#
# cial>
# <div>
print(line + "\n INTO.")
line = newLine
print(line)
for file in insideDir:
copyTextFiles(file)
regexFile(file)
#checkTags(file)
print("File checking finished.")