improved main.py, can wrap all names in document now

pythonCode/main.py

@@ -2,6 +2,8 @@ import spacy
 from collections import Counter
 import re as regex
 import os
+from saxonche import PySaxonProcessor
+
 
 #### Loads all of the necessary variables and functions.
 nlp = spacy.load("en_core_web_lg")
@@ -12,7 +14,6 @@ outputPath = os.path.join(workingDir, 'output/')
 insideDir = os.listdir(CollPath)
 print(insideDir)
 
-
 # Copies files in case they do not exist
 def copyTextFiles(file):
     content = []
@@ -20,7 +21,7 @@ def copyTextFiles(file):
     with open(CollPath + "/" + file, 'r', encoding='utf8') as inFile:
         for line in inFile:
             content.append(line)
-        print("copying " + file)
+        print(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~ copying " + file + " ~~~~~~~~~~~~~~~~~~~~~~~~~~~ ")
         inFile.close()
     # With the contents copied, a loop will go through the array and write it all in a new file in output folder.
     with open(outputPath + "/" + file, 'w', encoding='utf8') as f:
@@ -29,37 +30,50 @@ def copyTextFiles(file):
 
 # Function runs through the tokens of given file. Entities are stored in array, then returned. Called by regexFile().
 def entitycollector(tokens):
-    entities = []
-    for entity in tokens.ents:
-        if entity.label_ == "PERSON":
-            entities.append(entity.text)
+    with open('output.txt', 'w') as f:
+        entities = {}
+        for ent in sorted(tokens.ents):
+        # if entity.label_ == "NORP" or entity.label_ == "LOC" or entity.label_=="GPE":
+        # ebb: The line helps experiment with different spaCy named entity classifiers, in combination if you like:
+        # When using it, remember to indent the next lines for the for loop.
+            # print(entity.text, entity.label_, spacy.explain(entity.label_))
+            entityInfo = [ent.text, ent.label_, spacy.explain(ent.label_)]
+            stringify = str(entityInfo)
+            f.write(stringify)
+            f.write('\n')
+        # PRINT TO FILE
+            # entities.append(entity.text)
+            entities[ent.text] = ent.label_
         return entities
 
 # Function runs regex through given file.
 def regexFile(file):
-    #  First, it reads file given. Supposedly, the newly created file in output folder.
-    with open(outputPath + "/" + file, 'r', encoding='utf8') as inFile:
-        rawText = str(inFile.read())
-        # Regex finds all elements in a file and deletes them. Then Regex finds anything that is not a letter, and
-        # deletes. It is stored in a variable that is supposedly clean from anything extra.
-        cleanedText = regex.sub('[^A-z]+', ' ', regex.sub('<.+?>', ' ', rawText))
-        # token stuff
+    fileDir = os.path.join(outputPath, file)
+    with PySaxonProcessor(license=False) as proc:
+        # grabs the original xml file and stores it in a variable for later.
+        xml = open(fileDir, encoding='utf-8').read()
+        xp = proc.new_xpath_processor()
+        node = proc.parse_xml(xml_text=xml)
+        xp.set_context(xdm_item=node)
+        xpath = xp.evaluate('//p ! normalize-space() => string-join()')
+        string = xpath.__str__()
+        cleanedText = regex.sub('[^A-z]+', ' ', string)
         tokens = nlp(cleanedText)
+        wrappedText = xml
         listEntities = entitycollector(tokens)
-        # If the listEntity array has content in it, it will go through the list to see if the content is located
-        # anywhere in the original, raw text.
+        #print(listEntities)
         if listEntities:
-            for entity in listEntities:
-                wrappedText = regex.sub(str(entity), '<person>' + entity + '</person>',rawText)
-                # Saves newly wrapped elements and then writes it into the copied file.
-                with open(outputPath + "/" + file, 'w', encoding='utf8') as f:
-                    f.write(wrappedText)
-                    print("WRAPPING " + entity)
-                    f.close()
-        else:
-            print("No names... Probably did not detect any?")
+            for entity in listEntities.keys():
+                #print(entity, listEntities[entity])
+                if listEntities[entity] == "PERSON":
+                    key_template = "<ent type = 'person'>" + entity + "</ent>"
+                    wrappedText = wrappedText.replace(entity, key_template)
+                    # Saves newly wrapped elements and then writes it into the copied file.
+                    with open(fileDir, 'w', encoding='utf8') as f:
+                        f.write(wrappedText)
+                        print("WRAPPING " + entity)
 
-# Goes through all of the original conspiracy texts
 for file in insideDir:
     copyTextFiles(file)
-    regexFile(file)
+    regexFile(file)
+    print("File checking finished.")