mirror of
https://github.com/nhammer514/textfiles-politics.git
synced 2024-10-01 01:15:38 -04:00
81 lines
2.3 KiB
Python
81 lines
2.3 KiB
Python
|
import spacy
|
||
|
|
from collections import Counter
|
||
|
|
import os
|
||
|
|
# Uncomment this line if you need the language model.
|
||
|
|
# If you already have it, comment it ou.
|
||
|
|
# Let's try the different spaCy language models for this. We can compare _lg with _md or _sm
|
||
|
|
workingDir = os.getcwd()
|
||
|
|
CollPath = os.path.join(workingDir, '../regexConsp')
|
||
|
|
insideDir = os.listdir(CollPath)
|
||
|
|
print(insideDir)
|
||
|
|
|
||
|
|
nlp = spacy.load("en_core_web_lg")
|
||
|
|
def readTextFiles(filepath):
|
||
|
|
with open(filepath, 'r', encoding='utf8') as f:
|
||
|
|
readFile = f.read()
|
||
|
|
# print(readFile)
|
||
|
|
stringFile = str(readFile)
|
||
|
|
# lengthFile = len(readFile)
|
||
|
|
# print(lengthFile)
|
||
|
|
tokens = nlp(stringFile)
|
||
|
|
# print(tokens)
|
||
|
|
listEntities = entitycollector(tokens)
|
||
|
|
print(listEntities)
|
||
|
|
# cardinal_freq = Counter(listCardinals)
|
||
|
|
# topTen = cardinal_freq.most_common(10)
|
||
|
|
# print(topTen)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def entitycollector(tokens):
|
||
|
|
entities = []
|
||
|
|
for entity in tokens.ents:
|
||
|
|
# if entity.label_ == "CARDINAL":
|
||
|
|
print(entity.text, entity.label_, spacy.explain(entity.label_))
|
||
|
|
entities.append(entity.text)
|
||
|
|
return entities
|
||
|
|
|
||
|
|
|
||
|
|
for file in os.listdir(CollPath):
|
||
|
|
if file.endswith(".xml"):
|
||
|
|
filepath = f"{CollPath}/{file}"
|
||
|
|
print(filepath)
|
||
|
|
readTextFiles(filepath)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# print(listCardinals)
|
||
|
|
# cardinal_freq = Counter(listCardinals)
|
||
|
|
# topTen = cardinal_freq.most_common(10)
|
||
|
|
# print(topTen)
|
||
|
|
|
||
|
|
# grimm = open('grimm.txt', 'r', encoding='utf8')
|
||
|
|
# grimmDoc = grimm.read()
|
||
|
|
# grimmNLP = nlp(grimmDoc)
|
||
|
|
# grimmmSentences = grimmNLP.sents
|
||
|
|
|
||
|
|
# def sentenceLengths(sentences):
|
||
|
|
# lengths = []
|
||
|
|
# for s in sentences:
|
||
|
|
# length = len(s.text)
|
||
|
|
# lengths.append(length)
|
||
|
|
# return sorted(lengths)
|
||
|
|
|
||
|
|
|
||
|
|
# grimmLengths = sentenceLengths(grimmmSentences)
|
||
|
|
# # print(grimmLengths)
|
||
|
|
# maxVal = max(grimmLengths)
|
||
|
|
# minVal = min(grimmLengths)
|
||
|
|
# print('The shortest sentence is ' + str(minVal) + ' characters long.')
|
||
|
|
# print('The longest sentence is ' + str(maxVal) + ' characters long.')
|
||
|
|
|
||
|
|
# for sentence in grimmNLP.sents:
|
||
|
|
# # print(sentence.text)
|
||
|
|
# length = len(sentence.text)
|
||
|
|
# if length == minVal:
|
||
|
|
# print("The shortest sentence is: " + sentence.text)
|
||
|
|
# if len(sentence.text) == maxVal:
|
||
|
|
# print('The longest sentence is: ' + sentence.text + ' :' + str(maxVal) + 'characters')
|