Python nltk English Detection

Posted 2020-07-26 龟窝

http://blog.alejandronolla.com/2013/05/15/detecting-text-language-with-python-and-nltk/

 

>>> from nltk import wordpunct_tokenize

>>> wordpunct_tokenize("That‘s thirty minutes away. I‘ll be there in ten.")

[‘That‘, "‘", ‘s‘, ‘thirty‘, ‘minutes‘, ‘away‘, ‘.‘, ‘I‘, "‘", ‘ll‘, ‘be‘, ‘there‘, ‘in‘, ‘ten‘, ‘.‘]

 

>>> from nltk.corpus import stopwords

>>> stopwords.fileids()

[‘danish‘, ‘dutch‘, ‘english‘, ‘finnish‘, ‘french‘, ‘german‘, ‘hungarian‘, ‘italian‘, ‘norwegian‘, ‘portuguese‘, ‘russian‘, ‘spanish‘, ‘swedish‘, ‘turkish‘]

>>>

>>> stopwords.words(‘english‘)[0:10]

[‘i‘, ‘me‘, ‘my‘, ‘myself‘, ‘we‘, ‘our‘, ‘ours‘, ‘ourselves‘, ‘you‘, ‘your‘]

 

>>> languages_ratios = {}

>>>

>>> tokens = wordpunct_tokenize(text)

>>> words = [word.lower() for word in tokens]

>>> for language in stopwords.fileids():

... stopwords_set = set(stopwords.words(language))

... words_set = set(words)

... common_elements = words_set.intersection(stopwords_set)

...

... languages_ratios[language] = len(common_elements)

# language "score"

>>>