714 lines
27 KiB
Python
714 lines
27 KiB
Python
|
"""
|
||
|
|
Porter Stemmer
|
||
|
|
|
||
|
|
This is the Porter stemming algorithm. It follows the algorithm
|
||
|
|
presented in
|
||
|
|
|
||
|
|
Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137.
|
||
|
|
|
||
|
|
with some optional deviations that can be turned on or off with the
|
||
|
|
`mode` argument to the constructor.
|
||
|
|
|
||
|
|
Martin Porter, the algorithm's inventor, maintains a web page about the
|
||
|
|
algorithm at
|
||
|
|
|
||
|
|
http://www.tartarus.org/~martin/PorterStemmer/
|
||
|
|
|
||
|
|
which includes another Python implementation and other implementations
|
||
|
|
in many languages.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import print_function, unicode_literals
|
||
|
|
|
||
|
|
__docformat__ = 'plaintext'
|
||
|
|
|
||
|
|
import re
|
||
|
|
|
||
|
|
from nltk.stem.api import StemmerI
|
||
|
|
from nltk.compat import python_2_unicode_compatible
|
||
|
|
|
||
|
|
|
||
|
|
@python_2_unicode_compatible
|
||
|
|
class PorterStemmer(StemmerI):
|
||
|
|
"""
|
||
|
|
A word stemmer based on the Porter stemming algorithm.
|
||
|
|
|
||
|
|
Porter, M. "An algorithm for suffix stripping."
|
||
|
|
Program 14.3 (1980): 130-137.
|
||
|
|
|
||
|
|
See http://www.tartarus.org/~martin/PorterStemmer/ for the homepage
|
||
|
|
of the algorithm.
|
||
|
|
|
||
|
|
Martin Porter has endorsed several modifications to the Porter
|
||
|
|
algorithm since writing his original paper, and those extensions are
|
||
|
|
included in the implementations on his website. Additionally, others
|
||
|
|
have proposed further improvements to the algorithm, including NLTK
|
||
|
|
contributors. There are thus three modes that can be selected by
|
||
|
|
passing the appropriate constant to the class constructor's `mode`
|
||
|
|
attribute:
|
||
|
|
|
||
|
|
PorterStemmer.ORIGINAL_ALGORITHM
|
||
|
|
- Implementation that is faithful to the original paper.
|
||
|
|
|
||
|
|
Note that Martin Porter has deprecated this version of the
|
||
|
|
algorithm. Martin distributes implementations of the Porter
|
||
|
|
Stemmer in many languages, hosted at:
|
||
|
|
|
||
|
|
http://www.tartarus.org/~martin/PorterStemmer/
|
||
|
|
|
||
|
|
and all of these implementations include his extensions. He
|
||
|
|
strongly recommends against using the original, published
|
||
|
|
version of the algorithm; only use this mode if you clearly
|
||
|
|
understand why you are choosing to do so.
|
||
|
|
|
||
|
|
PorterStemmer.MARTIN_EXTENSIONS
|
||
|
|
- Implementation that only uses the modifications to the
|
||
|
|
algorithm that are included in the implementations on Martin
|
||
|
|
Porter's website. He has declared Porter frozen, so the
|
||
|
|
behaviour of those implementations should never change.
|
||
|
|
|
||
|
|
PorterStemmer.NLTK_EXTENSIONS (default)
|
||
|
|
- Implementation that includes further improvements devised by
|
||
|
|
NLTK contributors or taken from other modified implementations
|
||
|
|
found on the web.
|
||
|
|
|
||
|
|
For the best stemming, you should use the default NLTK_EXTENSIONS
|
||
|
|
version. However, if you need to get the same results as either the
|
||
|
|
original algorithm or one of Martin Porter's hosted versions for
|
||
|
|
compatibility with an existing implementation or dataset, you can use
|
||
|
|
one of the other modes instead.
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Modes the Stemmer can be instantiated in
|
||
|
|
NLTK_EXTENSIONS = 'NLTK_EXTENSIONS'
|
||
|
|
MARTIN_EXTENSIONS = 'MARTIN_EXTENSIONS'
|
||
|
|
ORIGINAL_ALGORITHM = 'ORIGINAL_ALGORITHM'
|
||
|
|
|
||
|
|
def __init__(self, mode=NLTK_EXTENSIONS):
|
||
|
|
if mode not in (
|
||
|
|
self.NLTK_EXTENSIONS,
|
||
|
|
self.MARTIN_EXTENSIONS,
|
||
|
|
self.ORIGINAL_ALGORITHM,
|
||
|
|
):
|
||
|
|
raise ValueError(
|
||
|
|
"Mode must be one of PorterStemmer.NLTK_EXTENSIONS, "
|
||
|
|
"PorterStemmer.MARTIN_EXTENSIONS, or "
|
||
|
|
"PorterStemmer.ORIGINAL_ALGORITHM"
|
||
|
|
)
|
||
|
|
|
||
|
|
self.mode = mode
|
||
|
|
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS:
|
||
|
|
# This is a table of irregular forms. It is quite short,
|
||
|
|
# but still reflects the errors actually drawn to Martin
|
||
|
|
# Porter's attention over a 20 year period!
|
||
|
|
irregular_forms = {
|
||
|
|
"sky": ["sky", "skies"],
|
||
|
|
"die": ["dying"],
|
||
|
|
"lie": ["lying"],
|
||
|
|
"tie": ["tying"],
|
||
|
|
"news": ["news"],
|
||
|
|
"inning": ["innings", "inning"],
|
||
|
|
"outing": ["outings", "outing"],
|
||
|
|
"canning": ["cannings", "canning"],
|
||
|
|
"howe": ["howe"],
|
||
|
|
"proceed": ["proceed"],
|
||
|
|
"exceed": ["exceed"],
|
||
|
|
"succeed": ["succeed"],
|
||
|
|
}
|
||
|
|
|
||
|
|
self.pool = {}
|
||
|
|
for key in irregular_forms:
|
||
|
|
for val in irregular_forms[key]:
|
||
|
|
self.pool[val] = key
|
||
|
|
|
||
|
|
self.vowels = frozenset(['a', 'e', 'i', 'o', 'u'])
|
||
|
|
|
||
|
|
def _is_consonant(self, word, i):
|
||
|
|
"""Returns True if word[i] is a consonant, False otherwise
|
||
|
|
|
||
|
|
A consonant is defined in the paper as follows:
|
||
|
|
|
||
|
|
A consonant in a word is a letter other than A, E, I, O or
|
||
|
|
U, and other than Y preceded by a consonant. (The fact that
|
||
|
|
the term `consonant' is defined to some extent in terms of
|
||
|
|
itself does not make it ambiguous.) So in TOY the consonants
|
||
|
|
are T and Y, and in SYZYGY they are S, Z and G. If a letter
|
||
|
|
is not a consonant it is a vowel.
|
||
|
|
"""
|
||
|
|
if word[i] in self.vowels:
|
||
|
|
return False
|
||
|
|
if word[i] == 'y':
|
||
|
|
if i == 0:
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
return not self._is_consonant(word, i - 1)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def _measure(self, stem):
|
||
|
|
"""Returns the 'measure' of stem, per definition in the paper
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
A consonant will be denoted by c, a vowel by v. A list
|
||
|
|
ccc... of length greater than 0 will be denoted by C, and a
|
||
|
|
list vvv... of length greater than 0 will be denoted by V.
|
||
|
|
Any word, or part of a word, therefore has one of the four
|
||
|
|
forms:
|
||
|
|
|
||
|
|
CVCV ... C
|
||
|
|
CVCV ... V
|
||
|
|
VCVC ... C
|
||
|
|
VCVC ... V
|
||
|
|
|
||
|
|
These may all be represented by the single form
|
||
|
|
|
||
|
|
[C]VCVC ... [V]
|
||
|
|
|
||
|
|
where the square brackets denote arbitrary presence of their
|
||
|
|
contents. Using (VC){m} to denote VC repeated m times, this
|
||
|
|
may again be written as
|
||
|
|
|
||
|
|
[C](VC){m}[V].
|
||
|
|
|
||
|
|
m will be called the \measure\ of any word or word part when
|
||
|
|
represented in this form. The case m = 0 covers the null
|
||
|
|
word. Here are some examples:
|
||
|
|
|
||
|
|
m=0 TR, EE, TREE, Y, BY.
|
||
|
|
m=1 TROUBLE, OATS, TREES, IVY.
|
||
|
|
m=2 TROUBLES, PRIVATE, OATEN, ORRERY.
|
||
|
|
"""
|
||
|
|
cv_sequence = ''
|
||
|
|
|
||
|
|
# Construct a string of 'c's and 'v's representing whether each
|
||
|
|
# character in `stem` is a consonant or a vowel.
|
||
|
|
# e.g. 'falafel' becomes 'cvcvcvc',
|
||
|
|
# 'architecture' becomes 'vcccvcvccvcv'
|
||
|
|
for i in range(len(stem)):
|
||
|
|
if self._is_consonant(stem, i):
|
||
|
|
cv_sequence += 'c'
|
||
|
|
else:
|
||
|
|
cv_sequence += 'v'
|
||
|
|
|
||
|
|
# Count the number of 'vc' occurences, which is equivalent to
|
||
|
|
# the number of 'VC' occurrences in Porter's reduced form in the
|
||
|
|
# docstring above, which is in turn equivalent to `m`
|
||
|
|
return cv_sequence.count('vc')
|
||
|
|
|
||
|
|
def _has_positive_measure(self, stem):
|
||
|
|
return self._measure(stem) > 0
|
||
|
|
|
||
|
|
def _contains_vowel(self, stem):
|
||
|
|
"""Returns True if stem contains a vowel, else False"""
|
||
|
|
for i in range(len(stem)):
|
||
|
|
if not self._is_consonant(stem, i):
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _ends_double_consonant(self, word):
|
||
|
|
"""Implements condition *d from the paper
|
||
|
|
|
||
|
|
Returns True if word ends with a double consonant
|
||
|
|
"""
|
||
|
|
return (
|
||
|
|
len(word) >= 2
|
||
|
|
and word[-1] == word[-2]
|
||
|
|
and self._is_consonant(word, len(word) - 1)
|
||
|
|
)
|
||
|
|
|
||
|
|
def _ends_cvc(self, word):
|
||
|
|
"""Implements condition *o from the paper
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
*o - the stem ends cvc, where the second c is not W, X or Y
|
||
|
|
(e.g. -WIL, -HOP).
|
||
|
|
"""
|
||
|
|
return (
|
||
|
|
len(word) >= 3
|
||
|
|
and self._is_consonant(word, len(word) - 3)
|
||
|
|
and not self._is_consonant(word, len(word) - 2)
|
||
|
|
and self._is_consonant(word, len(word) - 1)
|
||
|
|
and word[-1] not in ('w', 'x', 'y')
|
||
|
|
) or (
|
||
|
|
self.mode == self.NLTK_EXTENSIONS
|
||
|
|
and len(word) == 2
|
||
|
|
and not self._is_consonant(word, 0)
|
||
|
|
and self._is_consonant(word, 1)
|
||
|
|
)
|
||
|
|
|
||
|
|
def _replace_suffix(self, word, suffix, replacement):
|
||
|
|
"""Replaces `suffix` of `word` with `replacement"""
|
||
|
|
assert word.endswith(suffix), "Given word doesn't end with given suffix"
|
||
|
|
if suffix == '':
|
||
|
|
return word + replacement
|
||
|
|
else:
|
||
|
|
return word[: -len(suffix)] + replacement
|
||
|
|
|
||
|
|
def _apply_rule_list(self, word, rules):
|
||
|
|
"""Applies the first applicable suffix-removal rule to the word
|
||
|
|
|
||
|
|
Takes a word and a list of suffix-removal rules represented as
|
||
|
|
3-tuples, with the first element being the suffix to remove,
|
||
|
|
the second element being the string to replace it with, and the
|
||
|
|
final element being the condition for the rule to be applicable,
|
||
|
|
or None if the rule is unconditional.
|
||
|
|
"""
|
||
|
|
for rule in rules:
|
||
|
|
suffix, replacement, condition = rule
|
||
|
|
if suffix == '*d' and self._ends_double_consonant(word):
|
||
|
|
stem = word[:-2]
|
||
|
|
if condition is None or condition(stem):
|
||
|
|
return stem + replacement
|
||
|
|
else:
|
||
|
|
# Don't try any further rules
|
||
|
|
return word
|
||
|
|
if word.endswith(suffix):
|
||
|
|
stem = self._replace_suffix(word, suffix, '')
|
||
|
|
if condition is None or condition(stem):
|
||
|
|
return stem + replacement
|
||
|
|
else:
|
||
|
|
# Don't try any further rules
|
||
|
|
return word
|
||
|
|
|
||
|
|
return word
|
||
|
|
|
||
|
|
def _step1a(self, word):
|
||
|
|
"""Implements Step 1a from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
SSES -> SS caresses -> caress
|
||
|
|
IES -> I ponies -> poni
|
||
|
|
ties -> ti
|
||
|
|
SS -> SS caress -> caress
|
||
|
|
S -> cats -> cat
|
||
|
|
"""
|
||
|
|
# this NLTK-only rule extends the original algorithm, so
|
||
|
|
# that 'flies'->'fli' but 'dies'->'die' etc
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS:
|
||
|
|
if word.endswith('ies') and len(word) == 4:
|
||
|
|
return self._replace_suffix(word, 'ies', 'ie')
|
||
|
|
|
||
|
|
return self._apply_rule_list(
|
||
|
|
word,
|
||
|
|
[
|
||
|
|
('sses', 'ss', None), # SSES -> SS
|
||
|
|
('ies', 'i', None), # IES -> I
|
||
|
|
('ss', 'ss', None), # SS -> SS
|
||
|
|
('s', '', None), # S ->
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
def _step1b(self, word):
|
||
|
|
"""Implements Step 1b from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
(m>0) EED -> EE feed -> feed
|
||
|
|
agreed -> agree
|
||
|
|
(*v*) ED -> plastered -> plaster
|
||
|
|
bled -> bled
|
||
|
|
(*v*) ING -> motoring -> motor
|
||
|
|
sing -> sing
|
||
|
|
|
||
|
|
If the second or third of the rules in Step 1b is successful,
|
||
|
|
the following is done:
|
||
|
|
|
||
|
|
AT -> ATE conflat(ed) -> conflate
|
||
|
|
BL -> BLE troubl(ed) -> trouble
|
||
|
|
IZ -> IZE siz(ed) -> size
|
||
|
|
(*d and not (*L or *S or *Z))
|
||
|
|
-> single letter
|
||
|
|
hopp(ing) -> hop
|
||
|
|
tann(ed) -> tan
|
||
|
|
fall(ing) -> fall
|
||
|
|
hiss(ing) -> hiss
|
||
|
|
fizz(ed) -> fizz
|
||
|
|
(m=1 and *o) -> E fail(ing) -> fail
|
||
|
|
fil(ing) -> file
|
||
|
|
|
||
|
|
The rule to map to a single letter causes the removal of one of
|
||
|
|
the double letter pair. The -E is put back on -AT, -BL and -IZ,
|
||
|
|
so that the suffixes -ATE, -BLE and -IZE can be recognised
|
||
|
|
later. This E may be removed in step 4.
|
||
|
|
"""
|
||
|
|
# this NLTK-only block extends the original algorithm, so that
|
||
|
|
# 'spied'->'spi' but 'died'->'die' etc
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS:
|
||
|
|
if word.endswith('ied'):
|
||
|
|
if len(word) == 4:
|
||
|
|
return self._replace_suffix(word, 'ied', 'ie')
|
||
|
|
else:
|
||
|
|
return self._replace_suffix(word, 'ied', 'i')
|
||
|
|
|
||
|
|
# (m>0) EED -> EE
|
||
|
|
if word.endswith('eed'):
|
||
|
|
stem = self._replace_suffix(word, 'eed', '')
|
||
|
|
if self._measure(stem) > 0:
|
||
|
|
return stem + 'ee'
|
||
|
|
else:
|
||
|
|
return word
|
||
|
|
|
||
|
|
rule_2_or_3_succeeded = False
|
||
|
|
|
||
|
|
for suffix in ['ed', 'ing']:
|
||
|
|
if word.endswith(suffix):
|
||
|
|
intermediate_stem = self._replace_suffix(word, suffix, '')
|
||
|
|
if self._contains_vowel(intermediate_stem):
|
||
|
|
rule_2_or_3_succeeded = True
|
||
|
|
break
|
||
|
|
|
||
|
|
if not rule_2_or_3_succeeded:
|
||
|
|
return word
|
||
|
|
|
||
|
|
return self._apply_rule_list(
|
||
|
|
intermediate_stem,
|
||
|
|
[
|
||
|
|
('at', 'ate', None), # AT -> ATE
|
||
|
|
('bl', 'ble', None), # BL -> BLE
|
||
|
|
('iz', 'ize', None), # IZ -> IZE
|
||
|
|
# (*d and not (*L or *S or *Z))
|
||
|
|
# -> single letter
|
||
|
|
(
|
||
|
|
'*d',
|
||
|
|
intermediate_stem[-1],
|
||
|
|
lambda stem: intermediate_stem[-1] not in ('l', 's', 'z'),
|
||
|
|
),
|
||
|
|
# (m=1 and *o) -> E
|
||
|
|
(
|
||
|
|
'',
|
||
|
|
'e',
|
||
|
|
lambda stem: (self._measure(stem) == 1 and self._ends_cvc(stem)),
|
||
|
|
),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
def _step1c(self, word):
|
||
|
|
"""Implements Step 1c from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
Step 1c
|
||
|
|
|
||
|
|
(*v*) Y -> I happy -> happi
|
||
|
|
sky -> sky
|
||
|
|
"""
|
||
|
|
|
||
|
|
def nltk_condition(stem):
|
||
|
|
"""
|
||
|
|
This has been modified from the original Porter algorithm so
|
||
|
|
that y->i is only done when y is preceded by a consonant,
|
||
|
|
but not if the stem is only a single consonant, i.e.
|
||
|
|
|
||
|
|
(*c and not c) Y -> I
|
||
|
|
|
||
|
|
So 'happy' -> 'happi', but
|
||
|
|
'enjoy' -> 'enjoy' etc
|
||
|
|
|
||
|
|
This is a much better rule. Formerly 'enjoy'->'enjoi' and
|
||
|
|
'enjoyment'->'enjoy'. Step 1c is perhaps done too soon; but
|
||
|
|
with this modification that no longer really matters.
|
||
|
|
|
||
|
|
Also, the removal of the contains_vowel(z) condition means
|
||
|
|
that 'spy', 'fly', 'try' ... stem to 'spi', 'fli', 'tri' and
|
||
|
|
conflate with 'spied', 'tried', 'flies' ...
|
||
|
|
"""
|
||
|
|
return len(stem) > 1 and self._is_consonant(stem, len(stem) - 1)
|
||
|
|
|
||
|
|
def original_condition(stem):
|
||
|
|
return self._contains_vowel(stem)
|
||
|
|
|
||
|
|
return self._apply_rule_list(
|
||
|
|
word,
|
||
|
|
[
|
||
|
|
(
|
||
|
|
'y',
|
||
|
|
'i',
|
||
|
|
nltk_condition
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS
|
||
|
|
else original_condition,
|
||
|
|
)
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
def _step2(self, word):
|
||
|
|
"""Implements Step 2 from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
Step 2
|
||
|
|
|
||
|
|
(m>0) ATIONAL -> ATE relational -> relate
|
||
|
|
(m>0) TIONAL -> TION conditional -> condition
|
||
|
|
rational -> rational
|
||
|
|
(m>0) ENCI -> ENCE valenci -> valence
|
||
|
|
(m>0) ANCI -> ANCE hesitanci -> hesitance
|
||
|
|
(m>0) IZER -> IZE digitizer -> digitize
|
||
|
|
(m>0) ABLI -> ABLE conformabli -> conformable
|
||
|
|
(m>0) ALLI -> AL radicalli -> radical
|
||
|
|
(m>0) ENTLI -> ENT differentli -> different
|
||
|
|
(m>0) ELI -> E vileli - > vile
|
||
|
|
(m>0) OUSLI -> OUS analogousli -> analogous
|
||
|
|
(m>0) IZATION -> IZE vietnamization -> vietnamize
|
||
|
|
(m>0) ATION -> ATE predication -> predicate
|
||
|
|
(m>0) ATOR -> ATE operator -> operate
|
||
|
|
(m>0) ALISM -> AL feudalism -> feudal
|
||
|
|
(m>0) IVENESS -> IVE decisiveness -> decisive
|
||
|
|
(m>0) FULNESS -> FUL hopefulness -> hopeful
|
||
|
|
(m>0) OUSNESS -> OUS callousness -> callous
|
||
|
|
(m>0) ALITI -> AL formaliti -> formal
|
||
|
|
(m>0) IVITI -> IVE sensitiviti -> sensitive
|
||
|
|
(m>0) BILITI -> BLE sensibiliti -> sensible
|
||
|
|
"""
|
||
|
|
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS:
|
||
|
|
# Instead of applying the ALLI -> AL rule after '(a)bli' per
|
||
|
|
# the published algorithm, instead we apply it first, and,
|
||
|
|
# if it succeeds, run the result through step2 again.
|
||
|
|
if word.endswith('alli') and self._has_positive_measure(
|
||
|
|
self._replace_suffix(word, 'alli', '')
|
||
|
|
):
|
||
|
|
return self._step2(self._replace_suffix(word, 'alli', 'al'))
|
||
|
|
|
||
|
|
bli_rule = ('bli', 'ble', self._has_positive_measure)
|
||
|
|
abli_rule = ('abli', 'able', self._has_positive_measure)
|
||
|
|
|
||
|
|
rules = [
|
||
|
|
('ational', 'ate', self._has_positive_measure),
|
||
|
|
('tional', 'tion', self._has_positive_measure),
|
||
|
|
('enci', 'ence', self._has_positive_measure),
|
||
|
|
('anci', 'ance', self._has_positive_measure),
|
||
|
|
('izer', 'ize', self._has_positive_measure),
|
||
|
|
abli_rule if self.mode == self.ORIGINAL_ALGORITHM else bli_rule,
|
||
|
|
('alli', 'al', self._has_positive_measure),
|
||
|
|
('entli', 'ent', self._has_positive_measure),
|
||
|
|
('eli', 'e', self._has_positive_measure),
|
||
|
|
('ousli', 'ous', self._has_positive_measure),
|
||
|
|
('ization', 'ize', self._has_positive_measure),
|
||
|
|
('ation', 'ate', self._has_positive_measure),
|
||
|
|
('ator', 'ate', self._has_positive_measure),
|
||
|
|
('alism', 'al', self._has_positive_measure),
|
||
|
|
('iveness', 'ive', self._has_positive_measure),
|
||
|
|
('fulness', 'ful', self._has_positive_measure),
|
||
|
|
('ousness', 'ous', self._has_positive_measure),
|
||
|
|
('aliti', 'al', self._has_positive_measure),
|
||
|
|
('iviti', 'ive', self._has_positive_measure),
|
||
|
|
('biliti', 'ble', self._has_positive_measure),
|
||
|
|
]
|
||
|
|
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS:
|
||
|
|
rules.append(('fulli', 'ful', self._has_positive_measure))
|
||
|
|
|
||
|
|
# The 'l' of the 'logi' -> 'log' rule is put with the stem,
|
||
|
|
# so that short stems like 'geo' 'theo' etc work like
|
||
|
|
# 'archaeo' 'philo' etc.
|
||
|
|
rules.append(
|
||
|
|
("logi", "log", lambda stem: self._has_positive_measure(word[:-3]))
|
||
|
|
)
|
||
|
|
|
||
|
|
if self.mode == self.MARTIN_EXTENSIONS:
|
||
|
|
rules.append(("logi", "log", self._has_positive_measure))
|
||
|
|
|
||
|
|
return self._apply_rule_list(word, rules)
|
||
|
|
|
||
|
|
def _step3(self, word):
|
||
|
|
"""Implements Step 3 from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
Step 3
|
||
|
|
|
||
|
|
(m>0) ICATE -> IC triplicate -> triplic
|
||
|
|
(m>0) ATIVE -> formative -> form
|
||
|
|
(m>0) ALIZE -> AL formalize -> formal
|
||
|
|
(m>0) ICITI -> IC electriciti -> electric
|
||
|
|
(m>0) ICAL -> IC electrical -> electric
|
||
|
|
(m>0) FUL -> hopeful -> hope
|
||
|
|
(m>0) NESS -> goodness -> good
|
||
|
|
"""
|
||
|
|
return self._apply_rule_list(
|
||
|
|
word,
|
||
|
|
[
|
||
|
|
('icate', 'ic', self._has_positive_measure),
|
||
|
|
('ative', '', self._has_positive_measure),
|
||
|
|
('alize', 'al', self._has_positive_measure),
|
||
|
|
('iciti', 'ic', self._has_positive_measure),
|
||
|
|
('ical', 'ic', self._has_positive_measure),
|
||
|
|
('ful', '', self._has_positive_measure),
|
||
|
|
('ness', '', self._has_positive_measure),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
def _step4(self, word):
|
||
|
|
"""Implements Step 4 from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
Step 4
|
||
|
|
|
||
|
|
(m>1) AL -> revival -> reviv
|
||
|
|
(m>1) ANCE -> allowance -> allow
|
||
|
|
(m>1) ENCE -> inference -> infer
|
||
|
|
(m>1) ER -> airliner -> airlin
|
||
|
|
(m>1) IC -> gyroscopic -> gyroscop
|
||
|
|
(m>1) ABLE -> adjustable -> adjust
|
||
|
|
(m>1) IBLE -> defensible -> defens
|
||
|
|
(m>1) ANT -> irritant -> irrit
|
||
|
|
(m>1) EMENT -> replacement -> replac
|
||
|
|
(m>1) MENT -> adjustment -> adjust
|
||
|
|
(m>1) ENT -> dependent -> depend
|
||
|
|
(m>1 and (*S or *T)) ION -> adoption -> adopt
|
||
|
|
(m>1) OU -> homologou -> homolog
|
||
|
|
(m>1) ISM -> communism -> commun
|
||
|
|
(m>1) ATE -> activate -> activ
|
||
|
|
(m>1) ITI -> angulariti -> angular
|
||
|
|
(m>1) OUS -> homologous -> homolog
|
||
|
|
(m>1) IVE -> effective -> effect
|
||
|
|
(m>1) IZE -> bowdlerize -> bowdler
|
||
|
|
|
||
|
|
The suffixes are now removed. All that remains is a little
|
||
|
|
tidying up.
|
||
|
|
"""
|
||
|
|
measure_gt_1 = lambda stem: self._measure(stem) > 1
|
||
|
|
|
||
|
|
return self._apply_rule_list(
|
||
|
|
word,
|
||
|
|
[
|
||
|
|
('al', '', measure_gt_1),
|
||
|
|
('ance', '', measure_gt_1),
|
||
|
|
('ence', '', measure_gt_1),
|
||
|
|
('er', '', measure_gt_1),
|
||
|
|
('ic', '', measure_gt_1),
|
||
|
|
('able', '', measure_gt_1),
|
||
|
|
('ible', '', measure_gt_1),
|
||
|
|
('ant', '', measure_gt_1),
|
||
|
|
('ement', '', measure_gt_1),
|
||
|
|
('ment', '', measure_gt_1),
|
||
|
|
('ent', '', measure_gt_1),
|
||
|
|
# (m>1 and (*S or *T)) ION ->
|
||
|
|
(
|
||
|
|
'ion',
|
||
|
|
'',
|
||
|
|
lambda stem: self._measure(stem) > 1 and stem[-1] in ('s', 't'),
|
||
|
|
),
|
||
|
|
('ou', '', measure_gt_1),
|
||
|
|
('ism', '', measure_gt_1),
|
||
|
|
('ate', '', measure_gt_1),
|
||
|
|
('iti', '', measure_gt_1),
|
||
|
|
('ous', '', measure_gt_1),
|
||
|
|
('ive', '', measure_gt_1),
|
||
|
|
('ize', '', measure_gt_1),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
def _step5a(self, word):
|
||
|
|
"""Implements Step 5a from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
Step 5a
|
||
|
|
|
||
|
|
(m>1) E -> probate -> probat
|
||
|
|
rate -> rate
|
||
|
|
(m=1 and not *o) E -> cease -> ceas
|
||
|
|
"""
|
||
|
|
# Note that Martin's test vocabulary and reference
|
||
|
|
# implementations are inconsistent in how they handle the case
|
||
|
|
# where two rules both refer to a suffix that matches the word
|
||
|
|
# to be stemmed, but only the condition of the second one is
|
||
|
|
# true.
|
||
|
|
# Earlier in step2b we had the rules:
|
||
|
|
# (m>0) EED -> EE
|
||
|
|
# (*v*) ED ->
|
||
|
|
# but the examples in the paper included "feed"->"feed", even
|
||
|
|
# though (*v*) is true for "fe" and therefore the second rule
|
||
|
|
# alone would map "feed"->"fe".
|
||
|
|
# However, in THIS case, we need to handle the consecutive rules
|
||
|
|
# differently and try both conditions (obviously; the second
|
||
|
|
# rule here would be redundant otherwise). Martin's paper makes
|
||
|
|
# no explicit mention of the inconsistency; you have to infer it
|
||
|
|
# from the examples.
|
||
|
|
# For this reason, we can't use _apply_rule_list here.
|
||
|
|
if word.endswith('e'):
|
||
|
|
stem = self._replace_suffix(word, 'e', '')
|
||
|
|
if self._measure(stem) > 1:
|
||
|
|
return stem
|
||
|
|
if self._measure(stem) == 1 and not self._ends_cvc(stem):
|
||
|
|
return stem
|
||
|
|
return word
|
||
|
|
|
||
|
|
def _step5b(self, word):
|
||
|
|
"""Implements Step 5a from "An algorithm for suffix stripping"
|
||
|
|
|
||
|
|
From the paper:
|
||
|
|
|
||
|
|
Step 5b
|
||
|
|
|
||
|
|
(m > 1 and *d and *L) -> single letter
|
||
|
|
controll -> control
|
||
|
|
roll -> roll
|
||
|
|
"""
|
||
|
|
return self._apply_rule_list(
|
||
|
|
word, [('ll', 'l', lambda stem: self._measure(word[:-1]) > 1)]
|
||
|
|
)
|
||
|
|
|
||
|
|
def stem(self, word):
|
||
|
|
stem = word.lower()
|
||
|
|
|
||
|
|
if self.mode == self.NLTK_EXTENSIONS and word in self.pool:
|
||
|
|
return self.pool[word]
|
||
|
|
|
||
|
|
if self.mode != self.ORIGINAL_ALGORITHM and len(word) <= 2:
|
||
|
|
# With this line, strings of length 1 or 2 don't go through
|
||
|
|
# the stemming process, although no mention is made of this
|
||
|
|
# in the published algorithm.
|
||
|
|
return word
|
||
|
|
|
||
|
|
stem = self._step1a(stem)
|
||
|
|
stem = self._step1b(stem)
|
||
|
|
stem = self._step1c(stem)
|
||
|
|
stem = self._step2(stem)
|
||
|
|
stem = self._step3(stem)
|
||
|
|
stem = self._step4(stem)
|
||
|
|
stem = self._step5a(stem)
|
||
|
|
stem = self._step5b(stem)
|
||
|
|
|
||
|
|
return stem
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return '<PorterStemmer>'
|
||
|
|
|
||
|
|
|
||
|
|
def demo():
|
||
|
|
"""
|
||
|
|
A demonstration of the porter stemmer on a sample from
|
||
|
|
the Penn Treebank corpus.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from nltk.corpus import treebank
|
||
|
|
from nltk import stem
|
||
|
|
|
||
|
|
stemmer = stem.PorterStemmer()
|
||
|
|
|
||
|
|
orig = []
|
||
|
|
stemmed = []
|
||
|
|
for item in treebank.fileids()[:3]:
|
||
|
|
for (word, tag) in treebank.tagged_words(item):
|
||
|
|
orig.append(word)
|
||
|
|
stemmed.append(stemmer.stem(word))
|
||
|
|
|
||
|
|
# Convert the results to a string, and word-wrap them.
|
||
|
|
results = ' '.join(stemmed)
|
||
|
|
results = re.sub(r"(.{,70})\s", r'\1\n', results + ' ').rstrip()
|
||
|
|
|
||
|
|
# Convert the original to a string, and word wrap it.
|
||
|
|
original = ' '.join(orig)
|
||
|
|
original = re.sub(r"(.{,70})\s", r'\1\n', original + ' ').rstrip()
|
||
|
|
|
||
|
|
# Print the results.
|
||
|
|
print('-Original-'.center(70).replace(' ', '*').replace('-', ' '))
|
||
|
|
print(original)
|
||
|
|
print('-Results-'.center(70).replace(' ', '*').replace('-', ' '))
|
||
|
|
print(results)
|
||
|
|
print('*' * 70)
|