Skip to content

Add comments to make the working of the functions clear. #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pratap-is-here wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add comments to make the working of the functions clear. #1

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
10e9a77
Add comments to make the working of the functions clear.
pratap-is-here May 12, 2019
616c340
Change comments to doc string
pratap-is-here Jul 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 151 additions & 0 deletions stemmer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
class PorterStemmer:
def isCons(self, letter):
'''
This function returns true if a letter is a consonant otherwise false.
'''

if letter == 'a' or letter == 'e' or letter == 'i' or letter == 'o'
or letter == 'u':
return False
else:
return True

def isConsonant(self, word, i):
'''
This function returns true only if the letter at i th position
in the argument 'word' is a consonant. But if the letter is 'y' and the letter at i-1 th position
is also a consonant, then it returns false.
'''

letter = word[i]
if self.isCons(letter):
if letter == 'y' and isCons(word[i-1]):
Expand All @@ -17,24 +27,41 @@ def isConsonant(self, word, i):
return False

def isVowel(self, word, i):
'''
This function returns true if the letter at i th position in the argument 'word'
is a vowel.
'''

return not(isConsonant(word, i))

# *S
def endsWith(self, stem, letter):
'''
This function returns true if the word 'stem' ends with 'letter'.
'''

if stem.endswith(letter):
return True
else:
return False

# *v*
def containsVowel(self, stem):
'''
This function returns true if the word 'stem' contains a vowel.
'''

for i in stem:
if not self.isCons(i):
return True
return False

# *d
def doubleCons(self, stem):
'''
This function returns true if the word 'stem' ends with 2 consonants.
'''

if len(stem) >= 2:
if self.isConsonant(stem, -1) and self.isConsonant(stem, -2):
return True
Expand All @@ -44,6 +71,18 @@ def doubleCons(self, stem):
return False

def getForm(self, word):
'''
This function takes a word as an input, and checks for vowel and consonant sequences in that word.
vowel sequence is denoted by V and consonant sequences by C
For example, the word 'balloon' can be divived into following sequences:
'b' : C
'a' : V
'll': C
'oo': V
'n' : C
So form = [C,V,C,V,C] and formstr = CVCVC
'''

form = []
formStr = ''
for i in range(len(word)):
Expand All @@ -66,12 +105,22 @@ def getForm(self, word):
return formStr

def getM(self, word):
'''
This function returns value of M which is equal to number of 'VC' in formstr
So in the word 'balloon', we have 2 'VC'.
'''

form = self.getForm(word)
m = form.count('VC')
return m

# *o
def cvc(self, word):
'''
This function returns true if the last 3 letters of the word are of the following pattern: consonant,vowel,consonant
but if the last word is either 'w','x' or 'y', it returns false.
'''

if len(word) >= 3:
f = -3
s = -2
Expand All @@ -89,12 +138,24 @@ def cvc(self, word):
return False

def replace(self, orig, rem, rep):
'''
This function checks if string 'orig' ends with 'rem' and
replaces 'rem' by the substring 'rep'. The resulting string 'replaced'
is returned.
'''

result = orig.rfind(rem)
base = orig[:result]
replaced = base + rep
return replaced

def replaceM0(self, orig, rem, rep):
'''
This function is same as the function replace(), except that it checks the value of M for the
base string. If it is greater than 0 , it replaces 'rem' by 'rep', otherwise it returns the
original string.
'''

result = orig.rfind(rem)
base = orig[:result]
if self.getM(base) > 0:
Expand All @@ -104,6 +165,11 @@ def replaceM0(self, orig, rem, rep):
return orig

def replaceM1(self, orig, rem, rep):
'''
This function is same as replaceM0(), except that it replaces 'rem' by 'rep', only when M>1 for
the base string.
'''

result = orig.rfind(rem)
base = orig[:result]
if self.getM(base) > 1:
Expand All @@ -113,6 +179,20 @@ def replaceM1(self, orig, rem, rep):
return orig

def step1a(self, word):
'''
In a given word, this function replaces 'sses' by 'ss', 'ies' by 'i',
'ss' by 'ss' and 's' by ''.

step1a gets rid of plurals. e.g.

caresses -> caress
ponies -> poni
ties -> ti
caress -> caress
cats -> cat

'''

if word.endswith('sses'):
word = self.replace(word, 'sses', 'ss')
elif word.endswith('ies'):
Expand All @@ -126,6 +206,30 @@ def step1a(self, word):
return word

def step1b(self, word):
'''
This function checks if a word ends with 'eed','ed' or 'ing' and replces these substrings by
'ee','' and ''. If after the replacements in case of 'ed' and 'ing', the resulting word
-> ends with 'at','bl' or 'iz' : add 'e' to the end of the word
-> ends with 2 consonants and its last letter isn't 'l','s' or 'z': remove last letter of the word
-> has 1 as value of M and the cvc(word) returns true : add 'e' to the end of the word


step1b gets rid of -eed -ed or -ing. e.g.

feed -> feed
agreed -> agree
disabled -> disable

matting -> mat
mating -> mate
meeting -> meet
milling -> mill
messing -> mess

meetings -> meet

'''

flag = False
if word.endswith('eed'):
result = word.rfind('eed')
Expand Down Expand Up @@ -161,6 +265,12 @@ def step1b(self, word):
return word

def step1c(self, word):
'''
In words ending with 'y', this function replaces 'y' by 'i'.

step1c turns terminal y to i when there is another vowel in the stem."""
'''

if word.endswith('y'):
result = word.rfind('y')
base = word[:result]
Expand All @@ -170,6 +280,15 @@ def step1c(self, word):
return word

def step2(self, word):
'''
This function checks the value of M, and replaces the suffixes accordingly

step2 maps double suffices to single ones.
so -ization ( = -ize plus -ation) maps to -ize etc. note that the
string before the suffix must give m() > 0.

'''

if word.endswith('ational'):
word = self.replaceM0(word, 'ational', 'ate')
elif word.endswith('tional'):
Expand Down Expand Up @@ -213,6 +332,12 @@ def step2(self, word):
return word

def step3(self, word):
'''
This function checks the value of M, and replaces the suffixes accordingly.

step3 dels with -ic-, -full, -ness etc. similar strategy to step2.
'''

if word.endswith('icate'):
word = self.replaceM0(word, 'icate', 'ic')
elif word.endswith('ative'):
Expand All @@ -228,6 +353,12 @@ def step3(self, word):
return word

def step4(self, word):
'''
This function checks the value of M, and replaces the suffixes accordingly.

step4 takes off -ant, -ence etc., in context <c>vcvc<v>{meaning, M >1 for the word}.
'''

if word.endswith('al'):
word = self.replaceM1(word, 'al', '')
elif word.endswith('ance'):
Expand Down Expand Up @@ -274,6 +405,15 @@ def step4(self, word):
return word

def step5a(self, word):
'''
This function checks if the word ends with 'e'. If it does, it checks the value of
M for the base word. If M>1, OR, If M = 1 and cvc(base) is false, it simply removes 'e'
ending.

step5 removes a final -e if m() > 1.

'''

if word.endswith('e'):
base = word[:-1]
if self.getM(base) > 1:
Expand All @@ -283,12 +423,23 @@ def step5a(self, word):
return word

def step5b(self, word):
'''
This function checks if the value of M for the word is greater than 1 and it ends with 2 consonants
and it ends with 'l', it removes 'l'

step5b changes -ll to -l if m() > 1
'''

if self.getM(word) > 1 and self.doubleCons(word)
and self.endsWith(word, 'l'):
word = word[:-1]
return word

def stem(self, word):
'''
This functions puts together all the steps in porter stemming. :)
'''

word = self.step1a(word)
word = self.step1b(word)
word = self.step1c(word)
Expand Down