Подсчет частоты слов в документах с использованием регулярного выражения python

Created a python module which reads in a file, removes the stop words and outputs a python dictionary with the word and its frequency (How many times it occurred in the document).

def run():
filelist = os.listdir(path)
regex = re.compile(r'.*<div class="body">(.*?)</div>.*', re.DOTALL | re.IGNORECASE)
reg1 = re.compile(r'<\/?[ap][^>]*>', re.DOTALL | re.IGNORECASE)
quotereg = re.compile(r'&quot;', re.DOTALL | re.IGNORECASE)
puncreg = re.compile(r'[^\w]', re.DOTALL | re.IGNORECASE)
f = open(stopwordfile, 'r')
stopwords = f.read().lower().split()
totalfreq = {}

filewords = {}
htmlfiles = []
for file in filelist:
    if file[-5:] == '.html':
        htmlfiles.append(file)

for file in htmlfiles:
    f = open(path + file, 'r')
    words = f.read().lower()
    words = regex.findall(words)[0]
    words = quotereg.sub(' ', words)
    words = reg1.sub(' ', words)
    words = puncreg.sub(' ', words)
    words = words.strip().split()

    for w in stopwords:
        while w in words:
            words.remove(w)

     freq = {}
    for w in words:
       if w in freq:
           totalfreq[w] = totalfreq[w] + 1
           freq[w] = freq[w] + 1
       else:
           totalfreq[w] = 1
           freq[w] = 1
           filewords[file] = freq


    print totalfreq

This prints all the 'non-stop' words within that file and the frequency in which they occur in the file: The output looks like:

{{'saturday': 1, 'irish': 1, 'family': 1, 'give': 1, 'year': 2, 'weekend': 1, 'steve': 1, 'guests': 1, 'questions': 1, 'in': 2, 'effort': 1, 'partner': 1, 'extinction': 1, 'dress': 1, 'children': 4, 'utans': 1, '27': 1, 'raise': 1, 'closet': 1, 'haired': 2, 'make': 1, 'humphreys': 1, 'relatives': 1, 'zoo': 5, 'endangered': 1, 'sunday': 1, 'special': 1, 'answer': 1, 'public': 1, 'awareness': 1, 'planned': 1, 'activities': 1, 'rhiona': 1, 'orangutans': 4, 'plans': 1, 'leonie': 1, 'orang': 1, 'yesterday': 2, 'free': 2, 'hand': 1, 'wild': 1, 'independent': 1, 'part': 1, 'preparing': 1, 'revealed': 1, 'day': 1, 'man': 1, 'picture': 1, 'keane': 1, 'animals': 1, '14': 1, 'kevin': 1, '16': 1, '32': 1, 'age': 1, 'sibu': 1, 'dublin': 2, 'keepers': 1, 'face': 1, 'mujur': 1, 'red': 2, 'orangutan': 1, 'species': 1, 'entry': 1, 'efforts': 1, 'shows': 1, '11am': 1, 'influx': 1, '3pm': 1}

{'newest': 1, 'birth': 2, 'orang': 1, 'month': 1, 'steve': 1, 'questions': 1, 'utans': 1, 'children': 4, 'staff': 1, 'limelight': 1, '27': 1, 'based': 1, 'concerned': 1, 'sunday': 1, '3pm': 1, 'finally': 1, '4': 1, 'maeve': 1, 'awareness': 1, 'gave': 1, 'activities': 1, 'giraffe': 1, 'facebook': 1, 'preparing': 1, 'background': 1, 'nurturing': 1, 'day': 1, 'debut': 1, 'rothschild': 1, 'keepers': 1, 'email': 1, 'steps': 1, '11am': 1, 'page': 1, 'picture': 1, 'born': 1, 'result': 1, 'year': 2, 'saturday': 1, 'special': 1, 'closet': 1, 'haired': 2, 'section': 1, 'bennet': 2, 'mum': 3, 'mujur': 1, 'conditions': 1, 'public': 1, 'red': 2, 'shows': 1, 'orangutans': 4, 'free': 2, 'keeper': 1, 'november': 1, 'care': 1, 'sending': 1, 'great': 1, 'origins': 1, '32': 1, 'invited': 1, 'dublin': 2, 'planned': 1, 'orangutan': 1, 'efforts': 1, 'influx': 1, 'named': 1, 'family': 1, 'delighted': 1, 'weather': 1, 'guests': 1, 'extinction': 1, 'post': 1, 'impressed': 1, 'raise': 1, 'revealed': 1, 'remained': 1, 'humphreys': 1, 'confident': 1, 'calf': 3, 'entrance': 1, 'shane': 1, 'part': 1, 'helen': 1, 'attentive': 1, 'effort': 1, 'case': 1, 'made': 2, 'animals': 1, '14': 1, '16': 1, 'ms': 1, 'wild': 1, 'savanna': 1, 'irish': 1, 'give': 1, 'resident': 1, 'suggestions': 1, 'slip': 1, 'in': 2, 'partner': 1, 'dress': 1, 'species': 1, 'kevin': 1, 'rhiona': 1, 'make': 1, 'zoo': 3, 'endangered': 1, 'relatives': 1, 'answer': 1, 'poor': 1, 'independent': 1, 'plans': 1, 'leonie': 1, 'time': 1, 'yesterday': 1, 'hand': 1, 'hickey': 1, 'weekend': 1, 'man': 1, 'sibu': 1, 'age': 1, 'steady': 2, 'face': 1, 'confinement': 1, 'african': 2, 'entry': 1, 'keane': 1, 'clarke': 2, 'left': 1}

But I need both totals to be added together from both files or numerous amounts of files to give a total count of the word eg "zoo" in all files. 1st file zoo=5 2nd file zoo=3 total =8.

I can't seem to work out how I count the words for many files rather than just one at a time.

Any ideas?!

5
задан jenniem001 23 March 2011 в 12:56
поделиться