python实现指定目录下批量文件的单词计数：串行版本

       直接上代码。

 

       练习目标：

             1.  使用 Python 面向对象的方法封装逻辑和表达 ；

             2.  使用异常处理和日志API ；

             3.  使用文件目录读写API ； 

             4.  使用 list, map, tuple 三种数据结构 ；

             5.  lambda 、正则使用及其它。

       下一篇将实现并发版本。  

       

#-------------------------------------------------------------------------------
# Name:        wordstat_serial.py
# Purpose:     statistic words in java files of given directory by serial
#
# Author:      qin.shuq
#
# Created:     08/10/2014
# Copyright:   (c) qin.shuq 2014
# Licence:     <your licence>
#-------------------------------------------------------------------------------

import re
import os
import time
import logging

LOG_LEVELS = {
    'DEBUG': logging.DEBUG, 'INFO': logging.INFO,
    'WARN': logging.WARNING, 'ERROR': logging.ERROR,
    'CRITICAL': logging.CRITICAL
}

def initlog(filename) :

    logger = logging.getLogger()
    hdlr = logging.FileHandler(filename)
    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(LOG_LEVELS['INFO'])

    return logger


errlog = initlog("error.log")
infolog = initlog("info.log")

class WordReading(object):

    def __init__(self, fileList):
        self.fileList = fileList

    def readFileInternal(self, filename):
        lines = []
        try:
            f = open(filename, 'r')
            lines = f.readlines()
            infolog.info('[successful read file %s]
' % filename)
            f.close()
        except IOError, err:
            errorInfo = 'file %s Not found 
' % filename
            errlog.error(errorInfo)
        return lines

    def readFile(self):
        allLines = []
        for filename in self.fileList:
            allLines.extend(self.readFileInternal(filename))
        return allLines

class WordAnalyzing(object):
    '''
     return Map<Word, count>  the occurrence times of each word
    '''
    wordRegex = re.compile("[w]+")
    def __init__(self, allLines):
        self.allLines = allLines

    def analyze(self):
        result = {}
        lineContent = ''.join(self.allLines)
        matches = WordAnalyzing.wordRegex.findall(lineContent)
        if matches:
            for word in matches:
                if result.get(word) is None:
                    result[word] = 0
                result[word] += 1
        return result

class FileObtainer(object):

    def __init__(self, dirpath, fileFilterFunc=None):
        self.dirpath = dirpath
        self.fileFilterFunc = fileFilterFunc

    def findAllFilesInDir(self):
        files = []
        for path, dirs, filenames in os.walk(self.dirpath):
            if len(filenames) > 0:
                for filename in filenames:
                    files.append(path+'/'+filename)

        if self.fileFilterFunc is None:
            return files
        else:
            return filter(self.fileFilterFunc, files)

class PostProcessing(object):

    def __init__(self, resultMap):
        self.resultMap = resultMap

    def sortByValue(self):
        return sorted(self.resultMap.items(),key=lambda e:e[1], reverse=True)

    def obtainTopN(self, topN):
        sortedResult = self.sortByValue()
        sortedNum = len(sortedResult)
        topN = sortedNum if topN > sortedNum else topN
        for i in range(topN):
            topi = sortedResult[i]
            print topi[0], ' counts: ', topi[1]

if __name__ == "__main__":

    dirpath = "c:\Users\qin.shuq\Desktop\region_master\src"

    starttime = time.time()
    fileObtainer = FileObtainer(dirpath, lambda f: f.endswith('.java'))
    fileList = fileObtainer.findAllFilesInDir()
    endtime = time.time()
    print 'ObtainFile cost: ', (endtime-starttime)*1000 , 'ms'

    starttime = time.time()
    wr = WordReading(fileList)
    allLines = wr.readFile()
    endtime = time.time()
    print 'WordReading cost: ', (endtime-starttime)*1000 , 'ms'

    starttime = time.time()
    wa = WordAnalyzing(allLines)
    resultMap = wa.analyze()
    endtime = time.time()
    print 'WordAnalyzing cost: ', (endtime-starttime)*1000 , 'ms'

    starttime = time.time()
    postproc = PostProcessing(resultMap)
    postproc.obtainTopN(30)
    endtime = time.time()
    print 'PostProcessing cost: ', (endtime-starttime)*1000 , 'ms'