DocSegment.cpp

/*
 * DocSegment.cpp
 * Created on: 2011-11-10
 *   function:分析网页算法的实现
 *   将原始网页库中存储的网页转化为
 *   一组词的集合.
 */
#include <algorithm>
#include <fstream>
#include <iostream>
#include <map>
#include <vector>

#include "Md5.h"
#include "Url.h"
#include "Document.h"
#include "ChSeg/Dict.h"
#include "ChSeg/HzSeg.h"
#include "StrFun.h"

CDict iDict;

using namespace std;

const unsigned int HEADER_BUF_SIZE = 1024;//记录头和网页头信息的最大长度
//const unsigned int MAX_DOC_ID = 12932;        // you should change according "Doc.idx"
const unsigned int MAX_DOC_ID = 767;
//所要处理的原始网页库中文档的个数
//不同的原始网页库文档数不同，这个值
//需要更改,可以通过URL索引文件[Url.idx]得到

int main(int argc, char* argv[])
{
    string strLine, strFileName=argv[1];
    CUrl iUrl;
    vector<CUrl> vecCUrl;//为什么不是map
    CDocument iDocument;
    vector<CDocument> vecCDocument;//vector容器保存文档对象
    unsigned int docId = 0;

    //ifstream ifs("Tianwang.raw.2559638448");
    ifstream ifs(strFileName.c_str());//文档对象

    if (!ifs) {
        cerr << "Cannot open tianwang.img.info for input\n";
        return -1;
    }

    ifstream ifsUrl("Url.idx.sort_uniq");
    if (!ifsUrl) {
        cerr << "Cannot open Url.idx.sort_uniq for input\n";
        return -1;
    }
    ifstream ifsDoc("Doc.idx");
    if (!ifsDoc) {
        cout<<"不能打开网页索引文件"<<endl;
        cerr << "Cannot open Doc.idx for input\n";
        return -1;
    }

    while (getline(ifsUrl, strLine)) {//读入md5到id的映射
        char chksum[33];
        int  docid;

        memset(chksum, 0, 33);
        //cout<<strLine.c_str()<<endl<<endl;
        sscanf( strLine.c_str(), "%s%d", chksum, &docid );
        //cout<<strLine.c_str();
        iUrl.m_sChecksum = chksum;
        iUrl.m_nDocId = docid;
        vecCUrl.push_back(iUrl);
    }

    while (getline(ifsDoc,strLine)){
        /* docid：文档编号
         * pos：偏移
         * length:好像和DocIndex的版本有点问题*/

        int docid,pos,length;
        char chksum[33];

        //cout<<strLine<<endl<<endl;

        memset(chksum, 0, 33);
        sscanf( strLine.c_str(), "%d%d%d%s", &docid, &pos, &length,chksum );
        //cout<<endl<<docid<< pos<<length<<endl;
        iDocument.m_nDocId = docid;
        iDocument.m_nPos = pos;
        iDocument.m_nLength = length;
        iDocument.m_sChecksum = chksum;//网页的MD5
        vecCDocument.push_back(iDocument);
    }



    //保存网页分析的结果
    strFileName += ".seg";
    ofstream fout(strFileName.c_str(), ios::in|ios::out|ios::trunc|ios::binary);
    for ( docId=0; docId<MAX_DOC_ID; docId++ ){

        // find document according to docId
        int length = vecCDocument[docId+1].m_nPos - vecCDocument[docId].m_nPos -1;//当前网页文件的长度
        char *pContent = new char[length+1];//记录内容
        memset(pContent, 0, length+1);
        ifs.seekg(vecCDocument[docId].m_nPos);//移动读取位置
        ifs.read(pContent, length);

        char *s;
        s = pContent;

        // 过滤记录头
        int bytesRead = 0,newlines = 0;
        while (newlines != 2 && bytesRead != HEADER_BUF_SIZE-1) {
            if (*s == '\n')
                newlines++;
            else
                newlines = 0;
            s++;
            bytesRead++;
        }
        if (bytesRead == HEADER_BUF_SIZE-1) continue;


        // 过滤网页头部信息
        bytesRead = 0,newlines = 0;
        while (newlines != 2 && bytesRead != HEADER_BUF_SIZE-1) {
            if (*s == '\n')
                newlines++;
            else
                newlines = 0;
            s++;
            bytesRead++;
        }
        if (bytesRead == HEADER_BUF_SIZE-1) continue;//一般没有1024??

        //iDocument.m_sBody = s;
        //过滤网页体的正文信息
        iDocument.RemoveTags(s);
        iDocument.m_sBodyNoTags = s;

        delete[] pContent;
        string strLine = iDocument.m_sBodyNoTags;

        CStrFun::ReplaceStr(strLine, "&nbsp;", " ");//将网页体正文中的"&nbsp"替换成" "
        CStrFun::EmptyStr(strLine); // set " \t\r\n" to " "


        //将网页体正文信息进行分词
        CHzSeg iHzSeg;
        //cout<<strLine<<endl<<endl;
        strLine = iHzSeg.SegmentSentenceMM(iDict,strLine);
        cout<<docId<<";";
        cout<<strLine;
        fout << docId << endl << strLine;
        fout << endl;
        
    }

    return(0);
}