K-meams文本聚类算法C++实现

FROM:http://www.cnblogs.com/finallyliuyu/archive/2010/09/03/1817348.html

头文件：


 #ifndef _Preprocess_H
#define  _Preprocess_H
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include"ictclas30.h"
#include"boost	r1
egex.hpp"
#include"boost/algorithm/string.hpp"
#include"windows.h"
 
//一些谓词函数
using namespace std;
 
class Preprocess
{      
    //typedef  vector(Preprocess::*FUNCSEG)(string,set);
    private:
         char *bagofwordsAddress;//存放词袋子模型的位置
        char * featurewordsAddress;//存放特征词文件的位置；
        char *arffFileAddress;//存放ARFF文件的位置
        char *infoFromWekaAddress;//存放调用weka后的实验结果
        char *articleIdsAddress;//存放被聚类的文章的ID号
        char *dbconnection;//数据库的链接字符串
        char *dbselect;//数据库select语句
        char *dbfield;//数据库字段
        int beginIndex;//开始聚类的文章id
        int endIndex;//结束聚类的文章id
    public:
        typedef vector(Preprocess::*FUNCSEG)(string,set);
        Preprocess(int c_style_stringsize,const char *mydict,const char *keywordsinfo,const char *tobeCluster,const char * InfoFromWeka,const char *artileIds,const char *conn,const char *selectsql, int beginIndex,int endIndex)
        {
                bagofwordsAddress=new char[c_style_stringsize];
                featurewordsAddress=new char[c_style_stringsize];
                arffFileAddress=new char[c_style_stringsize];
                infoFromWekaAddress=new char[c_style_stringsize];
                articleIdsAddress=new char[c_style_stringsize];
                dbconnection=new char[c_style_stringsize];
                dbselect=new char[c_style_stringsize];
                this->beginIndex=beginIndex;
                this->endIndex=endIndex;
                sprintf_s(bagofwordsAddress,c_style_stringsize,mydict);
                sprintf_s(featurewordsAddress,c_style_stringsize,keywordsinfo);
                sprintf_s(arffFileAddress,c_style_stringsize,tobeCluster);
                sprintf_s(infoFromWekaAddress,c_style_stringsize,InfoFromWeka);
                sprintf_s(articleIdsAddress,c_style_stringsize,artileIds);
                sprintf_s(dbconnection,c_style_stringsize,conn);
                sprintf_s(dbselect,c_style_stringsize,selectsql);
             
 
 
        }
        
 
        ~Preprocess()
        {
            delete []bagofwordsAddress;
            delete []featurewordsAddress;
            delete []arffFileAddress;
            delete [] infoFromWekaAddress;
            delete []articleIdsAddress;
            delete []dbconnection;
            delete []dbselect;
             
 
        }
        void trim(string  &str,const string val);//去除字符串首尾空白
        //构建倒排表： key=word,val= a list of pairs which consists of articleid,and count, count=tf
        int ConstructMap(mapint,int>>>&mymap,char *dbfield,FUNCSEG seg);
        inline void TruncateArff()
        {
            ofstream ofile;
            ofile.open(arffFileAddress,ios::trunc);
            ofile.close();
        }
        //保存词袋子到硬盘
        void save(mapint,int> > >&mymap);
        //从内存中加载词袋子模型
        void load(mapint,int> > >&mymap);
        //打印词袋子模型
        void print(mapint,int> > >&mymap);
        //窄字符串转化成宽字符串
        wstring myMultibyteToWideChar(string sResult);
        //宽字符串转化成窄字符串
        string myWideCharToMultibyte(wstring wsResult);
        //调用ICTclass分词
        string ICTsplit(const char *sInput);
        //构造停用词表
        setMakeStopSet();
        //去除停用词，噪声词
        vectorgoodWordsinPieceArticle(string rawtext,set stopwords);
        //整数转化成字符串
        string do_fraction(int val);
        //浮点数转化成字符串
        string do_fraction(double val, int decplaces=5);
        //特征词选择算法
        void DFcharicteristicWordSelection(mapint,int>>> &mymap,int DFthreshold);
        //获取最后的特征词
        vector GetFinalKeyWords();
        //获取特征词的maxTF，DF
        vectorint,int> >GetfinalKeysMaxTFDF(mapint,int>>> &mymap);
        //文档向量模型规范化
        vectorint,double> > NormalizationVSM(vectorint,double> > tempVSM);
        //建立文档向量模型并且写到arff文件里
        void VSMFormation(mapint,int>>> &mymap);
        
        string FormatVSMtoString(vectorint,double> > tempVSM);
        //写Arff文件头部
        void WriteHeadArff();
        void WriteTotalArff(char * dbfield,int DFthreshlod,bool isbagOfwordsexsist,FUNCSEG seg);
        
        
        map<</code>int,vector<</code>double> >VSMConstruction(mapint,int>>> &mymap);
        
        map<</code>double> > GetClusters();
        
        double CalDotProductOfVectors(const vector<</code>double>&vector1,const vector<</code>double>&vector2);
        
        double CalCosineofVectors(const vector<</code>double>&vector1,const vector<</code>double>&vector2);
        
        vectorint,string> >GenerateClusterInfo(map<</code>int,vector<</code>double> >&vsmMatrix, map<</code>double> >&clusters);
        
        map<</code>int> >FetchArticlesOFClusters(map<</code>double> >&clusters,vectorint,string>>&resultInfo);
        void RetreiveArticleInfoFromDataBase();
        vector mySplit(string s,set stopwords);//分割关键词
 
         
 
 
 
 
 
 
};
 
 
 
#endif 


 Preprocess类的函数功能实现文件： 




 #include"stdafx.h"
#include "Preprocess.h"
 
#pragma comment(lib, "ICTCLAS30.lib")
using namespace std;
bool isLonger(const  pairint> &pair1, const pairint>  &pair2)
{
    return pair1.second>pair2.second;
}
bool cntAssist(const  pairint> &pair1)
{
    return pair1.second<=100;
}
bool PredTF(const pair<</code>int,int>& pair1,int articleId)
{
    return pair1.first==articleId;
 
}
class PredTFclass
{
private: const int m;
public:
    PredTFclass(int id):m(id){};
    bool operator()(const pair<</code>int,int>& pair1){return PredTF(pair1,m);};
};
bool myCmp(const pairdouble>&pair1,const pairdouble>&pair2 )
{
    return pair1.second>=pair2.second;
}
 
void Preprocess:: trim(string  &str,const string val)
{
    str.erase(0,str.find_first_not_of(val));
    str.erase(str.find_last_not_of(val)+val.size());
}
int Preprocess::ConstructMap(mapint,int>>>&mymap,char *dbfield,FUNCSEG seg)
{
    //setMakeStopSet();
    CoInitialize(NULL);
    _ConnectionPtr pConn(__uuidof(Connection));
    _RecordsetPtr pRst(__uuidof(Recordset));
    pConn->ConnectionString=dbconnection;
    pConn->Open("","","",adConnectUnspecified);
    pRst=pConn->Execute(dbselect,NULL,adCmdText);
    setstopwords=MakeStopSet();
     
    while(!pRst->rsEOF)
    {   vectorwordcollection;
       //string keywordstr=(_bstr_t)pRst->GetCollect("CKeyWord");
        string rawtext=(_bstr_t)pRst->GetCollect(dbfield);
        if(rawtext!="")
        {
            wordcollection=(this->*seg)(rawtext,stopwords);
            string tempid=(_bstr_t)pRst->GetCollect("ArticleId");
            int articleid=atoi(tempid.c_str());
            for(vector::iterator strit=wordcollection.begin();strit!=wordcollection.end();strit++)
            {
                vectorint,int>>::iterator it;
                if(mymap[*strit].empty())
                {
                    pair<</code>int,int>mytemppair=make_pair(articleid,1);
                    mymap[*strit].push_back(mytemppair);
 
                }
                else
                {
                    for(it=mymap[*strit].begin();it!=mymap[*strit].end();it++)
                    { 
                        if(it->first==articleid)
                        {
                            it->second=++(it->second);
                            break;
                        }
 
                }
                if(it==mymap[*strit].end())
                {
                    pair<</code>int,int>mytemppair=make_pair(articleid,1);
                    mymap[*strit].push_back(mytemppair);
                }
 
            }
 
        }
 
 
    }
 
 
    pRst->MoveNext();
    wordcollection.clear();
 }
    pRst->Close();
    pConn->Close();
    pRst.Release();
    pConn.Release();
    CoUninitialize();
     
    return 0;
 
}
void Preprocess::save(mapint,int> > >&mymap)
{
    ofstream outfile(bagofwordsAddress,ios::binary);
    outfile<<mymap.size()<<endl;
    mapint,int> > >::iterator it;
    for (it=mymap.begin();it!=mymap.end();it++)
    {   outfile<<it->first<<endl;
    vectorint,int>>::iterator subit;
    outfile<<it->second.size()<<endl;
    for(subit=(it->second).begin();subit!=(it->second).end();++subit)
    {
        outfile<<subit->first<<" "<<subit->second<<" "<<";"<<" ";
    }
    outfile<<endl;
    }
    //outfile.write((char *)&mymap,sizeof(mymap));
 
    outfile.close();
 
}
void Preprocess::load(mapint,int> > >&mymap)
{
    std::locale loc1 = std::locale::global(std::locale(".936"));
    {
        // 在这里使用std::ifstream 或者 std::fstream
        ifstream infile(bagofwordsAddress,ios::binary);
        int lenMyMap;//保存词典长度
        int lenVector;//保存每个词出现的文章数目
        string key;//保存读出的map的键值
        int articleId;//文章标号
        int count;//在该文章中刚出现的数目
        string comma;
        string semicolon;
        infile>>lenMyMap;
        while(!infile.eof())
        {
            infile>>key;
            infile>>lenVector;
            vectorint,int> >temp;
            for (int i=0;i
            {
                infile>>articleId>>count>>semicolon;
                temp.push_back(make_pair(articleId,count));
            }
            mymap[key]=temp;
 
 
        }
 
 
        infile.close();
    }
    std::locale::global(std::locale(loc1));
 
}
void print(mapint,int> > >&mymap)
{  
    cout<<mymap.size()<<endl;
    mapint,int> > >::iterator it;
    for (it=mymap.begin();it!=mymap.end();it++)
    {   cout<<it->first<<endl;
    vectorint,int>>::iterator subit;
    cout<<it->second.size()<<endl;
    for(subit=(it->second).begin();subit!=(it->second).end();++subit)
    {
        cout<<subit->first<<','<<subit->second<<";";
    }
    cout<<endl;
    }
 
}
set Preprocess::MakeStopSet()
{
    set stopwordsSet;
    ifstream ifile("stopwords.txt");
    while(!ifile.eof())
    {
        string temp;
        trim(temp," ");
        ifile>>temp;
        stopwordsSet.insert(temp);
    }
    return stopwordsSet;
}
 
string Preprocess::do_fraction(int val)
{
    ostringstream out;
    out<<val;
    string str= out.str(); //从流中取出字符串
    str.swap(string(str.c_str()));//删除nul之后的多余字符
    return str;
 
}
string Preprocess::do_fraction(double val,int decplaces)
{
     
    //int prec=numeric_limits::digits10;
    char DECIMAL_POINT='.';
    ostringstream out;
    //out.precision(prec);
    out<<val;
    string str=out.str();
    size_t n=str.find(DECIMAL_POINT);
    if((n!=string::npos)&&n+decplaces
    {
        str[n+decplaces]='';
    }
    str.swap(string(str.c_str()));
 
    return str;
}
wstring Preprocess::myMultibyteToWideChar(string sResult)
{
    int iWLen=MultiByteToWideChar( CP_ACP, 0, sResult.c_str(), sResult.size(), 0, 0 );// 计算转换后宽字符串的长度。（不包含字符串结束符）
    wchar_t *lpwsz= new wchar_t [iWLen+1];
    MultiByteToWideChar( CP_ACP, 0, sResult.c_str(), sResult.size(), lpwsz, iWLen ); // 正式转换。
    lpwsz[iWLen] = L'';
    wstring wsResult(lpwsz);
    delete []lpwsz;
    return wsResult;
}
string Preprocess::myWideCharToMultibyte(wstring wsResult)
{
    string sResult;
    int iLen= WideCharToMultiByte( CP_ACP, NULL, wsResult.c_str(), -1, NULL, 0, NULL, FALSE ); // 计算转换后字符串的长度。（包含字符串结束符）
    char *lpsz= new char[iLen];
    WideCharToMultiByte( CP_OEMCP, NULL, wsResult.c_str(), -1, lpsz, iLen, NULL, FALSE); // 正式转换。
    sResult.assign( lpsz, iLen-1 ); // 对string对象进行赋值。
    delete []lpsz;
    return sResult;
 
}
string Preprocess::ICTsplit(const char *sInput)
{
    if(!ICTCLAS_Init())
    {
        printf("ICTCLAS INIT FAILED!
");
        string strerr(sInput);
        return strerr;
    }
    ICTCLAS_SetPOSmap(ICT_POS_MAP_SECOND);
    //导入用户词典后
    
 
    const char* sResult = ICTCLAS_ParagraphProcess(sInput, 0);
    string strresult(sResult);
    //printf("%s
", sResult);
    //把字符串转化成宽字符串
    wstring wsResult=myMultibyteToWideChar(strresult);
    boost::wregex wreg(L"\s+");
    wsResult=boost::regex_replace(wsResult,wreg,wstring(L"|"));
    strresult=myWideCharToMultibyte(wsResult);
 
 
 
    //ofile<<str1;
    //ofile.close();
    //cout<<str1<<endl;
    //ICTCLAS_FileProcess("text.txt","test_result.txt",1);
    ICTCLAS_Exit();
 
    return strresult;
}
vectorPreprocess::goodWordsinPieceArticle(string rawtext,set stopwords)
{
    vector goodWordstemp;
    vector goodWords;
    const char* sInput=rawtext.c_str();
    string sResult=ICTsplit(sInput);
    wstring wsResult=myMultibyteToWideChar(sResult);
    boost::wregex wreg(L"\d+");//去掉中文空格
    wsResult=boost::regex_replace(wsResult,wreg,wstring(L""));
    //boost::regex_split(back_inserter(goodWordstemp),wsResult,wreg);
    boost::split(goodWordstemp,wsResult,boost::is_any_of("|"));
 
    for(vector::iterator it=goodWordstemp.begin();it!=goodWordstemp.end();it++)
    {
        string temp=myWideCharToMultibyte(*it);
        trim(temp," ");
        if(!stopwords.count(temp)&&!temp.empty())
        {
            goodWords.push_back(temp);
        }
 
 
    }
 
    return goodWords;
}
void Preprocess::DFcharicteristicWordSelection(mapint,int>>> &mymap,int DFthreshold)
{
    int finalKeyWordsCount=0;//计算共取了多少个关键词
    vectorint> >tempvector;
    for(mapint,int>>>::iterator it=mymap.begin();it!=mymap.end();++it)
    {
        tempvector.push_back(make_pair(it->first,(it->second).size()));
    }
 
    stable_sort(tempvector.begin(),tempvector.end(),isLonger);
    ofstream outfile(featurewordsAddress);
    for(vectorint> >::iterator it=tempvector.begin();it!=tempvector.end();it++)
    {  
        if(it->second>=DFthreshold)
        {
            //outfile<<it->first<<" "<<it->second<<endl;
            outfile<<it->first<<endl;
            finalKeyWordsCount++;
 
        }
 
    }
    outfile.close();
    cout<<"最后共选择特征词"<<finalKeyWordsCount<<endl;
    cout<<"by the way,DFthreshold equals"<<DFthreshold<<endl;
 
}
vectorPreprocess::GetFinalKeyWords()
{
    vectormyKeys;
    ifstream infile(featurewordsAddress);
    while(!infile.eof())
    {
        string temp;
        infile>>temp;
        if(temp!="")
        {
            myKeys.push_back(temp);
        }
 
 
    }
    return myKeys;
}
vectorint,int> >Preprocess::GetfinalKeysMaxTFDF(mapint,int>>> &mymap)
{
    vectorint,int> >maxTFandDF;
    vectormyKeys=GetFinalKeyWords();
    for(vector::iterator it=myKeys.begin();it!=myKeys.end();it++)
    { 
        int DF=mymap[*it].size();
        int maxTF=0;
        for(vectorint,int> >::iterator subit=mymap[*it].begin();subit!=mymap[*it].end();subit++)
        {
            if(subit->second>maxTF)
            {
                maxTF=subit->second;
            }
 
        }
        maxTFandDF.push_back(make_pair(maxTF,DF));
        //find_if(mymap[*it].begin(),mymap[*it].end(),
    }
    return maxTFandDF;
}
vectorint,double> >Preprocess::NormalizationVSM(vectorint,double> > tempVSM)
{
 
    double sum=0;
    for(vectorint,double> >::iterator vsmit=tempVSM.begin();vsmit!=tempVSM.end();++vsmit)
    {
        sum+=pow(vsmit->second,2);
    }
    for(vectorint,double> >::iterator vsmit=tempVSM.begin();vsmit!=tempVSM.end();++vsmit)
    {
        vsmit->second/=sqrt(sum);
    }
    return tempVSM;
 
}
string Preprocess::FormatVSMtoString(vectorint,double> > tempVSM)
{
    string ret="{";
    int commaindication=0;
    for(vectorint,double> >::iterator vsmit=tempVSM.begin();vsmit!=tempVSM.end();++vsmit)
    {  
 
        ret+=do_fraction(vsmit->first)+" "+do_fraction(vsmit->second,8);
        if(commaindication
        {
            ret+=",";
        }
        commaindication++;
    }
    ret+="}";
    return ret;
}
void Preprocess::WriteHeadArff()
{
    ofstream ofile(arffFileAddress,ios::binary);
    ofile<<"@relation aticle"<<endl;
    ofile<<"
";
    vector myKeys=GetFinalKeyWords();
    for(vector::iterator it=myKeys.begin();it!=myKeys.end();it++)
    {
        //string temp="@attribute "+"'"+(*it)+"'"+" real";
        string temp="";
        temp+="@attribute ";
        temp+="'";
        temp+=*(it);
        temp+="'";
        temp+=" real";
        
 
        ofile<<temp<<endl;
    }
    ofile<<"
"<<endl;
    ofile<<"@data"<<endl;
    ofile.close();
}
void Preprocess::VSMFormation(mapint,int>>> &mymap)
{   int corpus_N=endIndex-beginIndex+1;
    ofstream ofile1(articleIdsAddress,ios::binary);//保存文章编号的文件
    ofstream ofile2(arffFileAddress,ios::binary|ios::app);
 
    vector myKeys=GetFinalKeyWords();
    vectorint,int> >maxTFandDF=GetfinalKeysMaxTFDF(mymap);
    for(int i=beginIndex;i<=endIndex;i++)
    {   vectorint,double> >tempVSM;
        for(vector::size_type j=0;j
        {
        //vector >::iterator findit=find_if(mymap[myKeys[j]].begin(),mymap[myKeys[j]].end(),PredTFclass(i));
            double TF=(double)count_if(mymap[myKeys[j]].begin(),mymap[myKeys[j]].end(),PredTFclass(i));
 
 
            TF=0.5+0.5*(double)TF/(maxTFandDF[j].first);
            TF*=log((double)corpus_N/maxTFandDF[j].second);
            if(TF!=0)
            {
                tempVSM.push_back(make_pair(j,TF));
 
            }
 
 
 
        }
        if(!tempVSM.empty())
        {
            tempVSM=NormalizationVSM(tempVSM);
            string vsmStr=FormatVSMtoString(tempVSM);
            ofile1<<i<<endl;
            ofile2<<vsmStr<<endl;
        }
        tempVSM.clear();
 
 
 
    }
    ofile1.close();
    ofile2.close();
 
 
}
void Preprocess::WriteTotalArff(char *dbfield,int DFthreshold,bool isbagOfWordsExist,FUNCSEG seg)
{
     
     
    mapint,int>>> mymap;
    if(!isbagOfWordsExist)
    {
        ConstructMap(mymap,dbfield,seg);
        save(mymap);
        cout<<"词袋子信息已经保存到硬盘"<<endl;
    }
    else
    {
        load(mymap);
    }
    DFcharicteristicWordSelection(mymap,DFthreshold);
    WriteHeadArff();
    VSMFormation(mymap);
    cout<<"arff文件已经形成"<<endl;
     
     
    string temp(infoFromWekaAddress);
 
    cout<<"请您将使用weka聚类，并保存为"<<temp<<endl;
}
map<</code>int,vector<</code>double> > Preprocess::VSMConstruction(mapint,int>>> &mymap)
{  
    int corpus_N=endIndex-beginIndex+1;
    map<</code>int,vector<</code>double>> vsmMatrix;
    vector myKeys=GetFinalKeyWords();
    vectorint,int> >maxTFandDF=GetfinalKeysMaxTFDF(mymap);
    for(int i=beginIndex;i<=endIndex;i++)
    {  
        vectorint,double> >tempVSM;
        for(vector::size_type j=0;j
        {
            //vector >::iterator findit=find_if(mymap[myKeys[j]].begin(),mymap[myKeys[j]].end(),PredTFclass(i));
            double TF=(double)count_if(mymap[myKeys[j]].begin(),mymap[myKeys[j]].end(),PredTFclass(i));
            TF=0.5+(double)TF/(maxTFandDF[j].first);
            TF*=log((double)corpus_N/maxTFandDF[j].second);
            tempVSM.push_back(make_pair(j,TF));
 
        }
        if(!tempVSM.empty())
        {
            tempVSM=NormalizationVSM(tempVSM);
            for(vectorint,double> >::iterator it=tempVSM.begin();it!=tempVSM.end();it++)
            {
                vsmMatrix[i].push_back(it->second);
            }
 
 
 
        }
        tempVSM.clear();
 
 
 
    }
    return vsmMatrix;
 
}
map<</code>double> > Preprocess::GetClusters()
{
 
    map<</code>double> >clusters;
    ifstream ifile(infoFromWekaAddress);
    string temp;
    while(getline(ifile,temp))
    {   boost::smatch matchcluster;
    boost::regex regcluster("Cluster\s+\d+",boost::regex::icase);
    if(boost::regex_search(temp,matchcluster,regcluster))  
    {  
        string clustertmp=matchcluster[0].str();
        string ordinates="";
        getline(ifile,ordinates);
        boost::regex regordinates("\d+(\.\d{1,4})?");
        boost::smatch matchordinates;
        std::string::const_iterator it=ordinates.begin(); 
        std::string::const_iterator end=ordinates.end();
        while (boost::regex_search(it,end,matchordinates,regordinates))
        {      
            string digitstemp=matchordinates[0].str();
            double digitval=0.0;
            std::stringstream ss;
            ss<<digitstemp;
            ss>>digitval;
            clusters[clustertmp].push_back(digitval);
            it=matchordinates[0].second;
        }
 
 
 
 
 
    }
    }
    return clusters;
}
double Preprocess::CalDotProductOfVectors(const vector<</code>double>&vector1,const vector<</code>double>&vector2)
{
    double result = 0.0f;
    for (int i = 0; i < vector1.size(); i++)
        result += vector1[i] * vector2[i];
    return result;
}
double Preprocess::CalCosineofVectors(const vector<</code>double>&vector1,const vector<</code>double>&vector2)
{
    double numerator=CalDotProductOfVectors(vector1,vector2);
    double denominator=CalDotProductOfVectors(vector1,vector1)*CalDotProductOfVectors(vector2,vector2);
    denominator=sqrt(denominator);
    return numerator/denominator;
}
vectorint,string> > Preprocess::GenerateClusterInfo(map<</code>int,vector<</code>double> >&vsmMatrix, map<</code>double> >&clusters)
{
    vectorint,string> >resultInfo;
    for(map<</code>int,vector<</code>double> >::iterator it=vsmMatrix.begin();it!=vsmMatrix.end();it++)
    {
        vectordouble> >clusterDistanceAist;
        for(map<</code>double> >::iterator clusterit=clusters.begin();clusterit!=clusters.end();clusterit++)
        {
 
            double temp=CalCosineofVectors(it->second,clusterit->second);
            clusterDistanceAist.push_back(make_pair(clusterit->first,temp));
 
        }
        sort(clusterDistanceAist.begin(),clusterDistanceAist.end(),myCmp);
        vectordouble> >::iterator cDAit=clusterDistanceAist.begin();
 
        resultInfo.push_back(make_pair(it->first,cDAit->first));
        clusterDistanceAist.clear();
    }
    return  resultInfo;
 
}
map<</code>int> > Preprocess::FetchArticlesOFClusters(map<</code>double> >&clusters,vectorint,string>>&resultInfo)
{
    map<</code>int>> articlesInfo;
 
    for(vectorint,string>>::iterator retit=resultInfo.begin();retit!=resultInfo.end();retit++)
    {
        for(map<</code>double> >::iterator it=clusters.begin();it!=clusters.end();it++)
        {
            if(retit->second==it->first)
            {
                articlesInfo[it->first].push_back(retit->first);
            }
        }
    }
 
 
 
 
 
    return articlesInfo;
 
 
}
void Preprocess::RetreiveArticleInfoFromDataBase()
{
    mapint,int>>> mymap;
    vectorint,string>>resultInfo;
    map<</code>double> >clusters;
    map<</code>int,vector<</code>double> >vsmMatrix;
    map<</code>int>> articlesInfo;
    ofstream ofile("F:\cluster\ArticlesInPerCluster.txt");
    //boost::regex_replace(strresult)
    //ConstructMap(mymap,1,500);
    //save(mymap);
    load(mymap);
    vsmMatrix=VSMConstruction(mymap);
    clusters=GetClusters();
    resultInfo=GenerateClusterInfo(vsmMatrix,clusters);
    articlesInfo=FetchArticlesOFClusters(clusters,resultInfo);
 
    
    for(map<</code>int>>::iterator it=articlesInfo.begin();it!=articlesInfo.end();it++)
    {
        ostringstream out;
        string selectassist;
        char *selectsql=new char[5000];
        int count=0;
        CoInitialize(NULL);
        _ConnectionPtr pConn(__uuidof(Connection));
        _RecordsetPtr pRst(__uuidof(Recordset));
        pConn->ConnectionString=dbconnection;
        pConn->Open("","","",adConnectUnspecified);
        cout <<it->first<<endl;
        ofile<<it->first<<endl;
        out<<"(";
        count=0;
        for(int i=0;isecond.size();i++)
        {
            out<<(it->second)[i];
            if(countsecond.size()-1)
            {
                out<<",";
            }
            count++;
             
         
        }
        out<<")";
        selectassist=out.str();
        sprintf_s(selectsql,5000,"%s %s","Select ArticleTitle,class from News Where ArticleId in ",selectassist.c_str());
 
        pRst=pConn->Execute(selectsql,NULL,adCmdText);
        while(!pRst->rsEOF)
        {  
        //string keywordstr=(_bstr_t)pRst->GetCollect("CKeyWord");
            string title=(_bstr_t)pRst->GetCollect("ArticleTitle");
            //string rawtext=(_bstr_t)pRst->GetCollect("ArticleText");
            string categorization=(_bstr_t)pRst->GetCollect("class");
            cout<<"文章标题："<<title<<"文章所属类别: "<<categorization<<endl;
            ofile<<"文章标题："<<title<<"文章所属类别: "<<categorization<<endl;
 
 
             
         
 
 
            pRst->MoveNext();
             
        }
        pRst->Close();
        pConn->Close();
        pRst.Release();
        pConn.Release();
        CoUninitialize();
     
    }
     
     
 
 
ofile.close(); 
     
     
}
vectorPreprocess:: mySplit(string s,set stopwords)
{
    vector wordCollection;
    trim(s," ");
 
    int nPosBegin=0;
    int nPosEnd=s.find(' ',nPosBegin);
    while(nPosEnd!=string::npos)
    {
        string temp=s.substr(nPosBegin,nPosEnd-nPosBegin);
        trim(temp," ");
        wordCollection.push_back(temp);
        nPosBegin=s.find_first_not_of(' ',nPosEnd);
        nPosEnd=s.find(' ',nPosBegin);
    }
    string temp=s.substr(nPosBegin,s.size()-nPosBegin);
    trim(temp," ");
    wordCollection.push_back(temp);
 
 
    return wordCollection;
 
}