CPage

#ifndef _Page_H_030728_
#define _Page_H_030728_

#include <string>
#include <map>
#include <vector>
#include <list>
#include "Url.h"
#include "list.h"
#include "uri.h"
#include "hlink.h"


//large enough to hold sina's 437 links 

const int ANCHOR_TEXT_LEN       = 256;
const int MAX_URL_REFERENCES    = 1000;
const int URL_REFERENCE_LEN     = (URL_LEN+ANCHOR_TEXT_LEN)*MAX_URL_REFERENCES*1/2 ;
const int MAX_TAG_NUMBERS    = 10000;

using namespace std;

// plain text or other
enum page_type {
    PLAIN_TEXT,
    OTHER    
};

struct RefLink4SE // <href src...>, <area src...>
{
    char *link;
    char *anchor_text;
    string strCharset;
};

struct RefLink4History    // <img src...>,<script src...>
{
    char *link;
};

class CPage
{
public:
    // url & location
    string m_sUrl;        //网页对应的URL字符串

    // header
    string m_sHeader;//网页头信息
    int m_nLenHeader;//网页头信息的长度

    int m_nStatusCode;//状态码
    int m_nContentLength;;//从网页头信息中提取的网页体的长度，一般不是很准
    string m_sLocation;//网页的转向信息，可以判断这个网页是否重定向
    bool m_bConnectionState;    //是否支持持续链接Keep-Alive为true否则为false
    string m_sContentEncoding;//网页体的编码
    string m_sContentType;//网页体的类型
    string m_sCharset;//网页体的字符集
    string m_sTransferEncoding;//网页体的传输编码方式

    // content
    string m_sContent;//网页体信息
    int m_nLenContent;//网页体信息的长度
    string m_sContentNoTags;


    // link, in a lash-up state
    string m_sContentLinkInfo;
    //从网页体中提取出包含超链接信息的标识,例如<img src="www.baidu.com"/> ,
    //<a href="www.baidu.com">百度</a> <area href="www.baidu.com">百度</area>

    // links for SE, in a lash-up state
    string m_sLinkInfo4SE;
    //再从m_sContentLinkInfo提取出<a href="www.baidu.com">百度</a> <area href="www.baidu.com">百度</area>标识信息
    int m_nLenLinkInfo4SE;;//m_sLinkInfo4SE的长度

    // links for history archiving, in a lash-up state
    string m_sLinkInfo4History;//再从m_sContentLinkInfo提取出<img src="www.baidu.com">标识信息
    int m_nLenLinkInfo4History;//m_sLinkInfo4History的长度


    // links for SE, in a good state
    RefLink4SE m_RefLink4SE[MAX_URL_REFERENCES];//保存URL信息<-->URL的描述信息[这里URL指的是为搜索准备的链接] 即每个网页最多能保存1000个链接
    int m_nRefLink4SENum;//上面数组的长度

    // links for history archiving, in a good state
    RefLink4History m_RefLink4History[MAX_URL_REFERENCES/2];//保存URL信息[这个URL指的是为历史网页存档准备的链接]
    int m_nRefLink4HistoryNum;//上面数组的长度

    //map<string,string,less<string> > m_mapLink4SE;
    map<string,string> m_mapLink4SE;//保存URL信息<-->URL的描述信息[这里URL指的是为搜索准备的链接]
                                    //-----当然了这个map容器的作用主要是删除一个网页中相同的URL
    vector<string > m_vecLink4History;//保存URL信息--当然了这个vector容器的作用主要是删除一个网页中相同的URL

    // page type
    enum page_type m_eType;//网页的类型

    // parsed url lists
    //list<string>    m_listLink4SE;

public:
    CPage();
    CPage(string strUrl, string strLocation, char* header, char* body, int nLenBody);
    ~CPage();

    // parse header information from the header content
    void ParseHeaderInfo(string header);//解析网页头信息

    // parse hyperlinks from the page content
    bool ParseHyperLinks();//从网页中提取出链接信息

    bool NormalizeUrl(string& strUrl);//判断strUrl是不是正规的url

    bool IsFilterLink(string plink);//判断plink链接是不是要过滤掉

private:
    // parse header information from the header content
    void GetStatusCode(string header);//得到状态码
    void GetContentLength(string header);//从网页头信息中提取的网页体的长度，一般不是很准
    void GetConnectionState(string header);//得到连接状态
    void GetLocation(string header);//得到重定向信息
    void GetCharset(string header);//得到字符集
    void GetContentEncoding(string header);//得到网页体编码
    void GetContentType(string header);//得到网页体类型
    void GetTransferEncoding(string header);//得到网页体的传输编码方式

    // parse hyperlinks from the web page
    bool GetContentLinkInfo();//从网页体中提取出包含超链接信息的标识,
                              //例如<img src="www.baidu.com"/> ,<a href="www.baidu.com">百度</a> <area href="www.baidu.com">百度</area>


    bool GetLinkInfo4SE();//再从m_sContentLinkInfo提取出<a href="www.baidu.com">百度</a> <area href="www.baidu.com">百度</area>标识信息
    bool GetLinkInfo4History();//再从m_sContentLinkInfo提取出<img src="www.baidu.com">标识信息
    bool FindRefLink4SE();//最终得到为搜索引擎准备的超链接
    bool FindRefLink4History();//最终得到为历史网页存档准备的超链接

};

#endif /* _Page_H_030728_ */

/*Page handling
 */

#include <iostream>
#include <string>
#include <cstring>
#include <map>
#include <vector>
#include <iterator>
#include "Url.h"
#include "Page.h"
#include "StrFun.h"


//带参构造函数
CPage::CPage()
{
    //初始化成员变量
    m_nStatusCode = 0;
    m_nContentLength = 0;
    m_sLocation = "";
    m_bConnectionState = false;
    m_sContentEncoding = "";
    m_sContentType = "";
    m_sCharset = "";
    m_sTransferEncoding = "";

    m_sContentLinkInfo = "";
    m_sLinkInfo4SE = "";
    m_sLinkInfo4History = "";

    m_sContentNoTags = "";
    m_nRefLink4SENum = 0;
    m_nRefLink4HistoryNum = 0;
    m_eType = PLAIN_TEXT;


    //超链接信息以及超链接的描述信息初始化都为空
    for(int i=0; i< MAX_URL_REFERENCES; i++ ){
        m_RefLink4SE[i].link = NULL;
        m_RefLink4SE[i].anchor_text = NULL;
        m_RefLink4SE[i].strCharset = "";

        if(i < MAX_URL_REFERENCES/2){
            m_RefLink4History[i].link = NULL;
        }
    }

}

CPage::CPage( string strUrl, string strLocation, char* header, char* body, int nLenBody)
{
    //assert( header != NULL );
    //assert( body != NULL );
    //assert( nLenBody > 0 );

    // CPage();
    m_nStatusCode = 0;
    m_nContentLength = 0;
    m_sLocation = "";
    m_bConnectionState = false;
    m_sContentEncoding = "";
    m_sContentType = "";
    m_sCharset = "";
    m_sTransferEncoding = "";

    m_sContentLinkInfo = "";
    m_sLinkInfo4SE = "";
    m_sLinkInfo4History = "";

    m_sContentNoTags = "";
    m_nRefLink4SENum = 0;
    m_nRefLink4HistoryNum = 0;
    m_eType = PLAIN_TEXT;

    //超链接信息以及超链接的描述信息初始化都为空
    for(int i=0; i< MAX_URL_REFERENCES; i++ ){
        m_RefLink4SE[i].link = NULL;
        m_RefLink4SE[i].anchor_text = NULL;
        m_RefLink4SE[i].strCharset = "";

        if(i < MAX_URL_REFERENCES/2){
            m_RefLink4History[i].link = NULL;
        }
    }

    //将构造函数传入的参数赋值给成员变量
    m_sUrl = strUrl;//网页对应的URL
    m_sLocation = strLocation;//网页重定向的URL，没有重定向则传入为空，否则传入重定向的URL信息
    m_sHeader = header;//网页的头信息
    m_nLenHeader = strlen(header);//网页头信息的长度

    m_sContent.assign(body, nLenBody);//网页体信息,用body所指向数组的前nLenBody个字符副本替换m_sContent
    m_nLenContent = nLenBody;//网页体信息的长度

}

CPage::~CPage()
{
}


//解析网页头信息---调用8个私有的成员函数
void CPage::ParseHeaderInfo(string strHeader)
{
    GetStatusCode(strHeader);
    GetContentLength(strHeader);
    GetLocation(strHeader);
    GetConnectionState(strHeader);

    GetCharset(strHeader);

    GetContentEncoding(strHeader);
    GetContentType(strHeader);
    GetTransferEncoding(strHeader);
}

//得到状态码
void CPage::GetStatusCode(string headerBuf)
{
    //例如：

    //HTTP/1.0 200 OK     200就是状态码
    CStrFun::Str2Lower( headerBuf, headerBuf.length() );

    char *charIndex = strstr(headerBuf.c_str(), "http/");//在字符串headerBuf中查找第一出现"http/"的位置
    if (charIndex == NULL)
    {
        m_nStatusCode = -1;
        return;
    }
    //吃掉所有无关的字符
    while(*charIndex != ' '){
        charIndex++;
    }
    charIndex++;
    
    int ret = sscanf(charIndex, "%i", &m_nStatusCode);//格式化字符串输入
    if (ret != 1)  m_nStatusCode = -1;
}



//从网页头信息中提取的网页体的长度，一般不是很准
void CPage::GetContentLength(string headerBuf)
{
    //例如:

    //content-length: 21237     21237就是网页体的长度，这个属性值是服务器返回的，不一定正确
    CStrFun::Str2Lower( headerBuf, headerBuf.length() );

    char *charIndex = strstr(headerBuf.c_str(), "content-length");
    if (charIndex == NULL) return;

    while(*charIndex != ' '){
        charIndex++;
    }
    charIndex++;
    
    int ret = sscanf(charIndex, "%i", &m_nContentLength);
    if (ret != 1)  m_nContentLength = -1;
}


//得到重定向信息
void CPage::GetLocation(string headerBuf)
{
    string::size_type pre_idx,idx;
    const string delims("\r\n");

    string strBuf =  headerBuf;
    CStrFun::Str2Lower( headerBuf, headerBuf.length() );

    idx = headerBuf.find("location:");
    if (idx != string::npos)//若找到
    {
        pre_idx = idx + sizeof("location: ") -1;
        idx = headerBuf.find_first_of(delims, pre_idx );//查找换行符
        if (idx != string::npos)
        {
            //m_sLocation = headerBuf.substr(pre_idx, idx - pre_idx);
            m_sLocation = strBuf.substr(pre_idx, idx - pre_idx);
        }
    }
}


//得到网页字符集
void CPage::GetCharset(string headerBuf)
{
    string::size_type pre_idx,idx;
    const string delims(" \",;>");

    CStrFun::Str2Lower(headerBuf, headerBuf.size());

    idx = headerBuf.find("charset=");
    if( idx != string::npos) {
        m_sCharset = headerBuf.substr(idx + sizeof("charset=") -1);//保存从charset=开始的所有字符串
    }

    headerBuf = m_sContent;
    headerBuf = headerBuf.substr(0,2024) ;
    CStrFun::Str2Lower( headerBuf, headerBuf.length() );
    idx = headerBuf.find("charset=");
    if (idx != string::npos)//后边有可能有多余的信息
    {
        pre_idx = idx + sizeof("charset=") -1;
        idx = headerBuf.find_first_of(delims, pre_idx );
        if(idx != string::npos){
            m_sCharset = headerBuf.substr(pre_idx, idx - pre_idx);
        }
    }
}


//得到网页体编码
void CPage::GetContentEncoding(string headerBuf)
{
    string::size_type pre_idx,idx;
    const string delims("\r\n");

    CStrFun::Str2Lower( headerBuf, headerBuf.length() );

    idx = headerBuf.find("content-encoding:");
    if (idx != string::npos)
    {
        pre_idx = idx + sizeof("content-encoding: ") -1;
        idx = headerBuf.find_first_of(delims, pre_idx );
        if (idx != string::npos)
        {
            m_sContentEncoding = headerBuf.substr(pre_idx, idx - pre_idx);
        }
    }
}

//得到连接状态
void CPage::GetConnectionState(string headerBuf)
{
    string::size_type pre_idx,idx;
    const string delims(";\r\n");

    CStrFun::Str2Lower( headerBuf, headerBuf.length() );

    idx = headerBuf.find("connection:");
    if (idx != string::npos)
    {
        pre_idx = idx + sizeof("connection: ") -1;
        idx = headerBuf.find_first_of(delims, pre_idx );
        if (idx != string::npos)
        {
            string str = headerBuf.substr(pre_idx, idx - pre_idx);
            //cout << "Connection state: " << str << endl;
            //if (str == "close") m_bConnectionState = false;
            if (str == "keep-alive") m_bConnectionState = true;
        }
    }
}

//得到网页体类型
void CPage::GetContentType(string headerBuf)
{
    string::size_type pre_idx,idx;
    const string delims(";\r\n");

    CStrFun::Str2Lower( headerBuf, headerBuf.size() );

    idx = headerBuf.find("content-type:");
    if (idx != string::npos)
    {
        pre_idx = idx + sizeof("content-type: ") -1;
        idx = headerBuf.find_first_of(delims, pre_idx );
        if (idx != string::npos)
        {
            m_sContentType = headerBuf.substr(pre_idx, idx - pre_idx);
        }
    }
}

//得到网页体的传输编码方式
void CPage::GetTransferEncoding(string headerBuf)
{
    string::size_type pre_idx,idx;
    const string delims(";\r\n");

    CStrFun::Str2Lower( headerBuf, headerBuf.size() );

    idx = headerBuf.find("transfer-encoding:");
    if ( idx != string::npos)
    {
        pre_idx = idx + sizeof("transfer-encoding: ") -1;
        idx = headerBuf.find_first_of(delims, pre_idx );
        if(idx != string::npos)
        {
            m_sTransferEncoding = headerBuf.substr(pre_idx, idx - pre_idx);
        }
    }
}

/*
 * Filter spam links
 * If it is, return ture; otherwise false
 */
//判断一个URL是不是应该过滤，要过滤返回true否则返回false
bool CPage::IsFilterLink(string plink)
{
    if( plink.empty() ) return true;
    if( plink.size() > URL_LEN ) return true;

    string link = plink, tmp;
    string::size_type idx = 0;

    
    CStrFun::Str2Lower( link, link.length() );//link字符串中的字母全部变成小写

    // find two times following symbols, return false
    tmp = link;
    idx = tmp.find("?");//URL中出现2个'?'字符要过滤
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("?");
        if( idx != string::npos ) return true;
    }

    tmp = link;//先后出现'-'和'+'字符要过滤
    idx = tmp.find("-");
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("+");
        if( idx != string::npos ) return true;
    }

    //出现2个'&'字符要过滤
    tmp = link;
    idx = tmp.find("&");
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("&");
        if( idx != string::npos ) return true;
    }

    //出现2个"//"字符要过滤
    tmp = link;
    idx = tmp.find("//");
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("//");
        if( idx != string::npos ) return true;
    }

    //出现2个"http"要过滤
    tmp = link;
    idx = tmp.find("http");
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("http");
        if( idx != string::npos ) return true;
    }

    //出现2个"misc"要过滤
    tmp = link;
    idx = tmp.find("misc");
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("misc");
        if( idx != string::npos ) return true;
    }

    //出现2个"ipb"要过滤
    tmp = link;
    idx = tmp.find("ipb");
    if( idx != string::npos ){
        tmp = tmp.substr(idx+1);
        idx = tmp.find("ipb");
        if( idx != string::npos ) return true;
    }

    const char *filter_str[]={
    "cgi-bin",    "htbin",    "linder",    "srs5",        "uin-cgi",  // robots.txt of http://www.expasy.org/
    "uhtbin",    "snapshot",    "=+",        "=-",        "script",
    "gate",        "search",    "clickfile",    "data/scop",    "names",
    "staff/",    "enter",    "user",        "mail",    "pst?",
    "find?",    "ccc?",        "fwd?",        "tcon?",    "&amp",
    "counter?",    "forum",    "cgisirsi",    "{",        "}",
    "proxy",    "login",    "00.pl?",    "sciserv.pl",    "sign.asp",
    "<",        ">",        "review.asp?",    "result.asp?",    "keyword",
    "\"",        "'",        "php?s=",    "error",    "showdate",
    "niceprot.pl?",    "volue.asp?id",    ".css",        ".asp?month",    "prot.pl?",
    "msg.asp",    "register.asp", "database",    "reg.asp",    "qry?u",
    "p?msg",    "tj_all.asp?page", ".plot.",    "comment.php",    "nicezyme.pl?",
    "entr",        "compute-map?", "view-pdb?",    "list.cgi?",    "lists.cgi?",
    "details.pl?",    "aligner?",    "raw.pl?",    "interface.pl?","memcp.php?",
    "member.php?",    "post.php?",    "thread.php",    "bbs/",        "/bbs"
    };
    int filter_str_num = 75;

    //说明找到了上述字符串要过滤
    for(int i=0; i<filter_str_num; i++){
        if( link.find(filter_str[i]) != string::npos)
        return true;
    }    

    return false;
}

/////////////////////////////
// just for ImgSE
// e.g: http://www.people.com.cn/GB/tupian/index.html
//     http://news.xinhuanet.com/photo/
//     http://photo.tom.com/
/////////////////////////////
// comment previous one and open this one

/*
bool CPage::IsFilterLink(string plink)
{
    if( plink.empty() ) return true;
    if( plink.size() > URL_LEN ) return true;

    return false;

    string link = plink, tmp;
    string::size_type idx = 0;

    
    CStrFun::Str2Lower( link, link.length() );

    const char *filter_str[]={
        "tupian", "photo", "ttjstk"
        };
    int filter_str_num = 3;

    CStrFun::Str2Lower( link, link.length() );

    for(int i=0; i<filter_str_num; i++){
        if( link.find(filter_str[i]) != string::npos)
        return false;
    }    

    return true;
}
*/


/*****************************************************************
** Function name: ParseHyperLinks
** Input argv:
**      --
** Output argv:
**      --
** Return:
        true: success
        false: fail
** Function Description:  Parse hyperlinks from the web page
** Version: 1.0
** Be careful:
*****************************************************************/
bool CPage::ParseHyperLinks()
{
    if( GetContentLinkInfo() == false ) return false;

    if( m_sContentLinkInfo.empty() ) return false;

    bool bFind4SE = false;
    bool bFind4History = false;
    if( GetLinkInfo4SE() ){
        if( FindRefLink4SE() ) bFind4SE = true;
    } 

    if( GetLinkInfo4History() ){
        if( FindRefLink4History() ) bFind4History = true;
    }

    //如果没有从网页中提取出为搜索引擎或者为历史网页存档准备的超链接则返回false
    if( !bFind4SE && !bFind4History ){
         return false;
    }

    //return   GetHref(m_sContentLinkInfo.c_str(), "href", m_listLink4SE);

    return true;
}


/*****************************************************************
** Function name: GetContentLinkInfo
** Input argv:
**      --
** Output argv:
**      --
** Return:
        true: success
        false: fail
** Function Description:  Parse hyperlinks from the web page
** Version: 1.0
** Be careful:
*****************************************************************/

//从网页体中提取出包含超链接信息的标识
bool CPage::GetContentLinkInfo()
{
    if( m_sContent.empty() ) return false;
    
    m_sContentLinkInfo = m_sContent;

    string& s = m_sContentLinkInfo; //引用调用

    // transform all separation into one space character
    //CStrFun::ReplaceStr(s, "\t", " ");
    //CStrFun::ReplaceStr(s, "\r", " ");
    //CStrFun::ReplaceStr(s, "\n", " ");
    const string delims(" \t\r\n");
    string::size_type idx=0, pre_idx;
    
    //找到所有的"\t\r\n"并将'\t'替换为' ' 如果是\t\t\r\n则删除一个\t
    while( (idx = s.find_first_of(delims, idx)) != string::npos )
    {
        pre_idx = idx;
        s.replace(idx,1,1,' ');
        idx++;
        
        while( (idx = s.find_first_of(delims, idx)) != string::npos )
        {
            if( idx-pre_idx == 1 ){
                s.erase(idx, 1);
            } else {
                break;
            }
        }

        idx--;
    }

    // transform all "<br>" into one space character
    //将s中<br>标记全部替换为空格
    CStrFun::ReplaceStr(s, "<br>", " ");

    if( s.size() < 20 ) return false;

    // Keep only <img ...>, <area ...>,<script ...> and <a href ...> tags.
    string::size_type idxHref=0,idxArea=0,idxImg=0;
    string dest;

    do{
        if( s.empty() ) break;

        idxHref = CStrFun::FindCase(s, "href");
        idxArea = CStrFun::FindCase(s, "<area");
        idxImg = CStrFun::FindCase(s, "<img");

        pre_idx = idxHref > idxArea? idxArea: idxHref;
        pre_idx = idxImg > pre_idx? pre_idx: idxImg;
        if( pre_idx == string::npos) break;

        s = s.substr(pre_idx);
        idx = s.find_first_of('<',1);
        if( idx != string::npos ){
            dest = dest + s.substr(0,idx);
        }else{
            break;
        }

        s = s.substr(idx);
        idxHref=0; idxArea=0; idxImg=0;
    }while(1);

    s = dest;

    
    /* erase all '\' character
     * too avoid the following situations:
     *      document.write("<A href=\"/~webg/refpaper/index.html\">t2</A>");
    */
    CStrFun::EraseStr(s, "\\");

    if( s.size() < 20 ) return false;

    return true;
}

/*****************************************************************
** Function name: GetLinkInfo4SE()
** Input argv:
**      --  
** Output argv:
**      --
** Return:
       true: success
       false: fail
** Function Description:  Get links for SE
** Version: 1.0
** Be careful:
*****************************************************************/

//再从m_sContentLinkInfo提取出为搜索引擎准备的超链接
bool CPage::GetLinkInfo4SE()
{

    if( m_sContentLinkInfo.empty() ) return false;

    m_sLinkInfo4SE = m_sContentLinkInfo;
    string& s = m_sLinkInfo4SE;

     // Keep only <area ...>,and <a href ...> tags.
    string::size_type idxHref=0,idxArea=0,
        idx,pre_idx;
    string dest;








    /*

    例如:上面的m_sContentLinkInfo=href="http://www.baidu.com/">百度href="http://www.qq.com/">QQ<img src="http://www.google.com.hk">

    我们这里提取出href="http://www.baidu.com/">百度href="http://www.qq.com/">QQ    过滤掉<img src="http://www.google.com.hk">

    因为<img src="http://www.google.com.hk">的超链接是为历史网页存档准备的超链接

    */


    do{
        if( s.empty() ) break;

        //idxHref = CStrFun::FindCase(s, "<a href");
        idxHref = CStrFun::FindCase(s, "href");
        idxArea = CStrFun::FindCase(s, "<area ");

        pre_idx = idxHref > idxArea? idxArea: idxHref;
        //pre_idx = idxHref;
        if( pre_idx == string::npos) break;//终止条件

        s = s.substr(pre_idx);
        idx = s.find_first_of('<',1);

        if( !(s.length() < 4) )
        {
            idxHref = CStrFun::FindCaseFrom(s, "href", 4);
            idx = idx > idxHref ? idxHref: idx;
        }

        if( idx != string::npos ){
            dest = dest + s.substr(0,idx);
        }else if (idx == string::npos && pre_idx != string::npos){
            dest = dest + s;
            break;
        }else{
            break;
        }

        s = s.substr(idx);
        idxHref=0; idxArea=0;
    }while(1);
        
    s = dest;//dest保存着过滤后的数据
    if( s.length() < 20 ) return false;


    // erase all '"' , '\'', "&nbsp;".
    CStrFun::EraseStr(s, "\"");
    CStrFun::EraseStr(s, "'");
    CStrFun::EraseStr(s, "&nbsp");

     // Keep URLs and anchor text.

    idxHref=0;
    const string delims( " #>");
    dest.clear();



    /*

    通过上面的提取我们得到href="http://www.baidu.com/">百度href="http://www.qq.com/">QQ

    我们再次提取

    m_sLinkInfo4SE="http://www.baidu.com/">百度"http://www.qq.com/">QQ

    */



    do{
        if( s.empty() ) break;
        idxHref = CStrFun::FindCase(s, "href");

        if( idxHref == string::npos) break;
        pre_idx = idxHref;

        //####
        idx = s.find('=', idxHref);
        if( idx == string::npos ) break;
        s = s.substr(idx+1);

        while( s.length() > 0 && s[0] == ' ' ){
            s.erase(0,1);
        }
        if( s.length() == 0 ) break;

        idx = s.find_first_of(delims,1);
        //cout << endl << s.substr(0, idx) << endl;
        if( idx == string::npos ) break;

        dest += '"' + s.substr(0, idx);

        //cout << endl << dest << endl;
            
        idx = s.find('>');
        if( idx == string::npos ) break;
        dest += '>';
        s = s.substr(idx +1);
            
        idx = s.find('<');

        if( !s.empty() ){
            idxHref = CStrFun::FindCase(s, "href");
            idx = idx > idxHref ? idxHref: idx;
        }    

        if( idx == string::npos ){
            dest += s;
            break;
        }

/*
        if( idx == idxHref ){
            dest += '"' + s.substr(0,idx);
        }else{
*/
            dest += s.substr(0,idx);
        //}
        //####

        idxHref=0;
    }while(1);
        
    // look for empty filenames.
    idx = 0;
    while( (idx = dest.find("\"\"",idx)) != string::npos ){
        dest.erase(idx, 1);
    }

    s = dest;

    return( s.length() < 20 ? false: true );

}
                    
/*****************************************************************
** Function name: GetLinkInfo4History()
** Input argv:
**      --  
** Output argv:
**      --
** Return:
       true: success
       false: fail
** Function Description:  Get links for history archiving
** Version: 1.0
** Be careful:
*****************************************************************/
bool CPage::GetLinkInfo4History()
{
    /*

    例如:上面的m_sContentLinkInfo=href="http://www.baidu.com/">百度href="http://www.qq.com/">QQ<img src="http://www.google.com.hk">

    我们这里提取出<img src="http://www.google.com.hk">   过滤掉href="http://www.baidu.com/">百度href="http://www.qq.com/">QQ

    因为href="http://www.baidu.com/">百度href="http://www.qq.com/">QQ的超链接是为搜索引擎准备的超链接

    */






    if( m_sContentLinkInfo.empty() ) return false;

    m_sLinkInfo4History = m_sContentLinkInfo;
    string& s = this->m_sLinkInfo4History;

     // Keep only <img ...> tags.
    string::size_type idxImg=0,
        idx,pre_idx;
    string dest;

    do{
        if( s.empty() ) break;
        idxImg = CStrFun::FindCase(s, "<img");

        pre_idx = idxImg;
        if( pre_idx == string::npos) break;

        s = s.substr(pre_idx);
        idx = s.find_first_of('<',1);

        if( idx != string::npos ){
            dest = dest + s.substr(0,idx);
        }else if (idx == string::npos && pre_idx != string::npos){
            dest = dest + s;
            break;
        }else{
            break;
        }

        s = s.substr(idx);
        idxImg=0;
    }while(1);
        
    s = dest;
    if( s.length() < 20 ) return false;

    // erase all '"'. '\'',"&nbsp;".
    CStrFun::EraseStr(s , "\"");
    CStrFun::EraseStr(s , "'");
    CStrFun::EraseStr(s , "&nbsp");

     // Keep URLs and anchor text.

    idxImg=0;
    string::size_type idxSrc = 0;
    const string delims( " #>");
    dest.clear();


    /*

通过上面的提取我们得到<img src="http://www.google.com.hk">

    我们再次提取

    m_sLinkInfo4History="http://www.google.com.hk>

*/

    do{
        if( s.empty() ) break;
        idxImg = CStrFun::FindCase(s, "img");

        if( idxImg == string::npos) break;
        pre_idx = idxImg;

        s = s.substr(idxImg+3);        // skip "img"

        //####
        idx = s.find('>', idxImg);
        if( idxImg == string::npos) break;
        if( s.empty() ) break;
        idxSrc = CStrFun::FindCase(s, "src");
        if( idxSrc > idxImg ) continue;
        s = s.substr(idxSrc);

        idx = s.find('=', idxImg);
        if( idx == string::npos ) break;
        s = s.substr(idx+1);

        while( s.length() > 0 && s[0] == ' ' ){
            s.erase(0,1);
        }
        if( s.length() == 0 ) break;

        idx = s.find_first_of(delims,1);
        if( idx == string::npos ) break;

        if( s.at(0) == '"'){
            dest += s.substr(0, idx);
        }else{
            dest += '"' + s.substr(0, idx);
        }
            
        idx = s.find('>');
        if( idx == string::npos ) break;
        dest += '>';
        s = s.substr(idx +1);
            
        idx = s.find('<');
        if( idx == string::npos ){
            dest += s;
            break;
        }
        dest += s.substr(0,idx);
        //####

        idxImg=0;
    }while(1);
        

    // look for empty filenames.
    idx = 0;
    while( (idx = dest.find("\"\"",idx)) != string::npos ){
        dest.erase(idx, 1);
    }

    s = dest;

    return( s.length() < 20 ? false: true );

}




//判断strUrl是不是正规的url
bool CPage::NormalizeUrl(string& strUrl)
{
    string::size_type idx;


    //URL没有htp://协议名我们这里认为strUrl不是正规的URL
    if( CStrFun::FindCase(strUrl, "http://") == string::npos ) return false;

    // convert "http://e.pku.cn" to "http://e.pku.cn/"
    //将http://www.baidu.com转化为http://www.baidu.com/
    idx = strUrl.rfind('/');
    if( idx < 8 ) {
        strUrl = strUrl + "/";
        return true;
    }

    //将"/./"-->"/"
    while( (idx=strUrl.find("/./")) != string::npos ){
        if( idx != string::npos ) strUrl.erase(idx,2);
    }

    //将"xxx/x/../yyy"-->xxx/yyy
    while( (idx = strUrl.find("/../")) != string::npos ){
        string strPre,strSuf;

        strPre = strUrl.substr(0, idx);

        if( strUrl.length() > idx+4 )
            strSuf = strUrl.substr(idx+4);

        idx = strPre.rfind("/");
        if( idx != string::npos)
            strPre = strPre.substr(0,idx+1);
        if( strPre.length() < 10 ) return false;

        strUrl = strPre + strSuf;
    }

    if( CStrFun::FindCase(strUrl, "http://") != 0 ) return false;

    return true;
}







/*最终得到为搜索引擎准备的超链接

  并将相对路径的URL和绝对路径的URL分别处理，同时，我们发现从一个网页中提取的超链接可以是相同的，这个时候

  我们必须去重，这个函数用map容器很好的做到了这一点

  还有一些URL不是正规的URL也要过滤

  还有一些URL是必要过滤也要过滤--通过IsFilterLink(string strUrl)实现

 */
bool CPage::FindRefLink4SE()
{
    if( m_sLinkInfo4SE.empty() ) return false;

    char *buffer = (char*)m_sLinkInfo4SE.c_str();
    int urlnum=0,len;
    char *ptr ;

    static char buf[URL_REFERENCE_LEN];

    memset(buf, 0, URL_REFERENCE_LEN);
    len = strlen(buffer);
    if( len < 8 ) return false;

    len = len < URL_REFERENCE_LEN -1 ? len : URL_REFERENCE_LEN - 1;//len记录相对较小的值
    strncpy( buf, buffer, len);

/*first
 *------>
 */


    /*

    例如:m_sLinkInfo4SE="http://www.baidu.com/">百度"http://www.qq.com/">QQ

    我们这里提取为

    http://www.baidu.com  百度

    http://www.qqq.com    QQ

    */
    ptr = buf;
    while( ptr - buf < len  && *ptr )
    {
        while( *ptr == '"' && *ptr) ptr++;
        if ( !*ptr ) break;
        this->m_RefLink4SE[ urlnum].link = ptr;//每个网页里最多有1000个链接
        while( *ptr && *ptr != '>')
        {
            //在遇到'>'之前，出现了' '字符,我们必须将' '字符赋值为'\0'说明URL提取完了,因为URL不可能出现' '字符
            if(*ptr == ' ') *ptr = '\0';
            //例如: "http://www.baidu.com/" height=100 width=150>百度   出现空格说明还有其他的属性值
            ptr++;
        }

        if ( !*ptr ){
            urlnum++;
            break;
        }
        if ( *ptr == '>' )
        {
            *ptr++='\0';
            if( !*ptr )
            {
                urlnum++;
                break;
            }
            
            if( *ptr == '"' )
            {
                this->m_RefLink4SE[urlnum].anchor_text = NULL;
            }
            else
            {
                this->m_RefLink4SE[urlnum].anchor_text = ptr;
                while( *ptr && *ptr != '"') ptr++;
                if (!*ptr)
                {
                    urlnum++;
                    break;
                }
                if ( *ptr == '"') *ptr='\0';
            }

        }
        
        //cout << endl << this->m_RefLink4SE[ urlnum].link << '\t';
        //cout << this->m_RefLink4SE[ urlnum].anchor_text << endl;

        ptr++;
        urlnum++;
        if ( urlnum == MAX_URL_REFERENCES) break; //达到最多的url数目
    }
    //cout << endl << this->m_RefLink4SE[ urlnum].link << endl;
    //cout << this->m_RefLink4SE[ urlnum].anchor_text << endl;

    this->m_nRefLink4SENum = urlnum;

/*second
 *------>
 */
    //typedef map<string,string,less<string> >::value_type valType;
    typedef map<string,string>::value_type valType;

    m_mapLink4SE.clear();

    //string strRootUrl= m_sUrl;
    CUrl iUrl;
    if( iUrl.ParseUrlEx(m_sUrl) == false )
    {
        cout << "ParseUrlEx error in FindRefLink4SE(): " << m_sUrl << endl;
        return false;
    }
    
    for(int i=0; i<m_nRefLink4SENum; i++)
    {

        string str;
        string::size_type idx;
        const string delims(" #");

        str = m_RefLink4SE[i].link;
        idx = str.find_first_of(delims, 0 );
        if( idx != string::npos )//如果找到标志
        {
            str = str.substr(0, idx);//只取#前边的url
        }
        if( str.size() == 0 || str.size() > URL_LEN - 1 || str.size() < 4 ) 
            continue;


        string::size_type idx1;
        idx1 = CStrFun::FindCase(str, "http");
        if( idx1 != 0  )//str有可能是相对路径
        {
            char c1 = m_sUrl.at(m_sUrl.length()-1);
            char c2 = str.at(0);

            if( c2=='/' )//str一定是相对路径
            {
                if( iUrl.m_nPort != 80 )//若是http
                {
                    cout << iUrl.m_sHost << endl;
                    cout << str << endl;
                    //str = "http://" + iUrl.m_sHost + ":" + (const char*)(iUrl.m_nPort) + str;
                    str = "http://" + iUrl.m_sHost + ":" + CStrFun::itos(iUrl.m_nPort) + str;
                } 
                else 
                {
                    str = "http://" + iUrl.m_sHost + str;
                }
            } 
            else if( c1!='/' && c2!='/')//若两个都不是,则加上/构成新的url
            {
                string::size_type idx;

                idx = m_sUrl.rfind('/');
                if( idx != string::npos )//若不是最后
                {
                    if( idx > 6 )
                    { // > strlen("http://..")
                        str = m_sUrl.substr(0, idx+1) + str;
                    } 
                    else 
                    {
                        str = m_sUrl + "/" + str;
                    }

                } else {

                    continue;
                }

            } 
            else 
            {
                if( c1=='/' )
                {
                    str = m_sUrl + str;
                }
                else 
                {
                    str = m_sUrl + "/" + str;
                }
            }
        }

        if( NormalizeUrl(str) == false ) continue;

        if( IsFilterLink(str) ) continue;

        //debug
        //cout << "reflink: " << reflink << endl;

        if( str == m_sUrl )//一个网页中提取的超链接是其本身，我就不要了，因为我们已经有了这个网页的URL了
        {
            continue;
        }
        else
        {
            if( m_RefLink4SE[i].anchor_text )//有URL的描述符
            {
                if( m_mapLink4SE.find(str) == m_mapLink4SE.end() )
                {
                    m_mapLink4SE.insert( valType( str, m_RefLink4SE[i].anchor_text));
                }
            }
            else//没有URL的描述符---这个时候描述符为'\0'
            {
                if( m_mapLink4SE.find(str) == m_mapLink4SE.end() )
                {
                    m_mapLink4SE.insert( valType( str, "\0") );
                    cout << ".";
                }
            }
        }
            

    }

    m_nRefLink4SENum = m_mapLink4SE.size();

    //cout << endl;

    return true;
}





//最终得到为历史网页存档准备的超链接

//并将相对路径的URL和绝对路径的URL分别处理，同时，我们发现从一个网页中提取的超链接可以是相同的，这个时候

//我们必须去重，这个函数用vector容器很好的做到了这一点

//还有一些URL不是正规的URL也要过滤

//还有一些URL是必要过滤也要过滤--通过IsFilterLink(string strUrl)实现
bool CPage::FindRefLink4History()
{
    if( m_sLinkInfo4History.empty() ) return false;

    char *buffer = (char*)m_sLinkInfo4History.c_str();
    int urlnum=0,len;
    char *ptr ;

    static char buf[URL_REFERENCE_LEN/2];

    memset(buf, 0, URL_REFERENCE_LEN/2);
    len = strlen(buffer);
    if( len < 8 ) return false;

    len = len < URL_REFERENCE_LEN/2 - 1? len : URL_REFERENCE_LEN/2 -1;
    strncpy( buf, buffer, len);

/*first
 *------>
 */
    ptr = buf;
    while( ptr - buf < len  && *ptr ){
        while( *ptr == '"' && *ptr) ptr++;
        if ( !*ptr ) break;
        this->m_RefLink4History[ urlnum].link = ptr;

        while( *ptr && *ptr != '>'){
            if( *ptr == ' ') *ptr='\0';
            ptr++;
        }

        if( !*ptr){
            urlnum++;
            break;
        }
        if( *ptr == '>' ){
            *ptr++ = 0;
            if( !*ptr ){
                urlnum++;
                break;
            }
            if( *ptr == '"' ){
            
            }else{
                while( *ptr && *ptr != '"') ptr++;
                if( !*ptr ){
                    urlnum++;
                    break;
                }
                if ( *ptr == '"' ) *ptr++='\0';
            }
        }
        
        ptr++;
        urlnum++;
        if ( urlnum == MAX_URL_REFERENCES/2) break;
    }


    this->m_nRefLink4HistoryNum = urlnum;

/*second
 *------>
 */
    m_vecLink4History.clear();
    //string strRootUrl= m_sUrl;
        CUrl iUrl;
        if( iUrl.ParseUrlEx(m_sUrl) == false ){
        cout << "ParseUrlEx error in FindRefLink4History(): " << m_sUrl << endl;
        return false;
    }

    for(int i=0; i<m_nRefLink4HistoryNum; i++){
        string str;
        //string::size_type idx;

        str = m_RefLink4History[i].link;
        if( str.size()==0 || str.size() > URL_LEN - 1 
            || str.size() < 4 ) continue;

/*
        char *pdest1, *pdest2;
        pdest1 = strstr( str.c_str(), "http" );
        pdest2 = strstr( str.c_str(), "HTTP" );
        if( pdest1==NULL && pdest2==NULL ){
*/

        string::size_type idx1;
        idx1 = CStrFun::FindCase(str, "http");
        if( idx1 != 0 ){
            char c1 = m_sUrl.at(m_sUrl.length()-1);
            char c2 = str.at(0);

            if( c2=='/' ){
                if( iUrl.m_nPort != 80 ){
                    str = "http://" + iUrl.m_sHost + ":" + CStrFun::itos(iUrl.m_nPort) + str;
                } else {
                    str = "http://" + iUrl.m_sHost + str;
                }
            } else if( c1!='/' && c2!='/'){
                string::size_type idx;

                idx = m_sUrl.rfind('/');
                if( idx != string::npos ){
                    if( idx > 6 ){ // > strlen("http://..")
                        str = m_sUrl.substr(0, idx+1) + str;
                    } else {
                        str = m_sUrl + "/" + str;
                    }

                } else {

                    continue;
                }

            } else {
                if( c1=='/' ){
                    str = m_sUrl + str;
                } else {
                    str = m_sUrl + "/" + str;
                }
            }
        }

        // due to bad link parser
/*

        idx = reflink.find(' ');
        if(idx != string::npos){
            reflink = reflink.substr(0,idx);
        }
        idx = reflink.find('"');
        if(idx != string::npos){
            reflink = reflink.substr(0,idx);
        }
*/
        //#############

        if( NormalizeUrl(str) == false ) continue;


        if( IsFilterLink(str) ) continue;


        if( str == m_sUrl ){
            continue;
        }else{
            vector<string>::iterator it;
            it = find(m_vecLink4History.begin(), m_vecLink4History.end(),str);
            if( it == m_vecLink4History.end() ){

                m_vecLink4History.push_back( str);
                cout << ".";
            }
        }
            

    }
    m_nRefLink4HistoryNum = m_vecLink4History.size();
    //cout << endl;

    return true;
}