c++ 爬虫

这是一个简单的c++爬虫，效率并不是很高...

#include<stdio.h>
int s1[1000000],s2[1000000];
void fun(int a, int b)
{
    int i,ii;
    bool t1,t2,t3,t4;
    s1[0] = s2[0] = s1[1] = s2[1] = 0;
    for(i=a; i <= b; i++){
        ii = i;
        t1 = t2 = t3 = t4 =false;
        while(ii != 0 ){
            int a = ii %10;
            if( a == 5)
            {
                t1 = true;
            }
            else if( a == 2)
            {
                t2 = true;
            }
            else if( a == 1)
            {
                t3 = true;
            }
            ii = ii / 10;
        }
        if(t1 && t2 && t3){
            s1[i-1] = s1[i-2] + 1;
            ii = i;
        while(ii != 0 ){
            int a = ii % 10;
            int b = (ii / 10) % 10;
            int c = (ii / 100) % 10;
            if( c > 0 && a == 1 && b == 2 && c ==5)
                t4 = true;
            ii = ii / 10;
        }
        if(t4)
            s2[i-1] = s2[i-2] + 1;
        else 
            s2[i-1] = s2[i-2];
        }
        else{
            s2[i-1] = s2[i-2];
            s1[i-1] = s1[i-2];
        }
    }
}

int main()
{
    int a,b,i=1;
    fun(2,1000000);
    while(scanf("%d%d",&a,&b) != EOF){
        if(a == 1)
            printf("Case %d:%d %d
",i,s1[b-1]-s1[a-1],s2[b-1]-s2[a-1]);
        else
            printf("Case %d:%d %d
",i,s1[b-1]-s1[a-2],s2[b-1]-s2[a-2]);
        i++;
    }
    return 0;
}

#include"urlThread.h"
#include<QFile>
#include<QMessageBox>
#include<QTextStream>
#include <QMainWindow>
void urlThread::run()
{
    open();
}

void urlThread::startThread()
{
    start();
}

//显示找到的url
void urlThread::open()
{
    QString path = "url.txt";
    QFile file(path);
    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
       // QMessageBox::warning(this,tr("Read File"),
        //                     tr("Cannot open file:
%1").arg(path));
        send("error!cannot open url.txt!");
        return;
    }
    QTextStream in(&file);
    while(in.readLine().compare("") != 0){
        //ui->textBrowser->append(in.readLine());
        send(q2s(in.readLine()));
        Sleep(1);
    }
    file.close();
}

#include "mainwindow.h"
#include <QApplication>

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    MainWindow w;
    w.setWindowTitle("小小爬虫");
    w.show();

    return a.exec();
}

#include "mainwindow.h"
#include "ui_mainwindow.h"

MainWindow::MainWindow(QWidget *parent) :
    QMainWindow(parent),
    ui(new Ui::MainWindow)
{
    ui->setupUi(this);
    QObject::connect(ui->start,SIGNAL(released()),this,SLOT(beginGeturl()));
    //QObject::connect(ui->display,SIGNAL(released()),this,SLOT(open()));
    QObject::connect(ui->display,SIGNAL(released()),&uth,SLOT(startThread()));
    QObject::connect(&uth,&urlThread::sendMessage,this,&MainWindow::receiveMessage);
    QObject::connect(&crawler,&Crawler::sendMessage,this,&MainWindow::receiveMessage);
}

MainWindow::~MainWindow()
{
    delete ui;
}

void MainWindow::receiveMessage(const QString name)
{
    ui->textBrowser->append(name);
    ui->textBrowser->moveCursor(QTextCursor::End);
}

void MainWindow::open()
{
    QString path = "url.txt";
    QFile file(path);
    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
        QMessageBox::warning(this,tr("Read File"),
                             tr("Cannot open file:
%1").arg(path));
        return;
    }
    QTextStream in(&file);
    while(in.readLine().compare("") != 0){
        //ui->textBrowser->append(in.readLine());
        crawler.send(q2s(in.readLine()));
    }
    file.close();
}

void MainWindow::beginGeturl()
{
    //crawler = new Crawler();
    string url = "" ,dep, filter = "www";
    if(!ui->site->text().isEmpty())
        url = q2s(ui->site->text());
    crawler.addURL(url);
    int depth = 1;
    if(!ui->depth->text().isEmpty())
    {
        url = q2s(ui->depth->text());
        depth = atoi(url.c_str());
    }
    if(!ui->filter->text().isEmpty())
        filter = q2s(ui->filter->text());
    crawler.setJdugeDomain(filter);
    crawler.setDepth(depth);
    crawler.startThread();
}

#ifndef CRAWLER_H
#define CRAWLER_H

#include<set>
#include<string>
#include<queue>
#include "winsock2.h"
#include <iostream>
#include <fstream>
#include <stdio.h>
#include<time.h>
#include<winsock.h>
#include<QThread>

#pragma comment(lib, "ws2_32.lib")
using namespace std;

bool ParseURL(const string & url, string & host, string & resource);
bool GetHttpResponse(const string & url, char * &response, int &bytesRead);
QString s2q(const string &s);
string q2s(const QString &s);

#define DEFAULT_PAGE_BUF_SIZE 1000000

class Crawler: public QThread
{
    Q_OBJECT
private:
    queue<string> urlWaiting;
    set<string> urlWaitset;
    set<string> urlProcessed;
    set<string> urlError;
    set<string> disallow;
    set<string>::iterator it;
    int numFindUrl;
    time_t starttime, finish;
    string filter;
    int depth;

public:
    Crawler(){  filter = "";numFindUrl = 0;}
    ~Crawler(){}
    void begin();
    void setDepth(int depth);
    void processURL(string& strUrl);
    void addURL(string url);
    void log(string entry, int num);
    void HTMLParse(string & htmlResponse, const string & host);
    bool getRobotx(const string & url, char * &response, int &bytesRead);
    void setJdugeDomain(const string domain);
    long urlOtherWebsite(string url);
    void send(string s)
    {
        QString qs = s2q(s);
        emit sendMessage(qs);
    }
signals:
    void sendMessage(const QString name);

public slots:
        bool startThread();

protected:
    void run();

};
#endif // CRAWLER_H

#ifndef MAINWINDOW_H
#define MAINWINDOW_H

#include <QMainWindow>
#include<QFile>
#include<QMessageBox>
#include<QTextStream>
#include<QLineEdit>
#include<QDebug>
#include"crawler.h"
#include"urlThread.h"

namespace Ui {
class MainWindow;
}

class MainWindow : public QMainWindow
{
    Q_OBJECT

public:
    explicit MainWindow(QWidget *parent = 0);
    ~MainWindow();
    void receiveMessage(const QString name);

public slots:
        void beginGeturl();
        void open();

private:
    Ui::MainWindow *ui;
    Crawler crawler;
    urlThread uth;
};

#endif // MAINWINDOW_H

#ifndef URLTHREAD_H
#define URLTHREAD_H
#include"crawler.h"

class urlThread: public QThread
{
    Q_OBJECT
public slots:
        void startThread();
        void open();
        void send(string s)
        {
            QString qs = s2q(s);
            emit sendMessage(qs);
        }
signals:
        void sendMessage(const QString name);
protected:
        void run();
};

#endif // URLTHREAD_H

<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
 <class>MainWindow</class>
 <widget class="QMainWindow" name="MainWindow">
  <property name="geometry">
   <rect>
    <x>0</x>
    <y>0</y>
    <width>706</width>
    <height>381</height>
   </rect>
  </property>
  <property name="windowTitle">
   <string>MainWindow</string>
  </property>
  <widget class="QWidget" name="centralWidget">
   <widget class="QLabel" name="label">
    <property name="geometry">
     <rect>
      <x>10</x>
      <y>20</y>
      <width>54</width>
      <height>21</height>
     </rect>
    </property>
    <property name="text">
     <string>初始网址：</string>
    </property>
   </widget>
   <widget class="QLineEdit" name="site">
    <property name="geometry">
     <rect>
      <x>70</x>
      <y>20</y>
      <width>151</width>
      <height>20</height>
     </rect>
    </property>
    <property name="text">
     <string>http://www.scut.edu.cn</string>
    </property>
   </widget>
   <widget class="QLabel" name="label_2">
    <property name="geometry">
     <rect>
      <x>230</x>
      <y>20</y>
      <width>54</width>
      <height>21</height>
     </rect>
    </property>
    <property name="text">
     <string>搜索深度：</string>
    </property>
   </widget>
   <widget class="QLineEdit" name="depth">
    <property name="geometry">
     <rect>
      <x>290</x>
      <y>20</y>
      <width>51</width>
      <height>20</height>
     </rect>
    </property>
    <property name="text">
     <string>2</string>
    </property>
   </widget>
   <widget class="QLabel" name="label_3">
    <property name="geometry">
     <rect>
      <x>360</x>
      <y>20</y>
      <width>71</width>
      <height>21</height>
     </rect>
    </property>
    <property name="text">
     <string>过滤字符串：</string>
    </property>
   </widget>
   <widget class="QLineEdit" name="filter">
    <property name="geometry">
     <rect>
      <x>430</x>
      <y>20</y>
      <width>71</width>
      <height>20</height>
     </rect>
    </property>
    <property name="text">
     <string>scut</string>
    </property>
   </widget>
   <widget class="QPushButton" name="start">
    <property name="geometry">
     <rect>
      <x>510</x>
      <y>20</y>
      <width>51</width>
      <height>23</height>
     </rect>
    </property>
    <property name="text">
     <string>开始</string>
    </property>
   </widget>
   <widget class="QPushButton" name="display">
    <property name="geometry">
     <rect>
      <x>570</x>
      <y>20</y>
      <width>61</width>
      <height>23</height>
     </rect>
    </property>
    <property name="text">
     <string>显示url</string>
    </property>
   </widget>
   <widget class="QTextBrowser" name="textBrowser">
    <property name="geometry">
     <rect>
      <x>10</x>
      <y>60</y>
      <width>671</width>
      <height>271</height>
     </rect>
    </property>
   </widget>
  </widget>
  <widget class="QMenuBar" name="menuBar">
   <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
     <width>706</width>
     <height>23</height>
    </rect>
   </property>
  </widget>
  <widget class="QToolBar" name="mainToolBar">
   <attribute name="toolBarArea">
    <enum>TopToolBarArea</enum>
   </attribute>
   <attribute name="toolBarBreak">
    <bool>false</bool>
   </attribute>
  </widget>
  <widget class="QStatusBar" name="statusBar"/>
 </widget>
 <layoutdefault spacing="6" margin="11"/>
 <resources/>
 <connections/>
</ui>

ui文件要自己设计

爬虫还有一些小问题，抓取的url content并不完全，某些地址还有一点小问题...

貌似博客园没有上传附件的地方？还是我没找到？希望得到提示~