广州楼盘抓取分析-分析问题

上文其实还是有不少问题的。

1.顺序执行，效率比较慢；2.不能断点执行。

那么，解决办法是什么呢？

对于问题1，可以采用生产者消费者模式来改写，代码如下

# -*- coding: utf-8 -*-
#######################################################################
# Copyright (C) 2005-2016 UC Mobile Limited. All Rights Reserved
# File          : first_sale_spider.py
#
# Creation      : 2016/2/23 19:41
# Author        : shufeng.lsf@ucweb.com
#######################################################################
import random
from threading import Thread

import requests
import re

import time
from pyquery import PyQuery as pq
from Queue import Queue
import MySQLdb
import uniout
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

community_list = []

HOST = "127.0.0.1"
USER = "root"
PASSWD = ""
DB = "house_analysis"
PORT = 3306

queue = Queue(10)

class DBOperate(object):
    def __init__(self, host, user, passwd, db, port, charset="utf8"):
        self.host = host
        self.user = user
        self.passwd = passwd
        self.db = db
        self.port = port
        self.conn = MySQLdb.connect(self.host, self.user, self.passwd, self.db, self.port, charset="utf8")
        self.cur = self.conn.cursor()

    def insertSql(self,sql):
        self.cur.execute(sql)
        self.conn.commit()

    def __del__(self):
        self.cur.close()
        self.conn.close()


def requestByGet(url):
    r = requests.get(url)
    return r.content

def getNextPage(content):
    m = re.findall(r'<a href="(.+?)" class="next-page next-link">下一页</a>',content)
    if len(m)>0:
        next_url = m[0]
    else:
        next_url = ''
    return next_url

def getCommunityList(content):
    community_urls = re.findall(r'data-link="(http://gz.fang.anjuke.com/loupan/d+?.html)"',content)
    print "正在采集...",community_urls
    if len(community_urls)>0:
        return community_urls


def getHouseInfo(url):
    p = pq(url)
    name = p('h1').text().strip()
    style = p('.house-item').text().split(",")[0].strip()
    price = p('.sp-price').text().strip()
    l = p('.lpAddr-text').text()
    location = re.split('[ | ]',l)
    area = location[-2].split('-')[0].strip()
    zone = location[-2].split('-')[1].strip()
    address = location[-1].strip()
    detail_location = location[-1].strip()
    result = {
        "name": name,
        "area": area,
        "location": zone,
        "detail_location": detail_location,
        "house_style": style,
        "price": price
    }
    return result


def detailPageHandler(cur, detail_url):
    result = getHouseInfo(detail_url)
    print "result:",result
    cur.insertSql("insert into first_sale (name,area,location,detail_location,house_style,price) VALUES('%s','%s','%s','%s','%s','%s')" % (
        result['name'],
        result['area'],
        result['location'],
        result['detail_location'],
        result['house_style'],
        result['price']
    ))

class UrlProducer(Thread):
    def __init__(self, start_url):
        Thread.__init__(self)
        self.start_url = start_url

    def run(self):
        global queue
        while True:
            content = requestByGet(self.start_url)
            next_url = getNextPage(content)
            community_urls = getCommunityList(content)
            for url in community_urls:
                queue.put(url)
                time.sleep(random.random())
                print "进入队列的url:",url
            if next_url != '':
                self.start_url = next_url
                continue
            else:
                break

class GetHouseInfo(Thread):
    def __init__(self, cur):
        Thread.__init__(self)
        self.cur = cur

    def run(self):
        global queue
        while True:
            url = queue.get()
            detailPageHandler(self.cur, url)
            queue.task_done()
            time.sleep(random.random())
            print "处理完毕的url:", url


def main():
    cur = DBOperate(host=HOST, user=USER, passwd=PASSWD, db=DB, port=PORT)
    UrlProducer("http://gz.fang.anjuke.com/loupan/?from=navigation").start()
    GetHouseInfo(cur).start()


if __name__ == '__main__':
    main()

2.对于不能断点执行的问题，可以用异常捕获的方式将当前执行的url保存下来，下次直接从文件中读取执行即可。