爬取58二手数据.py

#第一个模块 抓取所有频道链接
from bs4 import BeautifulSoup
import requests


start_url = 'http://bj.58.com/sale.shtml'
url_host = 'http://bj.58.com'

def get_index_url(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    links = soup.select('ul.ym-submnu > li > b > a')
    for link in links:
        page_url = url_host + link.get('href')
        print(page_url)

get_index_url(start_url)


#第二个模块 抓取所有商品链接和详情数据

from bs4 import BeautifulSoup
import requests
import time
import pymongo

client = pymongo.MongoClient('localhost', 27017)
ceshi = client['ceshi']
url_list = ceshi['url_list4']
item_info = ceshi['item_info4']



# 在最左边是在python 中对象的名称，后面的是在数据库中的名称
# spider 1
def get_links_from(channel, pages):
    # td.t 没有这个就终止
    list_view = '{}/pn{}/'.format(channel, str(pages))
    wb_data = requests.get(list_view)
    time.sleep(1)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    if soup.find('td', 't'):
        for link in soup.select('td.t a.t'):
            item_link = link.get('href').split('?')[0]
            if item_link != 'http://jump.zhineng.58.com/jump':
                url_list.insert({'url':item_link})
                print(item_link)
            # return urls
    else:
        # It's the last page !
        pass

# spider 2
def get_item_info(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    if url[:25] == 'http://zhuanzhuan.58.com/':
        data={
            'title':soup.title.text,
            'price': soup.select('span.price_now')[0].text,
            'area':soup.select('div.palce_li > span > i')[0].text,
            'url':url
            }
        item_info.insert(data)
    else:
        data={
            'title': soup.title.text,
            'price':soup.select('span.price.c_f50')[0].text,
            'area':soup.select('div.su_con > a ')[0].get_text(),
            'sale_man':soup.select('ul.vcard > li > a ')[0].text,
            'url':url
            }
        item_info.insert(data)


#第三个模块 主文件运行开始抓取
from multiprocessing import Pool
from pages_parsing import get_item_info,url_list,item_info,get_links_from
from channel_extact import channel_list


item_url = (item['url'] for item in url_list.find())
index_urls0 = (item['url'] for item in item_info.find())
x = set(item_url)
y = set(index_urls0)
rest_of_urls = x-y

def get_all_links_from(channel):
    for i in range(1,100):
        get_links_from(channel,i)
    return rest_of_urls

if __name__ == '__main__':
    pool = Pool()
    # pool = Pool(processes=6)
    #pool.map(get_all_links_from,channel_list.split())
    pool.map(get_item_info,rest_of_urls)

# count = 0
# for url in rest_of_urls:
#     print(url)
#     count += 1
# print(count)

#第四个模块 查看数据流
import time
from pages_parsing import url_list

while True:
    print(url_list.find().count())
    time.sleep(5)