《爬虫学习》（六）（爬取58同城）

1.获取大页面下各个分类的小URL合集

from bs4 import BeautifulSoup
import requests


start_url = 'http://bj.58.com/sale.shtml'
url_host = 'http://bj.58.com'

def get_index_url(url):
    # url = start_url
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    links = soup.select('ul.ym-submnu > li > b > a')
    for link in links:
        page_url = url_host + link.get('href')
        print(page_url)

get_index_url(start_url)

channel_list = '''
    http://bj.58.com/shouji/
    http://bj.58.com/shoujihao/
    http://bj.58.com/tongxunyw/
    http://bj.58.com/diannao/
    http://bj.58.com/bijiben/
    http://bj.58.com/pbdn/
    http://bj.58.com/diannaopeijian/
    http://bj.58.com/zhoubianshebei/
    http://bj.58.com/shuma/
    http://bj.58.com/shumaxiangji/
    http://bj.58.com/mpsanmpsi/
    http://bj.58.com/youxiji/
    http://bj.58.com/jiadian/
    http://bj.58.com/dianshiji/
    http://bj.58.com/ershoukongtiao/
    http://bj.58.com/xiyiji/
    http://bj.58.com/bingxiang/
    http://bj.58.com/binggui/
    http://bj.58.com/chuang/
    http://bj.58.com/ershoujiaju/
    http://bj.58.com/yingyou/
    http://bj.58.com/yingeryongpin/
    http://bj.58.com/muyingweiyang/
    http://bj.58.com/muyingtongchuang/
    http://bj.58.com/yunfuyongpin/
    http://bj.58.com/fushi/
    http://bj.58.com/nanzhuang/
    http://bj.58.com/fsxiemao/
    http://bj.58.com/xiangbao/
    http://bj.58.com/meirong/
    http://bj.58.com/yishu/
    http://bj.58.com/shufahuihua/
    http://bj.58.com/zhubaoshipin/
    http://bj.58.com/yuqi/
    http://bj.58.com/tushu/
    http://bj.58.com/tushubook/
    http://bj.58.com/wenti/
    http://bj.58.com/yundongfushi/
    http://bj.58.com/jianshenqixie/
    http://bj.58.com/huju/
    http://bj.58.com/qiulei/
    http://bj.58.com/yueqi/
    http://bj.58.com/bangongshebei/
    http://bj.58.com/diannaohaocai/
    http://bj.58.com/bangongjiaju/
    http://bj.58.com/ershoushebei/
    http://bj.58.com/danche/
    http://bj.58.com/fzixingche/
    http://bj.58.com/diandongche/
    http://bj.58.com/sanlunche/
    http://bj.58.com/peijianzhuangbei/
    http://bj.58.com/tiaozao/
'''

2.针对每一个小URL进行信息提取

from bs4 import BeautifulSoup
import requests
import time
import pymongo

client = pymongo.MongoClient('localhost', 27017)
ceshi = client['ceshi']
url_list = ceshi['url_list4']
item_info = ceshi['item_info4']


# 在最左边是在python 中对象的名称，后面的是在数据库中的名称
# spider 1
def get_links_from(channel, pages, who_sells=0):
    # td.t 没有这个就终止
    #https://bj.58.com/shouji/pn2/
    list_view = '{}{}/pn{}/'.format(channel, str(who_sells), str(pages))
    wb_data = requests.get(list_view)
    time.sleep(1)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    #if else 为了防止https://bj.58.com/shouji/pn100/这样不存在的页面
    if soup.find('td', 't'):
        for link in soup.select('td.t a.t'):
            item_link = link.get('href').split('?')[0]
            url_list.insert_one({'url': item_link})
            #读取商品信息并且存入数据库
            get_item_info(item_link)
            time.sleep(1)
            # return urls
    else:
        # It's the last page !
        pass

# spider 2   解析每一个URL
def get_item_info(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    #如果爬取URL时候还存在，但是get_item_info解析每一个URL时候恰好被买走了，那么就会出现404错误
    #分析404错误的页面的源代码发现有这么一句：
    #<link rel="stylesheet" type="text/css" href="https://c.58cdn.com.cn/ui6/list/404news_v20161103135554.css">
    #所以使用'404' in soup.find('link', type="text/css", rel="stylesheet").get('href').split('/')来判断
    no_longer_exist = '404' in soup.find('link', type="text/css", rel="stylesheet").get('href').split('/')
    if no_longer_exist:#存在404错误的话就pass
        pass
    else:
        # title = soup.title.text.split('-')[0]
        # # print(title)
        # #网页源代码中存在这样一句：<title>OPPOreno10倍变焦版 - 北京58同城</title>
        # price = soup.select('span.infocard__container__item__main__text--price')[0].text
        # #<span class="infocard__container__item__main__text--price"> 360元</span>
        # date = soup.select('span.detail-title__info__text')[0].text
        # #<div class="detail-title__info__text">2020-01-24 更新</div>
        # area = list(soup.select('.infocard__container__item__main a')[0].stripped_strings) if soup.find_all('span', 'infocard__container__item__main') else None
        # #<div class="infocard__container__item__main"><a href='/chaoyang/shouji/' target="_blank">朝阳</a></div>
        # #存入数据库
        # item_info.insert_one({'title': title, 'price': price, 'date': date, 'area': area, 'url': url})
        # print({'title': title, 'price': price, 'date': date, 'area': area, 'url': url})
        if ((soup.title.text.split('-')[0]=="请输入验证码 ws:36.161.10.181")|(soup.title.text.split('-')[0]=='【58同城 58.com】六安分类信息 ')):
            title = ""
        else:
            title = soup.title.text.split('-')[0]

        if soup.select('.infocard__container__item__main__text--price')!=[]:
            price = soup.select('.infocard__container__item__main__text--price')[0].get_text().strip()
        else:
            price = []
        # price = soup.select('.infocard__container__item__main__text--price')
        # print(price)
        if soup.select('.detail-title__info__text')!=[]:
            date = soup.select('.detail-title__info__text')[0].get_text().strip()
        else:
            date = []

        if soup.select('.infocard__container__item__main a')!=[]:
            area = soup.select('.infocard__container__item__main a')[0].get_text().strip()
        else:
            area = []
        #area 这里还有不完善的地方：需要判断如果area不存在的话应该设置为None
        # if soup.find_all('span', 'infocard__container__item__main') else None
        item_info.insert_one({'title': title, 'price': price, 'date': date, 'area': area, 'url': url})
        print({'title': title, 'price': price, 'date': date, 'area': area, 'url': url})

# get_links_from("http://bj.58.com/shouji/",2)　

不知道怎么破解58的验证码反爬机制......在知乎上听大佬说，好像sleep可以解决

3.进行主函数编写+爬取次数的统计

from multiprocessing import Pool
from channel_extact  import channel_list
from pages_parsing   import get_links_from
from pages_parsing   import get_item_info


def get_all_links_from(channel):
    for i in range(1,100):
        get_links_from(channel,i)


if __name__ == '__main__':
    #多线程pool = Pool()
    pool = Pool()
    # pool = Pool(processes=6)
    #map方法:map(一个函数，传入该函数的值)
    pool.map(get_all_links_from,channel_list.split())

import time
from pages_parsing import url_list

while True:
    print(url_list.find().count())
    time.sleep(4)
    #爬取1000条数据
    if url_list.find().count()==1000:
        break