python 火车票爬取代码

1、根据搜索词下载百度图片：

# -*- coding: utf-8 -*-
"""根据搜索词下载百度图片"""
import re
import sys
import urllib

import requests


def get_onepage_urls(onepageurl):
    """获取单个翻页的所有图片的urls+当前翻页的下一翻页的url"""
    if not onepageurl:
        print('已到最后一页, 结束')
        return [], ''
    try:
        html = requests.get(onepageurl)
        html.encoding = 'utf-8'
        html = html.text
    except Exception as e:
        print(e)
        pic_urls = []
        fanye_url = ''
        return pic_urls, fanye_url
    pic_urls = re.findall('"objURL":"(.*?)",', html, re.S)
    fanye_urls = re.findall(re.compile(r'<a href="(.*)" class="n">下一页</a>'), html, flags=0)
    fanye_url = 'http://image.baidu.com' + fanye_urls[0] if fanye_urls else ''
    return pic_urls, fanye_url


def down_pic(pic_urls):
    """给出图片链接列表, 下载所有图片"""
    for i, pic_url in enumerate(pic_urls):
        try:
            pic = requests.get(pic_url, timeout=15)
            string = str(i + 1) + '.jpg'
            with open(string, 'wb') as f:
                f.write(pic.content)
                print('成功下载第%s张图片: %s' % (str(i + 1), str(pic_url)))
        except Exception as e:
            print('下载第%s张图片时失败: %s' % (str(i + 1), str(pic_url)))
            print(e)
            continue


if __name__ == '__main__':
    keyword = '火车票'  # 关键词, 改为你想输入的词即可, 相当于在百度图片里搜索一样
    url_init_first = r'http://image.baidu.com/search/flip?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1497491098685_R&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&ctd=1497491098685%5E00_1519X735&word='
    url_init = url_init_first + urllib.parse.quote(keyword, safe='/')
    all_pic_urls = []
    onepage_urls, fanye_url = get_onepage_urls(url_init)
    all_pic_urls.extend(onepage_urls)

    fanye_count = 0  # 累计翻页数
    while 1:
        onepage_urls, fanye_url = get_onepage_urls(fanye_url)
        fanye_count += 1
        # print('第页' % str(fanye_count))
        if fanye_url == '' and onepage_urls == []:
            break
        all_pic_urls.extend(onepage_urls)

    down_pic(list(set(all_pic_urls)))

链接：https://blog.csdn.net/xiligey1/article/details/73321152　　

2、根据搜索词下载谷歌、必应、百度图片

# coding:utf-8
# 基于icrawler第三方库同时爬取google，baidu，bing图片，并对名称进行重写，数据进行分类
# 图片存放路径为：base_dir='F:/文档/text'

import logging
import sys
import base64
from datetime import date
from icrawler.builtin import BaiduImageCrawler, BingImageCrawler, GoogleImageCrawler
from icrawler import ImageDownloader
from icrawler.builtin import GoogleImageCrawler
from six.moves.urllib.parse import urlparse

class PrefixNameDownloader(ImageDownloader):

    def get_filename(self, task, default_ext):
        filename = super(PrefixNameDownloader, self).get_filename(
            task, default_ext)
        return 'prefix_' + filename


class Base64NameDownloader(ImageDownloader):

    def get_filename(self, task, default_ext):
        url_path = urlparse(task['file_url'])[2]
        if '.' in url_path:
            extension = url_path.split('.')[-1]
            if extension.lower() not in [
                    'jpg', 'jpeg', 'png', 'bmp', 'tiff', 'gif', 'ppm', 'pgm'
            ]:
                extension = default_ext
        else:
            extension = default_ext
        filename = base64.b64encode(url_path.encode()).decode()
        return '{}.{}'.format(filename, extension)

def test_google(dir,keyword):
    print('启用google爬虫')
    google_crawler = GoogleImageCrawler(parser_threads=20,
                                        downloader_threads=20,
                                        downloader_cls=Base64NameDownloader,
                                        storage={'root_dir': dir},
                                        log_level = logging.INFO)
    google_crawler.crawl(keyword=keyword, offset=0, max_num=1000,min_size=(200,200), max_size=None)


def test_bing(dir,keyword):
    keyword = keyword.replace(': flickr.com', '')
    print('启用bing爬虫',keyword)
    bing_crawler = BingImageCrawler(
                                    # parser_threads=16,
                                    downloader_cls=Base64NameDownloader,
                                    downloader_threads=16,
                                    storage={'root_dir': dir},
                                    log_level=logging.DEBUG)
    bing_crawler.crawl(keyword=keyword,offset=0, max_num=1000,min_size=None,max_size=None)

def test_baidu(dir,keyword):
    keyword = keyword.replace(': flickr.com', '')
    print('启用百度爬虫',keyword)
    baidu_crawler = BaiduImageCrawler(
                                    # parser_threads=16,
                                    # downloader_threads=16,
                                    downloader_cls=Base64NameDownloader,
                                    storage={'root_dir': dir},
                                    log_level = logging.DEBUG)
    baidu_crawler.crawl(keyword=keyword, offset=0,max_num=1000,min_size=None,max_size=None)


def main():
##################################################################
            keyword='火车票'
            base_dir='F:/文档/text'
            if len(sys.argv) == 1:
                dst = 'all'
            else:
                dst = sys.argv[1:]
            if 'all' in dst:
                dst = ['google', 'bing', 'baidu',]
            if 'google' in dst:
                test_google(base_dir,keyword)
            if 'bing' in dst:
                test_bing(base_dir,keyword)
            if 'baidu' in dst:
                test_baidu(base_dir,keyword)


if __name__ == '__main__':
    main()

　　

链接：https://github.com/Crawler-y/Image_crawl-

3、github 搜索爬虫，有许多有趣的项目。