zoukankan      html  css  js  c++  java
  • python2.7 urllib2 爬虫

     # _*_ coding:utf-8 _*_

    import urllib2
    import cookielib
    import random
    import re
    from bs4 import BeautifulSoup
    import datetime

    dax = datetime.datetime.now().strftime('%Y-%m-%d')
    print(dax)

    url = 'http://ww=singlemessage&isappinstalled=0'

    cj = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    urllib2.install_opener(opener)
    request = urllib2.Request(url)
    headers = [
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
    'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
    'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'
    ]

    hds = random.choice(headers)
    # print(hds)
    request.add_header('User-Agent','%s' % hds)
    #response = urllib2.urlopen("http://www.hn1m=singlemessage&isappinstalled=0")
    response = urllib2.urlopen(request)
    cont = response.read()
    #print(cont)

    soup = BeautifulSoup(cont,'html.parser',from_encoding='utf-8')
    # print(soup)
    # listyj = soup.find_all('dl')
    # for listyjx in listyj:
    # print(listyjx.name,listyjx.attrs,listyjx.gettext())
    # # if dax in listyjx:
    # # print(listyjx)

  • 相关阅读:
    使用Application Insights 做分析
    UWP中GridView右击选中的实现
    Bing Map
    UWP深入学习六:Build better apps: Windows 10 by 10 development series
    从上往下打印二叉树
    二叉树中和为某一值的路径
    二叉树的镜像
    树的子结构
    由前序遍历和中序遍历构建二叉树
    二叉树常见题目
  • 原文地址:https://www.cnblogs.com/ruiy/p/9193940.html
Copyright © 2011-2022 走看看