爬取百度百科上中国所有城市的信息

# coding=utf-8
import xlrd
import xlwt
import requests
import re
import json
import os
import sys
reload(sys)
sys.setdefaultencoding('utf8')
from xlutils.copy import copy


data = xlrd.open_workbook(u'经济情况 - 副本.xlsx')
table=data.sheets()[0]
nrows = table.nrows
ncols = table.ncols
print nrows,ncols

print  '--'.join(table.row_values(1))
errortxt=open(u'错误城市.txt','a')


class WriteExcel(object):
    def __init__(self,filename):
        self.file=filename
    def write2excel(self,row, col, strx):
        rb = xlrd.open_workbook(self.file)#, formatting_info=True)
        wb = copy(rb)
        ws = wb.get_sheet(0)
        ws.write(row, col, unicode(strx))
        wb.save(self.file)

class Crawlcity(WriteExcel):
    def __init__(self,cityname):
        super(Crawlcity,self).__init__(u'经济情况 - 副本.xlsx')
        self.cityname=cityname
        print '爬取%s的百科信息'%cityname

    def myserch(self,fieldx,content):
        matchx=re.search('%s</dt>s*?<dd class="basicInfo-item value">s*?(.*?)s*?</dd>'%fieldx, content)
        if matchx:
            #print '%s的原始匹配是：%s' % (fieldx,matchx.group(1))
            matchx_str=re.sub('<sup.*','',matchx.group(1))
            return   re.sub('<.*?>|<.*?>','',matchx_str)
        else:
            print '###########没找到%s的%s字段的匹配'%(self.cityname,fieldx)
            return '--'

    def get_baike_city(self,citynamex=None):
        if not citynamex:
            url='https://baike.baidu.com/item/%s'%self.cityname
        else:
            url='https://baike.baidu.com/item/%s'%citynamex
        print '%s的百科链接是%s'%(self.cityname,url)
        resp=requests.get(url)
        #print resp.content
        if 'https://baike.baidu.com/item' not in  resp.url:
            print '%s跳转到错误链接%s'%(self.cityname,resp.url)
            return self.get_baike_city(citynamex=self.cityname.replace('市',''))
            # errortxt.write('%s    %s'%(self.cityname,url))
            # return


        belong_area = self.myserch('所属地区',resp.content)
        governs_area = self.myserch('下辖地区',resp.content)
        region_type = self.myserch('行政区类别',resp.content)
        area = self.myserch('面&nbsp;&nbsp;&nbsp;&nbsp;积',resp.content)
        population = self.myserch('人&nbsp;&nbsp;&nbsp;&nbsp;口',resp.content)
        plata_code = self.myserch('车牌代码',resp.content)
        gdp = self.myserch('地区生产总值',resp.content)
        avarage_gdp = self.myserch('人均生产总值',resp.content)
        per_capita_income = self.myserch('人均支配收入',resp.content)

        print '%s
的所属地区是：%s
下辖地区是：%s
行政区类别是：%s
面积是：%s
人口是：%s
车牌代码是：%s
地区生产总值是：%s
人均生产总值是：%s
人均支配收入是：%s
'%(self.cityname,belong_area,governs_area,
                region_type,area,population,plata_code,gdp,avarage_gdp,per_capita_income)
        print '*'*100

        for n in range(1,nrows):
            if self.cityname==table.row_values(n)[0]:
                self.write2excel(n,2,belong_area)
                self.write2excel(n, 3, governs_area)
                self.write2excel(n, 4, region_type)
                self.write2excel(n, 5, area)
                self.write2excel(n, 6, population)
                self.write2excel(n, 7, plata_code)
                self.write2excel(n, 8, gdp)
                self.write2excel(n, 9, avarage_gdp)
                self.write2excel(n, 10, per_capita_income)
                self.write2excel(n, 11, url)
                break


if __name__=="__main__":
    for n in range(1,nrows):
        cityname = table.row_values(n)[0]
        if table.row_values(n)[2]!='':
            pass
            print 'excel已记录了，跳过%s这个城市城市'%cityname
        else:
            Crawlcity(cityname).get_baike_city()

百度百科的规律是https://baike.baidu.com/item/xxxx

例如要爬取黄冈市的信息，就用https://baike.baidu.com/item/黄冈市，然后请求会自动重定向到该词条。注意结尾不要加一个/，否则会是一个错误的页面。

 从excel读取要爬的城市，然后爬取城市信息，把数据插入到excel中。