获取安居客小区信息

# -*- coding: utf-8 -*-
"""
Created on Sat Jun 24 22:03:17 2017

@author: willowj
"""
import sys
stdout, stdin, stderr  =  sys.stdout, sys.stdin, sys.stderr
reload(sys)
sys.stdout, sys.stdin, sys.stderr =  stdout, stdin, stderr
sys.setdefaultencoding('utf8')  


import requests
#import codecs
import pandas as pd 
import numpy as np
from lxml import html
import math
import time
import re
#安居客 小区

#广州

    
        
headers = {
'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'accept-encoding':'gzip, deflate, sdch',
'accept-language':'zh-CN,zh;q=0.8,en;q=0.6',
'cache-control':'max-age=0',
#'cookie':'als=0; ctid=12; Hm_lvt_c5899c8768ebee272710c9c5f365a6d8=1498312808; sessid=6D343DE2-F344-C0B1-A8FD-794CF9851F6B; lps=http%3A%2F%2Fguangzhou.anjuke.com%2Fcommunity%2Fview%2F853712%7C; _ga=GA1.2.1457521041.1498312777; _gid=GA1.2.341883465.1498312777; aQQ_ajkguid=5E03C75B-5FDF-6879-5AF9-5DDAB323E51F; twe=2; 58tj_uuid=632d5e6b-78a6-483e-9c41-7e2e21023e74; new_session=0; init_refer=; new_uv=8',
'referer':'https://guangzhou.anjuke.com/community/',
'Upgrade-Insecure-Requests':"1",
#"Connection": "close",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
        }

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
        ]

def not_nulist_first(list_):
    if not list_:
        return None
    else:
        return list_[0]

def get_comnuities(page_response_):
    #从网页解析小区数据
    print page_response_.url, 'getting'
    etree = html.document_fromstring(page_response_.text)
    communities = etree.xpath('//div[@_soj="xqlb"]')
    
    #for community in communities:
    page_communities1 = []
    for community in communities:
    
        name = community.xpath("./a/@title")[0]

        community_web = html.urljoin(init_url, community.xpath("./a/@href")[0])
        address = community.xpath(".//address/text()")[0].strip()
        established_date = community.xpath('.//p[@class="date"]/text()')[0]
        price_perm2 = community.xpath('.//strong/text()')
        price_perm2 = not_nulist_first(price_perm2)

        page_communities1.append([name, price_perm2, established_date, address, community_web])
    print name
    return page_communities1   
    


class AnjukeCommunity(object):
 """docstring for ClassName"""
    def __init__(self, init_url=None):
        #广州安居客
        self.init_url = 'https://guangzhou.anjuke.com/community/'
        self.requ = requests.session()
        self.requ.adapters.DEFAULT_RETRIES = 5
        self.pages_max = self.pages_max()

    def pages_max(self):
        #最大页数
        page_response = self.requ.get(self.init_url, headers=headers)
        init_etree = html.document_fromstring(page_response.text)
        community_Nums = int(init_etree.xpath('//span[@class="tit"]/em/text()')[-1])
        communities = init_etree.xpath('//div[@_soj="xqlb"]')
        return int(math.ceil(float(community_Nums)/len(communities)))       

    def get_all_comnunities(self):

        page_communities = []

        for page in range(1, pages_max+1):
            url_ = '%s/p%s/'%(self.init_url, page)
            #打开网页
            headers["User-Agent"] = np.random.choice(USER_AGENTS)
            page_response_ = self.requ.get(url_, headers=headers)
            print  page_response_.url, page_response_.status_code, 'start'
            time.sleep(20)
            page_response_.close()
            page_response_.connection.close()

            #解析网页数据
            communi = get_comnuities(page_response_)
            page_communities.extend(communi)


#网页标题
print init_etree.xpath('//div[@_soj="xqlb"]/a/@title')[0]


init_url = 'https://guangzhou.anjuke.com/community/'
Anjuke_guangzhou = AnjukeCommunity(init_url)
page_communities = Anjuke_guangzhou.get_all_comnunities()

#pandas 规整数据    
communities_pd = pd.DataFrame(page_communities, columns=['name', 'price_perm2', 'established_date', 'address', 'community_web'])
communities_pd['price_perm2'] = communities_pd['price_perm2'].astype(float)
communities_pd['price_part'] = communities_pd['price_perm2']//5000 * 5000
communities_pd['established_year'] = communities_pd['established_date'].str.extract('(d+)',expand=False).astype(int) 
#communities_pd['religion'] = communities_pd['address'].str.extract(u"［(.+?)-", expand=False)
communities_pd['religion'] = communities_pd['address'].str.slice(1,3)

cols = ['religion', 'name', 'price_part', 'price_perm2', 'established_year', 'established_date', 'address', 'community_web']

communities_pd[cols].to_excel(u"安居客广州小区.xlsx",  index=False,  encoding='gb18030')