python爬虫

#! /usr/bin/env python
#coding=utf-8

import requests
import re,json
import sys,os
import Queue,threading
from bs4 import BeautifulSoup
reload(sys)
sys.setdefaultencoding("utf8")

def http_req_get(siteurl):
    headers = {
        "Host": "www.xuebang.com.cn",
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36",
        "Accept-Encoding": "gzip, deflate, sdch",
        "Accept-Language": "zh-CN,zh;q=0.8",
        "Cookie": "__cfduid=da7335f4b0e760976f98697b651fc10041447572288; pgv_pvi=7944819712; deptNumOf11=140; deptNumOf89=60; deptNumOf711=60; commentNumOf11215=1074; deptNumOf1411=56; JSESSIONID=abcqLyMOLKEVDbynTTtev; a2666_pages=1; a2666_times=4; pgv_si=s4040530944; Hm_lvt_8147cdaed425fa804276ea12cd523210=1447572289,1447678990,1447734730; Hm_lpvt_8147cdaed425fa804276ea12cd523210=1447734730; CNZZDATA5928106=cnzz_eid%3D1168227404-1447570407-%26ntime%3D1447729389; Hm_lvt_863e19f68502f1ae0f9af1286bb12475=1447572289,1447678990,1447734730; Hm_lpvt_863e19f68502f1ae0f9af1286bb12475=1447734730; _ga=GA1.3.122575526.1447572289; _gat=1"}
    try:
        urlobj = requests.get(
            url = siteurl,
            headers = headers,
        )
        return urlobj
    except Exception,e:
        print 'yichang'
        pass


#def request_get(siteurl):

class LinksParser(object):
    def __init__(self,urlobj):
        self.urlobj = urlobj
        self.soup = BeautifulSoup(self.urlobj.text, "html.parser")

    #创建大学目录
    def createDaXueDir(self):
        #获取当前文件路径 如果不存在就创建文件夹
        path = sys.argv[0]
        current_dir = os.path.dirname(path)
        real_path = current_dir + '\' + self.soup.title.text.encode('gb18030')
        if os.path.exists(real_path):
            pass
        else:
            try:
                os.mkdir(real_path)
            except:
                pass
        return real_path

    #获取学院 并把学校的院系写入文件
    def xueyuan(self,path):
        try:
            fh = open(real_path + '/xueyuan.txt','wb')
            for line in self.soup.find_all('a',{'class':'yxcologe'}):#.encode('gb18030')
                fh.writelines(line.text.encode('gb18030').strip() + '
')
            fh.close()
        except:
            pass

    #获取该学院的教师列表的url
    def teacher(self,path):
        lst = []
        length = len(self.soup.find_all('a',{'class':'yxcologe'}))
        for i in xrange(length):
            #依次获取每个院系的老师链接
            url = self.soup.find_all('a',{'class':'yxcologe'})[i]['href'].encode('gb18030')
            lst.append(url)
        return lst


    #获取所有教师列表
    def teacher_lst(self):
        length = len(self.soup.find('span',{'class','TJszlist'}).find_all('li'))
        for i in xrange(length):
            lst = self.soup.find('span',{'class','TJszlist'}).find_all('li')[i].find('a')['title']
            #开始截取该系的名称
            yuanxi = str(self.soup.find('span',{'class','t_dqwz'}))
            yuanxi = yuanxi[-40:]
            yuanxi = yuanxi.split('»')[1]
            yuanxi = yuanxi.split('<')[0]
            teacher_lst.append({'department':yuanxi,'name':lst})
            url = self.soup.find('span',{'class','TJszlist'}).find_all('li')[i].find('a')['href'].encode('gb18030')
            teacher_comment_url.append(url)

    #获取教师和该教师下所有评论
    def comment_teacher(self):
        length = len(self.soup.find_all('span',{'class','TJR_info'}))
        if length == 0:
            return 'no comments'
        else:
            for i in xrange(length):
                teacher_content = self.soup.find_all('span',{'class','TJR_info'})[i].find('p',{'class','TJlycon'}).text
                teacher_name = self.soup.find(color='#0088cc').text
                teacher_time = self.soup.find_all('span',{'class','TJR_info'})[i].find('span').string
                teacher_all_comment.append({'teacher_id':teacher_name,'comment':teacher_content,'time':teacher_time})
            json_data = json.dumps(teacher_all_comment, encoding="UTF-8", ensure_ascii=False)
            return json_data

class myThreads(threading.Thread):
    def __init__(self,queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            if self.queue.empty():
                break
            else:
                try:
                    url = self.queue.get_nowait()
                    res_obj = LinksParser(http_req_get(url))
                    res_obj.teacher_lst()
                except Exception,e:
                    break
class commentThreads(threading.Thread):
    def __init__(self,queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            if self.queue.empty():
                break
            else:
                try:
                    url = self.queue.get_nowait()
                    res_obj = LinksParser(http_req_get(url))
                    test = res_obj.comment_teacher()
                    fh = open(real_path + '/teacher_comment_lst.txt','wb')
                    fh.write(test)
                    fh.close()
                except Exception,e:
                    break

if __name__ == '__main__':
    #输入学院的ID
    #i = sys.argv[1]
    idlist = [11, 129, 70, 71]
    for i in idlist:
        i = str(i)
        thread_number = 50
        url = 'http://www.xuebang.com.cn/' + i + '/deptlist'
        try:
            urlobj = http_req_get(url)
            #生成院系列表    
            response_obj = LinksParser(urlobj)
            #文件所保存的路径
            real_path = response_obj.createDaXueDir()
            response_obj.xueyuan(real_path)
            
            #获取该学院的教师列表的url 即每个系学院下面的教师
            xi_to_teacher = response_obj.teacher(real_path)
            #获取所有的教师列表


            global teacher_lst
            teacher_lst = []


            #教师comment链接
            global teacher_comment_url
            teacher_comment_url = []

            #多线程加快速度
            queue = Queue.Queue()
            for line in xi_to_teacher:
                queue.put(line)
            threads = []
            for i in xrange(thread_number):
                threads.append(myThreads(queue))
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            teacher_lst = json.dumps(teacher_lst, encoding="UTF-8", ensure_ascii=False)
            #print len(teacher_lst)

            #将教师写入文件
            try:
                fh = open(real_path + '/teacher_lst.txt','wb')
                fh.write(teacher_lst)
                fh.close()
            except:
                pass
         
            global teacher_all_comment
            teacher_all_comment = []
            #获取教师及教师评论
            queu = Queue.Queue()
            for line_url in teacher_comment_url:
                queu.put(line_url)

            comments = []
            for line in xrange(thread_number):
                comments.append(commentThreads(queu))
            for t in comments:
                t.start()
            for t in comments:
                t.join()
        except:
            pass

        #print teacher_all_comment