爬取爱笔智能招聘职位

import urllib.request
import urllib.parse
import requests
from urllib.parse import urlencode
from pyquery import PyQuery as pq
from pymongo import MongoClient
import json


url = 'http://aibee.com/cn/joinus.aspx?action=jobinfo'

headers = {
    'Host': 'aibee.com',
    'Referer': 'http://aibee.com/cn/joinus.aspx',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
} 

client = MongoClient()
db = client['aibee']
collection = db['aibee']
max_id = 50

def get_page(id):  

    formData = {  
            'id': id,
        }  
     
    #将str类型转换为bytes类型  
    data = urllib.parse.urlencode(formData).encode("utf-8")   
    request = urllib.request.Request(url, data=data, headers=headers)  
    response = urllib.request.urlopen(request)
    #print(response.read().decode('utf-8'))
    result = response.read().decode('utf-8')
    #print(result)
    #print(len(result))
    #print(id)

    if len(result)!=12:
        
       # print(id)
        content=result.replace(",",":")
    
        id=content.split(':')[2].strip()
        #print(id)
        title=content.split(':')[4].strip()
        #print(title)
        zhize=content.split(':')[6].strip().replace("	","").replace("<ul>","").replace("<li>","").replace("</li>","").replace("</ul>","").replace("\t","")
        #print(zhize)
        yaoqiu=content.split(':')[8].strip().replace("	","").replace("<ul>","").replace("<li>","").replace("</li>","").replace("</ul>","").replace("\t","")
        #print(yaoqiu)
        dtt=content.split(':')[12].strip()[:-3]
        #print(dtt)
        emailaddr=content.split(':')[16].strip()
        #print(emailaddr)

        
        aibee={
                'id':id,
                'title':title,
                'zhize':zhize,
                'yaoqiu':yaoqiu,
                'dtt': dtt,
                'emailaddr': emailaddr
            }
        #print(aibee) 
    else:
        aibee=0

    return aibee
    

def write_to_file(content):
    with open('aibee.json','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'
')
        f.close()


def save_to_mongo(result):
    if collection.insert(result):
        print('Saved to Mongo')        


if __name__ == "__main__":  
    
    for id in range(1, max_id + 1):
        #get_page(id)
        content = get_page(id)
        if content!=0:
            print(content)
            write_to_file(content)
            save_to_mongo(content)