51job爬虫

51job爬虫

　　项目使用Python3.7爬取前程无忧对应关键字的招聘，保存到mongodb，爬取下来的数据,可分析出目前互联网的近况,可统计到每个招聘岗位有多少，每个岗位的薪资分布情况

　　github地址：https://github.com/HowName/51job

• 统计结果图，java还是老大哥
• 
• 爬取效果图
• 
• mongodb数据图
• 
• 使用到的库(第三方库建议使用pip进行安装)

• BeautifulSoup4,pymongo,requests,re,time
  

  　　

• 项目主代码

• import re
  import time
  from bs4 import BeautifulSoup
  from pack.DbUtil import DbUtil
  from pack.RequestUtil import RequestUtil
  
  db = DbUtil()
  
  # 要查找的岗位
  keywords = ['php', 'java', 'python', 'node.js', 'go', 'hadoop', 'AI', '算法工程师', 'ios', 'android', '区块链', '大数据']
  
  for keyword in keywords:
  
      cur_page = 1
      url = 'https://search.51job.com/list/030200,000000,0000,00,9,99,@keyword,2,@cur_page.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=' 
          .replace('@keyword', str(keyword)).replace('@cur_page', str(cur_page))
      req = RequestUtil()
      html_str = req.get(url)
  
      # 从第一页中查找总页数
      soup = BeautifulSoup(html_str, 'html.parser')  # 推荐使用lxml
      the_total_page = soup.select('.p_in .td')[0].string.strip()
      the_total_page = int(re.sub(r"D", "", the_total_page))  # 取数字
  
      print('keyword:', keyword, 'total page: ', the_total_page)
      print('start...')
  
      while cur_page <= the_total_page:
          """
          循环获取每一页
          """
  
          url = 'https://search.51job.com/list/030200,000000,0000,00,9,99,@keyword,2,@cur_page.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=' 
              .replace('@keyword', str(keyword)).replace('@cur_page', str(cur_page))
          req = RequestUtil()
          html_str = req.get(url)
  
          if html_str:
              soup = BeautifulSoup(html_str, 'html.parser')
  
              #  print(soup.prettify()) #格式化打印
  
              the_all = soup.select('.dw_table .el')
              del the_all[0]
  
              # 读取每一项招聘
              dict_data = []
              for item in the_all:
                  job_name = item.find(name='a').string.strip()
                  company_name = item.select('.t2')[0].find('a').string.strip()
                  area = item.select('.t3')[0].string.strip()
                  pay = item.select('.t4')[0].string
                  update_time = item.select('.t5')[0].string.strip()
  
                  dict_data.append(
                      {'job_name': job_name, 'company_name': company_name, 'area': area, 'pay': pay,
                       'update_time': update_time, 'keyword': keyword}
                  )
  
              # 插入mongodb
              db.insert(dict_data)
  
              print('keyword:', keyword, 'success page:', cur_page, 'insert count:', len(dict_data))
              time.sleep(0.5)
  
          else:
              print('keyword:', keyword, 'fail page:', cur_page)
  
          # 页数加1
          cur_page += 1
  
      else:
          print('keyword:', keyword, 'fetch end...')
  
  else:
      print('Mission complete!!!')