回顾之前学过的,经过一年考研,忘差不多了,要拾起来
import json import re from multiprocessing.pool import Pool import requests from requests.exceptions import RequestException from selenium import webdriver headers={ "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" } def get_one_page(url): try: # browser = webdriver.Chrome() //两种方法获取源码,requests必须要自加headers,webdriver不需要 response =requests.get(url,headers=headers) if response.status_code == 200: return response.text return None except RequestException: return None def parse_one_page(html): pattern = re.compile('<dd.*?board-index.*?>(d+)</i>.*?data-src="(.*?)".*?name"><a' +'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>' +'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S) items = re.findall(pattern,html) for item in items: yield { 'index': item[0], 'title': item[2], 'actor': item[3].strip()[3:], 'time': item[4].strip()[5:], 'score': item[5]+item[6], 'image': item[1], } def write_to_file(content): with open('result.txt','a',encoding='utf-8') as f: f.write(json.dumps(content,ensure_ascii=False) + ' ') f.close() def main(offset): url = 'https://maoyan.com/board/4?offset='+str(offset) html = get_one_page(url) for item in parse_one_page(html): print(item) write_to_file(item) if __name__ == '__main__': # for i in range(10): # main(i*10) pool = Pool() pool.map(main,[i*10 for i in range(10)]) //多线程爬取,速度比较快