淘宝店铺所有ID爬取

from selenium import webdriver
import re
import time
from lxml import etree
from selenium.webdriver.chrome.options import Options
import pymysql
import requests
with open('taobao',encoding='utf-8') as f:
    a = f.readlines()
    for i in range(25,len(a)):
        print(i)
        print('-'*30)
        shop_id,data_widgetid=a[i].replace('
','').split(',')
        goods_id_list=[]
        for j in range(1, 6):
            heades={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
                    'cookie': 'cna=YmdIFYYEcVMCATy+6cYV7pO7; t=adaf0c3bf86835603c597adfc1b0b875; hng=CN%7Czh-CN%7CCNY%7C156; thw=cn; tg=0; ali_ab=60.190.233.198.1556248533986.3; UM_distinctid=16a873f95f6326-0ab1211d27a5ca-f353163-1fa400-16a873f95f7987; mt=ci=33_1&np=; _tb_token_=NdGOJfjtYxmUxF6eBNYE; cookie2=1129bac788da762f2fc3dc2c31b61a38; v=0; _m_h5_tk=d3bac98f67e17d0491be6ac24fe972dd_1557735439567; _m_h5_tk_enc=d3390800dc68df57fc1b75bbb70cc8a5; unb=363984921; sg=312; _l_g_=Ug%3D%3D; skt=e40a50c529c9ca8e; cookie1=ACPl%2Ff%2BzGp66ZdFRG9knxI5XoCVXhR9WG1xO%2FUG4dwc%3D; csg=31b7be14; uc3=vt3=F8dBy3qIhOt5DMx3%2Bxo%3D&id2=UNaE8tOwscZl&nk2=szPwK%2FJkWAu%2FhoZP1cgV&lg2=UtASsssmOIJ0bQ%3D%3D; existShop=MTU1NzcyNTc0MQ%3D%3D; tracknick=%5Cu8C22%5Cu4F5C%5Cu5A01707924553; lgc=%5Cu8C22%5Cu4F5C%5Cu5A01707924553; _cc_=W5iHLLyFfA%3D%3D; dnk=%5Cu8C22%5Cu4F5C%5Cu5A01707924553; _nk_=%5Cu8C22%5Cu4F5C%5Cu5A01707924553; cookie17=UNaE8tOwscZl; uc1=cookie16=VT5L2FSpNgq6fDudInPRgavC%2BQ%3D%3D&cookie21=UIHiLt3xTIkz&cookie15=U%2BGCWk%2F75gdr5Q%3D%3D&existShop=false&pas=0&cookie14=UoTZ48JVyDnTlA%3D%3D&tag=8&lng=zh_CN; enc=4sdKSidVGl7W9qpdANZb5oGoKUO%2F1hBeMD%2F5tLQjGIhgPZ2J22LhXD%2FydDw7UFOd%2F%2FJ6YLTNDwpwUQY2wNZeHg%3D%3D; x=479184430%26e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0; swfstore=299154; pnm_cku822=098%23E1hvnQvUvbpvUpCkvvvvvjiPRLSwAjlbn2SW1j3mPmPhtjiRRLM96jE8n2dO0jtPR8wCvvBvpvpZRphvChCvvvvPvpvhMMGvvvhCvvXvppvvvvvEvpCWBVTuv8RAnhhAcUmxdBQaUmx%2Fgj7xhpydfvc6lCAXal%2B0747BhC3qVmHoDOmOVzIUExjxALwp8BpDN%2BBljdUf8B6s%2BXZz%2BsBwN6fW0byCvm3vpvvvvvCvphCvh8pvvhR8phvwv9vvBj1vpCQmvvChxhCvjvUvvhBZkphvCyEmmvofVuyCvv3vpvoill%2FQg9%3D%3D; x5sec=7b2273686f7073797374656d3b32223a223233626530613037323138396631373133663036326535623934333633386633434f434c354f5946454a6a477a7669427770436278414561437a4d324d7a6b344e446b794d54737a227d; whl=-1%260%260%261557726692034; l=bBMrGiYVvCcIjkC2BOCwZQKXGi_OxIRAguSJGNzMi_5Ie1L_jw_Old9Lfep6Vj5R_VYB4UaStkw9-etkO; isg=BKWlldC2FX0WonE5blob6kQStGFz8kl4KxbRgqeKM1zrvsUwbzZBRQVWSGJtvnEs'}
            url = 'https://' + str(shop_id) + '.taobao.com/i/asynSearch.htm?&mid=w-' + data_widgetid + '-0' + '&wid='+data_widgetid+'&orderType=hotsell_desc&pageNo='+str(j)
            # print(url)
            response=requests.get(url,headers=heades)
            # print(response.text.replace('\',''))
            res=response.text.replace('\','')
            print(re.findall('itemIds=(d+,.*)&source=shop', res))
            goods_id_part=re.findall('itemIds=(d+,.*)&source=shop', res)[0]
            goods_id_new=list(set(goods_id_part.split(',')))
            goods_id_new.sort(key=goods_id_part.index)
            goods_id_list.extend(goods_id_new)
        goods_id_list = goods_id_list[0:100]
        print("-"*30)
        # print(goods_id_list)
        for num in range(len(goods_id_list)):
            goods_id=goods_id_list[num]
            conn = pymysql.connect(*******)
            sql = "replace  into 淘宝id(ID,店铺ID,名次)values('" + goods_id+ "','" + shop_id+ "','" + str(num)+ "')"
            conn.query(sql)
            conn.commit()
        print('入库结束')