def parse(self, response): pattern=re.compile('token=(.*?);') token=pattern.findall( response.headers.get("set-cookie").decode("utf-8"))[0] cookie = { '__cfduid': 'd67f5270ed84c0000af9c771fdee950631551004073', '_ga': 'GA1.2.2009295084.1551004056', '_gid': 'GA1.2.513859849.1551004056', 'token': token } yield scrapy.Request(url='https://scrapingclub.com/exercise/ajaxdetail_cookie/?token='+str(token),cookies=cookie,headers=header,callback=self.parse_json)
settings.py
COOKIES_ENABLED = True
DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # 'Accept-Language': 'en', # 'cookie': '__cfduid=d67f5270ed84c0000af9c771fdee950631551004073; _ga=GA1.2.2009295084.1551004056; _gid=GA1.2.513859849.1551004056; token=685NVSGV2O', # 'dnt': '1', 'referer': 'https://scrapingclub.com/exercise/detail_cookie/', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36', 'x-requested-with': 'XMLHttpRequest' }