### 需求：爬取https://m.vmall.com/help/hnrstoreaddr.htm荣耀线下门店中的门店详情信息。页面显示如下：
- 首页显示
- 详情页显示
### 基于抓包工具分析如下：
- 
### 查看定位到数据包的请求头信息：
- 请求的url和请求方式如下：
- 请求携带的请求参数如下：
- 注意：请求参数为字典格式并非常规的键值对，所以在代码实现中需要使用dumps将字典转成json串作为请求参数
### 代码实现：爬取到首页对应的门店信息
import requests import json url = 'https://openapi.vmall.com/mcp/offlineshop/getShopList' data = { "portal":"2", "lang":"zh-CN", "country":"CN", "brand":"1", "province":"河北省", "city":"邯郸市", "pageNo":"2", "pageSize":"20"} #必须使用dumps操作 json_data = requests.post(url,data=json.dumps(data)).json()
- 请求到的数据为：
{ "code": "0", "shopInfos": [{ "address": "河北省邯郸市磁县建设西路与京深线交叉口美食林一楼底商", "brand": 1, "city": "邯郸市", "dist": "磁县", "distance": 0.0, "id": 111616, "isStar": 1, "latitude": "36.364752", "longitude": "114.390096", "name": "荣耀授权体验店(建设西路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "9:00-18:00", "shopCode": "RYRA03100602", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市邱县育新街100号移动公司斜对过", "brand": 1, "city": "邯郸市", "dist": "邱县", "distance": 0.0, "id": 111897, "isStar": 1, "latitude": "36.815125", "longitude": "115.173926", "name": "荣耀授权体验店(育新街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100954", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市磁县友谊北大街199号", "brand": 1, "city": "邯郸市", "dist": "磁县", "distance": 0.0, "id": 108067, "isStar": 1, "latitude": "36.387417", "longitude": "114.379099", "name": "荣耀授权体验店(友谊北大街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03104185", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市肥乡县建安路中段", "brand": 1, "city": "邯郸市", "dist": "肥乡县", "distance": 0.0, "id": 111932, "isStar": 1, "latitude": "36.559004", "longitude": "114.813224", "name": "荣耀授权体验店(建安路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100613", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市临漳县邺都南大街移动公司旁", "brand": 1, "city": "邯郸市", "dist": "临漳县", "distance": 0.0, "id": 111678, "isStar": 1, "latitude": "36.33081", "longitude": "114.613037", "name": "荣耀授权体验店(邺都大街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100610", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市成安县西关口西行200路南", "brand": 1, "city": "邯郸市", "dist": "成安县", "distance": 0.0, "id": 107586, "isStar": 1, "latitude": "36.447977", "longitude": "114.692388", "name": "荣耀授权体验店(政府街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03103736", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市馆陶县新华街国土资源局斜对过", "brand": 1, "city": "邯郸市", "dist": "馆陶县", "distance": 0.0, "id": 111751, "isStar": 1, "latitude": "36.548521", "longitude": "115.294026", "name": "荣耀授权体验店(新华街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100615", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市武安市中兴路1666号", "brand": 1, "city": "邯郸市", "dist": "武安市", "distance": 0.0, "id": 106905, "isStar": 1, "latitude": "36.692459", "longitude": "114.191067", "name": "荣耀授权体验店(中兴路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100255", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市永年区洺兴路电子一条街口", "brand": 1, "city": "邯郸市", "dist": "永年区", "distance": 0.0, "id": 111771, "isStar": 1, "latitude": "36.782304", "longitude": "114.494515", "name": "荣耀授权体验店(洺兴路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100611", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市鸡泽县鸡泽镇会盟南大街89", "brand": 1, "city": "邯郸市", "dist": "鸡泽县", "distance": 0.0, "id": 111786, "isStar": 1, "latitude": "36.919418", "longitude": "114.882834", "name": "荣耀授权体验店(会盟南大街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:00-18:30", "shopCode": "RYRA03100614", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市曲周县振兴路移动手机大卖场", "brand": 1, "city": "邯郸市", "dist": "曲周县", "distance": 0.0, "id": 107216, "isStar": 1, "latitude": "36.780474", "longitude": "114.964419", "name": "荣耀授权体验店(振兴路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "9:00-18:30", "shopCode": "RYRA03101174", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市武安市新华大街145号", "brand": 1, "city": "邯郸市", "dist": "武安县", "distance": 0.0, "id": 107729, "isStar": 1, "latitude": "36.695474", "longitude": "114.184811", "name": "荣耀授权体验店(新华大街店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03103735", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市大名县大名府路东段路南（妇幼保健院对过）", "brand": 1, "city": "邯郸市", "dist": "大名县", "distance": 0.0, "id": 111832, "isStar": 1, "latitude": "36.288208", "longitude": "115.171058", "name": "荣耀授权体验店(大名府路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:00-18:30", "shopCode": "RYRA03100612", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市广平县人民路166号", "brand": 1, "city": "邯郸市", "dist": "广平县", "distance": 0.0, "id": 108001, "isStar": 1, "latitude": "36.488035", "longitude": "114.957763", "name": "荣耀授权体验店(人民路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03103980", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市邯山区陵园路114号金茂大厦一楼", "brand": 1, "city": "邯郸市", "dist": "邯山区", "distance": 0.0, "id": 126945, "isStar": 1, "latitude": "36.59796329", "longitude": "114.4948675", "name": "荣耀授权体验店(陵园路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "8:30-18:30", "shopCode": "RYRA03100100", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "河北省邯郸市涉县振兴路与龙山大街交叉口东南角", "brand": 1, "city": "邯郸市", "dist": "涉县", "distance": 0.0, "id": 111607, "isStar": 1, "latitude": "36.567255", "longitude": "113.680533", "name": "荣耀授权体验店(振兴路店)", "pictures": [], "province": "河北省", "score": 5, "serviceTime": "9:00-18:00", "shopCode": "RYRA03100599", "starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg", "type": 1 }, { "address": "中长街老检查院对面", "brand": 1, "city": "邯郸市", "dist": "鸡泽县", "distance": 0.0, "id": 137612, "isStar": 0, "latitude": "36.921823", "longitude": "114.882514", "name": "邯郸鸡泽航天中长街旗舰店", "pictures": [], "province": "河北省", "score": 4, "serviceTime": "10:00-18:30", "shopCode": "RY0000000001", "starShopPic": "", "type": 0 }, { "address": "磁州路130号", "brand": 1, "city": "邯郸市", "dist": "磁县", "distance": 0.0, "id": 137613, "isStar": 0, "latitude": "36.379835", "longitude": "114.395066", "name": "邯郸磁县盛尧磁州路电信店", "pictures": [], "province": "河北省", "score": 4, "serviceTime": "10:00-18:30", "shopCode": "RY0000000001", "starShopPic": "", "type": 0 }, { "address": "滏阳北大街新世纪商业广场1楼", "brand": 1, "city": "邯郸市", "dist": "峰峰矿区", "distance": 0.0, "id": 137614, "isStar": 0, "latitude": "36.428341", "longitude": "114.217691", "name": "邯郸峰峰太行新世纪店", "pictures": [], "province": "河北省", "score": 4, "serviceTime": "10:00-18:30", "shopCode": "RY0000000001", "starShopPic": "", "type": 0 }, { "address": "建设街移动公司路南西侧泰山通讯", "brand": 1, "city": "邯郸市", "dist": "肥乡区", "distance": 0.0, "id": 137625, "isStar": 0, "latitude": "36.559058", "longitude": "114.811038", "name": "邯郸肥乡泰山建设街店", "pictures": [], "province": "河北省", "score": 4, "serviceTime": "10:00-18:30", "shopCode": "RY0000000001", "starShopPic": "", "type": 0 }], "success": true, "totalRows": 60 }
- 请求到的数据分析：
- 数据为门店相关数据，其中每一个门店有其对应的一个id值，我们需要将id值解析出来，在后面请求详情页会使用到
import requests import json url = 'https://openapi.vmall.com/mcp/offlineshop/getShopList' data = { "portal":"2", "lang":"zh-CN", "country":"CN", "brand":"1", "province":"河北省", "city":"邯郸市", "pageNo":"2", "pageSize":"20"} #从中解析出id json_data = requests.post(url,data=json.dumps(data)).json() for dic in json_data['shopInfos']: _id = dic['id'] #解析出门店的id值
### 请求每一个门店详情页的数据
- 请求的url和请求方式：
- 请求参数：
发现只有shopId为动态变化的请求参数其他都是固定不变的，然后该shopId就是上一步我们解析出来的门店id，则基于门店id作为请求详情页的请求参数
import requests import json url = 'https://openapi.vmall.com/mcp/offlineshop/getShopList' data = { "portal":"2", "lang":"zh-CN", "country":"CN", "brand":"1", "province":"河北省", "city":"邯郸市", "pageNo":"2", "pageSize":"20"} #从中解析出id json_data = requests.post(url,data=json.dumps(data)).json() for dic in json_data['shopInfos']: _id = dic['id'] #拼接详情页的url detail_url = 'https://openapi.vmall.com/mcp/offlineshop/getShopById?portal=2&version=10&country=CN&shopId={}&lang=zh-CN'.format(_id) finally_data = requests.get(url=detail_url).json() print(finally_data)#每一页详情页url的数据