zoukankan      html  css  js  c++  java
  • 文本分类

    #-*- coding:utf-8 -*-
    
    import re
    import requests
    
    class Spider:
    #页面初始化
    	def __init__(self):
    		self.url = 'http://gz.meituan.com/category/meishi?mtt=1.index%2Ffloornew.nc.1.irj38puy'
    		
    	def get_class_index(self):
    		r = requests.get(self.url)
    		#print r.encoding
    		#print r.text.encode('UTF-8')
    		pattern1 = re.compile(r'<div class="label has-icon">分类:</div>(.*?)</div>',re.S)
    		items = re.findall(pattern1,r.text.encode('UTF-8'))
    		pattern2 = re.compile(r'<li.*?href="(.*?)">(.*?)</a></li>',re.S)
    		items2 = re.findall(pattern2,items[0])
    		f = open('mt_class_index.txt',"w+")
    		llink=[]
    		lname=[]
    		lclass=[]
    		for i in range(len(items2)):
    			x,y=items2[i]
    			llink.append(x)
    			lname.append(y)
    			lclass.append(str(i+1))
    			f.write(str(i+1)+','+x+','+y+'
    ')   
    		f.close()
    		result=zip(lclass,llink,lname)
    		return result
    		
    	def getEverryClass(self,link):
    		r = requests.get(link)
    		pattern1 = re.compile(r'<div class="paginator-wrapper">(.*?)</div>',re.S)
    		items = re.findall(pattern1,r.text.encode('UTF-8'))
    		pattern2 = re.compile(r'<li.*?href="(.*?)".*?</li>',re.S)
    		pattern3 = re.compile(r'<i class="icon icon-shangjia">.*?<a class="link f3 J-mtad-link".*?target="_blank">(.*?)</a>',re.S)#首页商家名称匹配
    		items3 = re.findall(pattern3,r.text.encode('UTF-8'))
    		if items:
    			items2 = re.findall(pattern2,items[0]) #每一页的链接不含首页
    			for i in range(len(items2)):
    				curl ='http://gz.meituan.com'+items2[i]
    				r1 = requests.get(curl)
    				items4 = re.findall(pattern3,r1.text.encode('UTF-8'))
    				items3+=items4
    		return items3
    	def savePageInfo(self):
    		result=self.get_class_index()
    		f1 = open('sj_names.txt',"w+")
    		for i in range(len(result)):
    			cl,lk,na =result[i]
    			sjnames =self.getEverryClass(lk)
    			for i in range(len(sjnames)):
    				f1.write(cl+','+na+','+sjnames[i]+'
    ')
    		f1.close()
    		
    Spider().savePageInfo()
  • 相关阅读:
    MS SQL 错误 :17883,严重度: 1,状态: 0
    秒杀架构中高性能可扩展高可用的一点思考
    让IE10等支持classList2.0
    判定元素是否刚插入到DOM树
    accept巨坑
    for in 循环的输出顺序问题
    css斜线
    angular的directive笔记
    avalon最佳实践
    迷你MVVM框架 avalonjs 0.97发布
  • 原文地址:https://www.cnblogs.com/longyongzhen/p/6698578.html
Copyright © 2011-2022 走看看