zoukankan      html  css  js  c++  java
  • python爬虫学习(2)__抓取糗百段子,与存入mysql数据库

    import pymysql
    import requests
    from bs4 import BeautifulSoup
    #pymysql链接数据库 conn
    =pymysql.connect(host='127.0.1',unix_socket='/tmp/mysql.sock',user='root',passwd='19950311',db='mysql') cur=conn.cursor() cur.execute("USE scraping")
    #存储段子标题,内容
    def store(title,content): cur.execute("insert into pages(title,content) values("%s","%s")",(title,content)) cur.connection.commit() global links class QiuShi(object): def __init__(self,start_url): self.url=start_url def crawing(self): try: html=requests.get(self.url,'lxml') return html.content except ConnectionError as e: return '' def extract(self,htmlContent): if len(htmlContent)>0: bsobj=BeautifulSoup(htmlContent,'lxml') #print bsobj jokes=bsobj.findAll('div',{'class':'article block untagged mb15'}) for j in jokes: text=j.find('h2').text content=j.find('div',{'class':'content'}).string if text != None and content != None: # print text,content,数据库编码为utf-8 store(text.encode('utf-8'),content.encode('utf-8')) print text.encode('utf-8'),content.encode('utf-8') print '------------------------------------------------------------------------------' else: print '' def main(self): text=self.crawing() self.extract(text) try: qiushi=QiuShi('http://www.qiushibaike.com/') qiushi.main() finally:
    #关闭cursor,connection cur.close() conn.close()
  • 相关阅读:
    navigator
    历史记录跳转
    更改URL
    计数器
    窗口位置和大小
    open用法
    confirm用法
    项目中访问本地node服务跨域问题
    jenkins使用
    基于Vue的SSR
  • 原文地址:https://www.cnblogs.com/yunwuzhan/p/5765963.html
Copyright © 2011-2022 走看看