zoukankan      html  css  js  c++  java

  • 多线程队列下载天涯帖子

    版本一：


    版本二：

    1. #coding:utf-8
    3. import Queue
    4. import threading
    5. import time
    6. import urllib2
    7. import re
    8. import sys
    10. queue = Queue.Queue()
    11. url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
    12. urls=[]
    13. Dict_txt = {}
    14. file_name = 'abc.txt'
    16. class ThreadNum(threading.Thread):
    17.   def __init__(self, queue):
    18.     threading.Thread.__init__(self)
    19.     self.queue = queue
    21.   def run(self):
    22.     while True:
    23.       #消费者端，从队列中获取num
    24.       num = self.queue.get()
    25.       sys.stdout.flush()
    26.       down_text(Dict_txt, num)
    27.       print '**finished download %s**' % num
    28.       time.sleep(1)
    29.       #在完成这项工作之后，使用 queue.task_done() 函数向任务已经完成的队列发送一个信号
    30.       self.queue.task_done()
    33. def down_text(txt_dict, url):
    34.     """根据传入的url抓出各页内容，按页数做键存入字典"""
    35.     print '--downling %s--' % url
    36.     html_content =urllib2.urlopen(url, timeout=30).read()
    37.     text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>s*(.*?)</div>', re.DOTALL)
    38.     text = text_pattern.findall(html_content)
    39.     text_join = ['



'.join(item) for item in text]
    40.     txt_dict[url] = text_join
    42. def page(url):
    43.     """根据第一页地址抓取总页数"""
    44.     html_page = urllib2.urlopen(url, timeout=30).read()
    45.     page_pattern = re.compile(r'<a href="S*?">(d*)</a>s*<a href="S*?" class="S*?">下页</a>')
    46.     page_result = page_pattern.search(html_page)
    47.     list1 = []
    48.     if page_result:
    49.         page_num = int(page_result.group(1))
    50.         page_range = range(1, page_num+1)
    51.         for num in page_range:
    52.                 myurl = '%s%s.shtml' % (url[:-7], num)
    53.                 list1.append(myurl)
    54.     return list1
    56. def write_text(filename, dict, urls):
    57.     """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
    58.     tx_file = open(filename, 'w+')
    59.     for i in urls:
    60.         tx_list = dict[i]
    61.         for tx in tx_list:
    62.             tx = tx.replace('<br>', '
').replace('<br />', '
').replace(' ', '')
    63.             tx_file.write(tx.strip()+'
'*4)
    64.     tx_file.close()   
    67. start = time.time()
    68. def main():
    69.   #产生一个 threads pool, 并把消息传递给thread函数进行处理，这里开启10个并发
    70.   urls = page(url)
    72.   for i in range(5):
    73.     t = ThreadNum(queue)
    74.     t.start()
    75.   
    76.   #往队列中填错数据
    77.   for num in urls:
    78.       queue.put(num)
    79.   #wait on the queue until everything has been processed
    80.   queue.join()
    83.   print '---- start write ----'
    84.   write_text(file_name, Dict_txt, urls)
    86. main()
    87. print"Elapsed Time: %s" % (time.time() - start)


    --版本三
    1. #coding:utf-8
    3. import urllib
    4. import re
    5. import threading
    6. import os, time
    8. class Down_Tianya(threading.Thread):
    9.     """多线程下载"""
    10.     def __init__(self, url, num, dt):
    11.         threading.Thread.__init__(self)
    12.         self.url = url
    13.         self.num = num
    14.         self.txt_dict = dt
    16.     def run(self):
    17.         print 'downling from %s' % self.url
    18.         self.down_text()
    20.     def down_text(self):
    21.         """根据传入的url抓出各页内容，按页数做键存入字典"""
    22.         html_content =urllib.urlopen(self.url).read()
    23.         text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>s*(.*?)</div>', re.DOTALL)
    24.         text = text_pattern.findall(html_content)
    25.         text_join = ['



'.join(item) for item in text]
    26.         self.txt_dict[self.num] = text_join
    31.   
    32. def page(url):
    33.     """根据第一页地址抓取总页数"""
    34.     html_page = urllib.urlopen(url).read()
    35.     page_pattern = re.compile(r'<a href="S*?">(d*)</a>s*<a href="S*?" class="S*?">下页</a>')
    36.     page_result = page_pattern.search(html_page)
    37.     if page_result:
    38.         page_num = int(page_result.group(1))
    39.         return page_num
    43. def write_text(dict, fn):
    44.     """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
    45.     tx_file = open(fn, 'w+')
    46.     pn = len(dict)
    47.     for i in range(1, pn+1):
    48.         tx_list = dict[i]
    49.         for tx in tx_list:
    50.             tx = tx.replace('<br>', '
').replace('<br />', '
').replace(' ', '')
    51.             tx_file.write(tx.strip()+'
'*4)
    52.     tx_file.close()
    55. def main():
    56.     url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
    57.     file_name ='abc.txt'
    58.     my_page = page(url)
    59.     my_dict = {}
    61.     print 'page num is : %s' % my_page
    63.     threads = []
    64.    
    65.     """根据页数构造urls进行多线程下载"""
    66.     for num in range(1, my_page+1):
    67.         myurl = '%s%s.shtml' % (url[:-7], num)
    68.         downlist = Down_Tianya(myurl, num, my_dict)
    69.         downlist.start()
    70.         threads.append(downlist)
    71.    
    72.     """检查下载完成后再进行写入"""
    73.     for t in threads:
    74.         t.join()
    76.     write_text(my_dict, file_name)
    78.     print 'All download finished. Save file at directory: %s' % os.getcwd()
    80. if __name__ == '__main__':
    81.     main()
  • 相关阅读:
    操作元素
    CSS3 制作网格动画效果
    网页边栏过渡动画
    超炫的 CSS3 页面切换动画效果
    多种鼠标悬停效果
    各种 SVG 制作单选和多选框动画
    全屏遮罩层效果(10种)
    FancyBox – 经典 Lightbox 效果插件
    CSS3 & SVG 制作钟表
    20种新颖的按钮风格和效果【附源码】
  • 原文地址：https://www.cnblogs.com/highroom/p/da69c3af3d0b2be8b4db5946e2de99f1.html
Copyright © 2011-2022 走看看