zoukankan      html  css  js  c++  java
  • 有道语料库爬虫

    from bs4 import BeautifulSoup
    import urllib.parse
    import re
    import requests
    import time

    index = 0
    session = requests.session()
    f = open('C:\Users\Administrator\Desktop\dictionary\words.txt','r',encoding='gb2312')
    output = open("C:\Users\Administrator\Desktop\dictionary\output.txt", "w")
    for line in f.readlines():
    time.sleep(10)
    if(line != ' '):
    m = re.match(r'([a-zA-Z ]+)', line)
    if m:
    index += 1
    url = 'http://dict.youdao.com/example/blng/eng/'+ m.group(1) + '/#keyfrom=dict.main.moreblng'
    s = '(' + str(index) + ')銆? + line + " "
    print("%s"% (s) ,file = output)
    soup = BeautifulSoup(session.get(url).text)
    blingual = soup.findAll(id = 'bilingual')
    if blingual:
    ol = blingual[0].ul
    if ol:
    ul = ol.findAll('li')
    if(len(ul)<6):
    num = len(ul)
    else:
    num = 6
    for i in range(num):
    li = ul[i]
    if li.p:
    if li.p.a:
    if li.p.a.get('data-rel'):
    s1 = urllib.parse.unquote(li.p.a['data-rel'].replace('+',' ').split('.')[0])
    print(s1+" ")
    print("%s" % (s1), file = output)
    if len(li.findAll('p')) > 1:
    s2 = urllib.parse.unquote(li.findAll('p')[1].get_text().split()[0]).encode("gb18030").decode('gbk','ignore')
    print("%s" % (s2), file = output)
    print(" ", file = output)

    由于这里格式变化了。所以,附上我的github链接。

    https://github.com/myheed/Spider

  • 相关阅读:
    数组元素的查找1
    排序——选择排序
    排序——冒泡排序
    内部类——匿名内部类
    跳一跳
    数组元素换位置游戏
    六 java和Tomcat
    九 Jenkins持续集成
    八 ip和子网详解
    七 Git版本控制
  • 原文地址:https://www.cnblogs.com/Alex0111/p/7662242.html
Copyright © 2011-2022 走看看