github文件的爬取

　　第一次写爬虫，爬取了简单的百度搜索界面的信息，感觉要成为爬虫大佬，要走的路还很长。所以这次就挑战一下更高的难度，爬取github上的信息。

　　过程是先在github上搜索关键词，搜索后刷新出来搜索结果的页面，再根据搜索结果页面提取出每条结果的URL链接，可根据自己想要的文件进行函数过滤或者正则过滤，保存到list里，然后逐条访问url链接，获取该url链接的页面，用xpath或者正则表达式提取自己想要的信息，最后保存到本地文件夹，欢迎大佬们批评指正呀！

#_*_encoding:utf-8_*_
import urllib
from urllib import request
import requests
from lxml.html import etree
from selenium import webdriver
import time

def page(number):
    data =  {}
#Language
    data['l'] = 'XML' 
#page    
    data['p'] = number    
#添加搜索关键词    
    data['q'] = 'xml password '  
    data['type'] = 'Code'
    url_values = urllib.parse.urlencode(data)
    url = 'https://github.com/search?'
    full_url = url + url_values
    full_url = urllib.parse.unquote(full_url)
    print(full_url)
    return full_url 

#获取url页面
def pro(item):        
    
    driver.get(item)                    
    content = driver.page_source
    try :
        content = content.decode('UTF-8')
    except:
        print("爬取成功")
    else:
        print("解码...")
#构造一个XPath解析对象并对HTML文本进行自动修正        
    content = etree.HTML(content)
    return content

def makeurl(content):        #获取页面url的链接
#找到大标签
    codelist = content.xpath("/html/body/div[4]/main/div/div[3]/div/div[2]/div[1]/*") 
#大标签内有10个链接    
    for item in range(len(codelist)):                            
        url = codelist[item].xpath("./div[1]/div[2]/a/@href")
        url = "https://github.com"+url[0]
        check(url)

#获取xml文件
def getxmlcontent(count):            
    driver.get(urls[count])
    time.sleep(2)
    print(urls[count])
    content = driver.page_source
    try :
        content = content.decode('UTF-8')
    except:
        print("爬取成功")
    else:
        print("继续...")
    content = etree.HTML(content)
#找到大标签
    xmlcontent = content.xpath("/html/body/div[4]/div/main/div[3]/div/div[3]/div[2]/table/tbody/*")     
#存储xml文件
    xmlcontentss = [] 
#逐行提取    
    for item in range(len(xmlcontent)):  
        xmlcontents = ''.join(xmlcontent[item].xpath(".//td[2]//text()"))
        xmlcontentss.append(xmlcontents+"
")
#转化为字符串    
    xmlcontentss = ''.join(xmlcontentss)      
    return xmlcontentss

#将爬取的数据写入文件
def save(i,xmlcontents):    
    filename = time.strftime('%Y%m%d-%H%M%S ',time.localtime(time.time()))
    filename = str(filename)                        
    file =r"D:Myfilesxmlwhitegitxml-14\" +filename+ ".xml"
    f = open(file,'a+')
    f.write(xmlcontents)
    f.close()        

#检查是否为xml后缀的文件
def check(isxml):    
    if isxml.endswith("xml") :
        urls.append(isxml)    


if __name__ == "__main__":
    baseurl = "https://github.com/login"
    driver = webdriver.Chrome()
    driver.get(baseurl)
    driver.find_element_by_id("login_field").send_keys("账号")
    driver.find_element_by_id("password").send_keys("密码")
    driver.find_element_by_name("commit").click()
    time.sleep(20)
    i = 0
    s = 0
    k = 0
#github上最多只显示100页搜索结果，要想爬取到一定数量的文件是不可能的了。。。。可以更换关键词搜索。。。。    
    for pitem in range(100):        
        urls =[]
        makeurl(pro(page(pitem+1)))
        print(urls)
        print("-----------------第%d页"%(pitem+1))    
        for item in range(len(urls)):
            i = i+1    
            try:
                xmlcontents = getxmlcontent(item)
                if xmlcontents :
                    s = s + 1
                    save(s,xmlcontents)
                else :
                    k = k+1
                    print("-----------------当前文件内容为空！")    
            except:
                print("-----------------写入失败...")
            else:
                print("-----------------继续抓取ing")
            print("第%d页：正在抓取第%d个文件,已经抓取到%s个文件，空数据文件%d个，抓取下一个文件..."%((pitem+1),i,s,k))
    print("-----------------此次数据爬取已完成！--------------")