python3爬虫.4.下载煎蛋网妹子图

开始我学习爬虫的目标 ----> 煎蛋网

通过设置User-Agent获取网页，发现本该是图片链接的地方被一个js函数代替了

于是全局搜索到该函数

function jandan_load_img(b){
    var d = $(b);
    var f = d.next("span.img-hash");
    var e = f.text();
    f.remove();
    var c = jdPwA5ybKhQWGy2rZybAF2StIBxrQ6NvYC(e, "pAqWmGD1GsmY5kVokg1a2eyQ3Shj1Usq");
    var a = $('<a href = "'+c.replace(/(//w+.sinaimg.cn/)(w+)(/.+.(gif|jpg|jpeg))/, "$1large$3")+'" target = "_blank" class = "view_img_link">[查看原图]</a>');
    d.before(a);
    d.before("<br>");
    d.removeAttr("onload");
    d.attr("src", location.protocol+c.replace(/(//w+.sinaimg.cn/)(w+)(/.+.gif)/, "$1thumb180$3"));
    if(/.gif$/.test(c)){
        d.attr("org_src", location.protocol+c);
        b.onload = function(){
            add_img_loading_mask(this, load_sina_gif)
        }
    }
}

该函数提取span.img-hsah传到另一个函数中，继续查找，有两个定义，于是选择靠后的那个

var jdTzcXZnL0V2WZZ8eq9786xeOdkyoBXlDR=function(m,r,d){
    var e="DECODE";
    var r=r?r:"";
    var d=d?d:0;
    var q=4;
    r=md5(r);
    var o=md5(r.substr(0,16));
    var n=md5(r.substr(16,16));
    if(q){if(e=="DECODE"){var l=m.substr(0,q)}}
    else{var l=""}
    var c=o+md5(o+l);
    var k;
    if(e=="DECODE"){m=m.substr(q);
    k=base64_decode(m)}
    var h=new Array(256);
    for(var g=0;g<256;g++){h[g]=g}
    var b=new Array();
    for(var g=0;g<256;g++){b[g]=c.charCodeAt(g%c.length)}
    for(var f=g=0;g<256;g++){f=(f+h[g]+b[g])%256;
    tmp=h[g];
    h[g]=h[f];
    h[f]=tmp}
    var t="";
    k=k.split("");
    for(var p=f=g=0;
    g<k.length;
    g++){p=(p+1)%256;
    f=(f+h[p])%256;
    tmp=h[p];
    h[p]=h[f];
    h[f]=tmp;
    t+=chr(ord(k[g])^(h[(h[p]+h[f])%256]))}
    if(e=="DECODE"){if((t.substr(0,10)==0||t.substr(0,10)-time()>0)&&t.substr(10,16)==md5(t.substr(26)+n).substr(0,16)){t=t.substr(26)}
    else{t=""}
    }
    return t
};

参考文章:　　http://www.tendcode.com/article/jiandan-meizi-spider/

其中有对js的函数的改写

最后代码如下

# -*- coding = UTF-8 -*- 
'''
目标：煎蛋网妹子图
2018/4/22
环境：pyhton3

'''


import urllib.request           #使用url处理包，urllib.request模块是用来打开和读取URLs的
import re                       #使用正则表达式
import hashlib                  #
import base64                   #
from bs4 import BeautifulSoup   #
import time                     #time
import logging                  #log
import sys                      #

'''
下载单张图片到制定的文件夹下
'''
def load_img(imgurl, file):
    name = imgurl.split('/')[-1]
    item = urllib.request.urlretrieve('http:'+imgurl,
        #'C:\Users\74172\Pictures\jandan2\%s'%(name))
        file+'\%s'%(name))
    print(name+' is loaded')

'''
md5加密
'''
def _md5(value):
    m = hashlib.md5()
    m.update(value.encode('utf-8'))
    return m.hexdigest()

'''
bash64解码
注意 原字符串长度报错问题
'''
def _base64_decode(data):
    missing_padding = 4 - len(data) % 4
    if missing_padding:
        data += '=' * missing_padding
    return base64.b64decode(data)

'''
解密获取图片链接
'''
def get_imgurl(m, r='', d=0):
    e = "DECODE"
    q = 4
    r = _md5(r)
    o = _md5(r[0:0 + 16])
    n = _md5(r[16:16 + 16])
    l = m[0:q]
    c = o + _md5(o + l)
    m = m[q:]
    k = _base64_decode(m)
    h = list(range(256))
    b = [ord(c[g % len(c)]) for g in range(256)]

    f = 0
    for g in range(0, 256):
        f = (f + h[g] + b[g]) % 256
        tmp = h[g]
        h[g] = h[f]
        h[f] = tmp

    t = ""
    p, f = 0, 0
    for g in range(0, len(k)):
        p = (p + 1) % 256
        f = (f + h[p]) % 256
        tmp = h[p]
        h[p] = h[f]
        h[f] = tmp
        t += chr(k[g] ^ (h[(h[p] + h[f]) % 256]))
    t = t[26:]
    return t

'''
获取关键字符串
'''
def get_r(js_url):
    js_respon = urllib.request.urlopen(js_url)
    js = js_respon.read().decode('utf-8')
    _r = re.findall('c=[wd]+(e,"(.*?)")', js)
    return _r

'''
获取一个页面的所有图片的链接
'''
def get_urls(url,pages,file):
    page = 0
    imagNum = 0
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
        'Host': 'jandan.net'
    }
    #########################################
    while page < pages:
        req = urllib.request.Request(url, headers=headers)
        respon = urllib.request.urlopen(req)
        html = respon.read().decode('utf-8')
        ##########################################
        js_url = 'http:' + re.findall('<script src="(//cdn.jandan.net/static/min/[wd]+.d+.js)"></script>', html)[-1]
        _r = get_r(js_url)[0]
        soup = BeautifulSoup(html, 'lxml')
        tags = soup.select('.img-hash')
        for tag in tags:
            img_hash = tag.text
            img_url = get_imgurl(img_hash,_r)        
            print(imagNum,'------>',img_url)
            imagNum = imagNum+1
            load_img(img_url,file)
        ############################################
        nextUrl = re.findall(r'Older Comments" href=".+?.#comments"',html)[0]
        print('page#',90-page,'---->done!')
        url = 'http:' + nextUrl[22:-1]
        page += 1
        time.sleep(10)
    print('done all!')
    print('located---->',file)
    
if __name__ == '__main__':
    url = 'http://jandan.net/ooxx/'
    pages = 1　　　　　　
    file = 'C:\jandan_meizi'
    get_urls(url,pages,file)

 //************2018.05.03*******************************

刚下班回来爬图，发现中间有个报错，

是get_url一次返回的多张图片链接，于是的改段

    for tag in tags:
            img_hash = tag.text
            img_urls = get_imgurl(img_hash,_r)        
            img_urls = re.findall(r'//wx.+?.jpg',img_urls)
            for img_url in img_urls:
                print(imagNum,'------>',img_url)
                imagNum = imagNum+1
                load_img(img_url,file)    

 //************2018.5.23***********************************

又报错了，暂时没有解决。