世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态

利用这种方法，可以把所有会员信息存储下来，多线程发信息，10秒钟就可以对几百个会员完成发信了。

首先是筛选信息后爬取账号信息，

#-*-coding:utf-8-*-
import requests,re,json,time,threadpool,os
from mydba import MySql
from gevent import monkey


#monkey.patch_all()

header={
    'Cookie':'guider_quick_search=on; SESSION_HASH=xxxxxxxxcd4523713c3d3700579350daa9992f1a; user_access=1; save_jy_login_name=1314xxxx; sl_jumper=%26cou%3D17; last_login_time=1498576743; user_attr=000000; pclog=%7B%22160961843%22%3A%221498576757268%7C1%7C0%22%7D; IM_S=%7B%22IM_CID%22%3A8677484%2C%22svc%22%3A%7B%22code%22%3A0%2C%22nps%22%3A0%2C%22unread_count%22%3A%220%22%2C%22ocu%22%3A0%2C%22ppc%22%3A0%2C%22jpc%22%3A0%2C%22regt%22%3A%221486465367%22%2C%22using%22%3A%2240%2C33%2C2%2C%22%2C%22user_type%22%3A%2210%22%2C%22uid%22%3A160961843%7D%2C%22IM_SV%22%3A%22123.59.161.3%22%2C%22m%22%3A26%2C%22f%22%3A0%2C%22omc%22%3A0%7D; FROM_BD_WD=%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598; FROM_ST_ID=416640; FROM_ST=.jiayuan.com; REG_ST_ID=15; REG_ST_URL=http://bzclk.baidu.com/adrc.php?t=06KL00c00f7t0wC0Gfum0QkHAsjnX7Fu00000PNeYH300000uybcI1.THL2sQ1PEPZRVfK85yF9pywd0ZnqryRkryfdryDsnj0kuWR3u0Kd5RuKP17Knj97rDDvPRNAPH9APbRkwHc4njIAnDPjP1NA0ADqI1YhUyPGujYzrH0dnWTYnHckFMKzUvwGujYkP6K-5y9YIZ0lQzqzuyT8ph-9XgN9UB4WUvYETLfE5v-b5HfkPWmYnaudThsqpZwYTjCEQLILIz4Jpy74Iy78QhPEUfKWThnqnWRdnjn&tpl=tpl_10762_15668_1&l=1053916117&attach=location%3D%26linkName%3D%25E6%25A0%2587%25E9%25A2%2598%26linkText%3D%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598%25E7%25BD%2591%25EF%25BC%2588Jiayuan.com%25EF%25BC%2589%253A2017%25EF%25BC%258C%26xp%3Did(%2522m78828180%2522)%252FDIV%255B1%255D%252FDIV%255B1%255D%252FDIV%255B1%255D%252FH2%255B1%255D%252FA%255B1%255D%26linkType%3D%26checksum%3D191&ie=UTF-8&f=8&tn=baidu&wd=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&oq=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&rqlang=cn; REG_REF_URL=http://www.jiayuan.com/usercp/profile.php?action=work; PHPSESSID=d5557cc5a4560c14c2dc685bcb7fc009; stadate1=159xxxx; myloc=44%7C4403; myage=xx; PROFILE=160xxxxxx%3A%25E9%25A3%258E%3Am%3Aat1.jyimg.com%2F42%2Ffb%2F4558c12f370e446240ba148bcac4%3A1%3A%3A1%3A4558c12f3_2_avatar_p.jpg%3A1%3A1%3A61%3A10; mysex=m; myuid=159xxxxxxx; myincome=40; mylevel=2; main_search:160961843=%7C%7C%7C00; RAW_HASH=J3ewrCGVZG5eU2agrymP2-bNz0IQQpivXOncdsOO63oS3%2AH4o4%2AsAifV0twuNFXqWippm3rMHXvTK%2APmGbap-ZYZpz-18ogWLnBXkcpY87GlFps.; COMMON_HASH=424558c12fxxxxx40ba148bcac4fb; IM_CON=%7B%22IM_TM%22%3A1498577459991%2C%22IM_SN%22%3A3%7D; IM_M=%5B%5D; pop_time=1498577573654; IM_CS=2; IM_ID=12; is_searchv2=1; IM_TK=1498577921417',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
}

def fun(i):

    url='http://search.jiayuan.com/v2/search_v2.php?sex=f&key=&stc=1%3A4403%2C2%3A23.27%2C3%3A155.170%2C23%3A1&sn=default&sv=1&p='+str(i)+'&f=select&listStyle=bigPhoto&pri_uid=xxxx&jsversion=v5'

    while(1):
        res=''
        try:
            res=requests.get(url,headers=header)
        except Exception,e:
            print e
        if res!='':
            break



    content=res.content
    dictx= json.loads(re.findall('##jiayser##([sS]*?)##jiayser##',content)[0])   ##content是一个json但加了其他东西，需要把json提取出来
    #print dictx
    for dcx in dictx['userInfo']:
        #print dcx
        uhash=dcx['helloUrl'].encode('utf8').split('uhash=')[-1]
        crawled_time=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
        #print uhash
        sql='''insert into shijijiayuan values(NULL,"%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")'''%(dcx['nickname'],dcx['age'],dcx['sex'],dcx['education'],dcx['height'],uhash,dcx['image'],dcx['marriage'],dcx['matchCondition'],
        dcx['uid'],dcx['work_location'],dcx['work_sublocation'],dcx['shortnote'],crawled_time)
        #print '要插入的sql语句是：',sql
        if not os.path.exists('images/%s_%s.jpg'%(dcx['nickname'],dcx['uid'])):
            pass

            while(1):
                res2=''
                try:
                    res2=requests.get(dcx['image'],headers=header,timeout=30)
                except Exception,e:
                    print e
                if res2!='':
                    break
            try:
                with open('images/%s_%s.jpg'%(dcx['nickname'],dcx['uid']),'wb') as f:
                    f.write(res2.content)
                    print '%s保存图片成功%s'%(dcx['uid'])
            except Exception,e:

                print '%s保存图片失败%s'%(dcx['uid'],str(e))

        sql2 = 'select 1 from shijijiayuan where uid=%s'%dcx['uid']

        mysqlx2 = MySql('localhost', 'root', '123456', 'test', 'utf8')
        mysqlx2.query(sql2)         
        is_exist = mysqlx2.cursor.fetchone()            ##判断下数据库有没有这个uid，uid是账号的唯一
        if is_exist==(1,):
            print '已存在%s'%dcx['uid']
        else:
            mysqlx2.query(sql)

        del mysqlx2



pool = threadpool.ThreadPool(300)              ##用thredpool线程池开300线程
requestsx = threadpool.makeRequests(fun,[i for i in range(1,500)])
[pool.putRequest(req) for req in requestsx]
pool.wait()

执行这个后，就可以看到数据库里面会有很多会员信息了。

附上建表语句

CREATE TABLE `shijijiayuan` (
`id` int(10) NOT NULL AUTO_INCREMENT,
`nickname` varchar(20) DEFAULT NULL,
`age` int(5) DEFAULT NULL,
`sex` varchar(5) DEFAULT NULL,
`education` varchar(30) DEFAULT NULL,
`height` int(5) DEFAULT NULL,
`uhash` varchar(50) DEFAULT NULL,
`image` varchar(160) DEFAULT NULL,
`marriage` varchar(10) DEFAULT NULL,
`matchCondition` varchar(200) DEFAULT NULL,
`uid` int(15) DEFAULT NULL,
`work_location` varchar(10) DEFAULT NULL,
`work_sublocation` varchar(10) DEFAULT NULL,
`shortnote` varchar(1000) DEFAULT NULL,
`crawled_time` datetime DEFAULT NULL,
`last_hello_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`hello_times` int(3) DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8

这个是下载的图片。

 第二部分是对会员发送信息。

#-*-coding:utf-8-*-
import requests,re,json,time,threadpool,os,random
from concurrent.futures import ThreadPoolExecutor
from mydba import MySql
from gevent import monkey
from greetings import greeting_list

header={
    'Cookie':'guider_quick_search=on; SESSION_HASH=yyyyyyya6f4b944ea522fb669bd758f17ea00; FROM_BD_WD=%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598; FROM_ST_ID=416640; FROM_ST=.jiayuan.com; REG_ST_ID=15; REG_ST_URL=http://bzclk.baidu.com/adrc.php?t=06KL00c00f7t0wC0Gfum0QkHAsjb97wu00000PNeYH300000v9ZrmW.THL2sQ1PEPZRVfK85yF9pywd0Znqrj6vPWm1nH0snj0sm1N9PsKd5HTYwHTvrRmLfH9anDmLwHFjfRwArj6sfRcLPWTkn1610ADqI1YhUyPGujYzrH0dnWTYnHckFMKzUvwGujYkP6K-5y9YIZ0lQzqzuyT8ph-9XgN9UB4WUvYETLfE5v-b5HfkPWmYnaudThsqpZwYTjCEQLILIz4Jpy74Iy78QhPEUfKWThnqPWfdns&tpl=tpl_10762_15668_1&l=1053916117&attach=location%3D%26linkName%3D%25E6%25A0%2587%25E9%25A2%2598%26linkText%3D%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598%25E7%25BD%2591%25EF%25BC%2588Jiayuan.com%25EF%25BC%2589%253A2017%25EF%25BC%258C%26xp%3Did(%2522m78828180%2522)%252FDIV%255B1%255D%252FDIV%255B1%255D%252FDIV%255B1%255D%252FH2%255B1%255D%252FA%255B1%255D%26linkType%3D%26checksum%3D191&ie=UTF-8&f=8&tn=baidu&wd=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&oq=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&rqlang=cn; user_access=1; _gscu_1380850711=98711257z5x28w21; _gscbrs_1380850711=1; save_jy_login_name=xxxxx; last_login_time=1498711280; user_attr=000000; pclog=%7B%22160961843%22%3A%221498711275466%7C1%7C0%22%7D; IM_S=%7B%22IM_CID%22%3A7816174%2C%22IM_SV%22%3A%22123.59.161.3%22%2C%22svc%22%3A%7B%22code%22%3A0%2C%22nps%22%3A0%2C%22unread_count%22%3A%221%22%2C%22ocu%22%3A0%2C%22ppc%22%3A0%2C%22jpc%22%3A0%2C%22regt%22%3A%221486465367%22%2C%22using%22%3A%2240%2C33%2C2%2C%22%2C%22user_type%22%3A%2210%22%2C%22uid%22%3A160961843%7D%2C%22m%22%3A12%2C%22f%22%3A0%2C%22omc%22%3A0%7D; REG_REF_URL=http://www.jiayuan.com/msg/?from=menu; PHPSESSID=8902adb4512b9e8904156c8b4267dc4c; stadate1=159961843; myloc=44%7C4403; myage=xx; PROFILE=160961843%3A%25E9%25A3%258E%3Am%3Aat1.jyimg.com%2F42%2Ffb%2F4558c12f370e446240ba148bcac4%3A1%3A%3A1%3A4558c12f3_2_avatar_p.jpg%3A1%3A1%3A61%3A10; mysex=m; myuid=15xxxx; myincome=40; mylevel=2; main_search:160xxxx=%7C%7C%7C00; RAW_HASH=suldoq0mMteEqQAs3qWKnMx-9sll5N8EDN-tu7bwGBA%2AtITMUwUbnL59YPR8ydVDXCxjt19iNPlu8kzi-jp8cU%2AzbhYS1iuOJyt%2APg32BhCebEE.; COMMON_HASH=424xxxx446240ba148bcac4fb; IM_CS=2; IM_ID=3; is_searchv2=1; pop_time=1498713117447; IM_CON=%7B%22IM_TM%22%3A1498713128148%2C%22IM_SN%22%3A9%7D; IM_M=%5B%5D; IM_TK=1498713324458',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
}


mysqlx = MySql('localhost', 'root', '123456', 'test', 'utf8')

mysqlx.query("SELECT nickname,uhash FROM shijijiayuan order by id desc  limit 10,5")   #SELECT * FROM shijijiayuan where hello_times=0 limit 1,1 #select语句选取几个账号发信
fetchallx= mysqlx.cursor.fetchall()
nickname_uhash_list=[ft for ft in fetchallx] 


url2='http://www.jiayuan.com/msg/dosend.php?type=hello&randomfrom=4'
url3='http://www.jiayuan.com/165508547'
print nickname_uhash_list
def hello(nickname,uhash):


    datax2={
    'textfield':'嗨。%s妹子。'%(nickname)+random.choice(greeting_list),   ##问候语是一个列表，随机发送一个信息给他。
    'list':'',
    'category':'2',
    'xitong_zidingyi_wenhouyu':'1',
    'xitong_zidingyi_wenhouyu':'0',
    'new_type':'1',
    'pre_url':'',
    'sendtype':'20',
    'hellotype':'hello',
    'pro_id':'0',
    'new_profile':'3',
    'to_hash':uhash,       #9c248d2c96868695640e76ed74e9d3b0   f11e574d44c7967b88afb9940392c14b uhash的值决定对谁发送信息，不是uid来决定，接口中没有uid参数
    'fxly':'search_v2',
    'tj_wz':'none',
    'need_fxtyp_tanchu':'0',
    'self_pay':'0',
    'fxbc':'0',
    'cai_xin':'0',
    'zhuanti':'0',
    'liwu_nofree':'0',
    'liwu_nofree_id':'88',
    }
    try:
        contentx='空'
        resx= requests.post(url2,data=datax2,headers=header)
        contentx= resx.content
        print contentx
        if  '发送成功' in contentx:    ##如果发送信息成功，就把数据库中的最后发信时间更新下，然后把发信次数加1，以便于之后用slect语句选定没有发信的人对其发信
            mysqlx2 = MySql('localhost', 'root', '123456', 'test', 'utf8')
            #update student set score=score+1 where id = 1
            sql='''UPDATE shijijiayuan  SET hello_times=hello_times+1 WHERE  uhash="%s"'''%(uhash)
            #print sql
            mysqlx2.query(sql)
            del  mysqlx2
    except Exception,e:
        print e



poolx = ThreadPoolExecutor(max_workers=10)  ##这也是多线程发信，世纪佳缘网站发信接口，响应时间非常巨大，浪费时间，用多线程Threadpoolexurot非常好的包，是未来要做的事，把一切东西扔到这里面，本篇没有体现出比threadpool好的优点。


[poolx.submit(hello,i[0],i[1]) for i in nickname_uhash_list]

执行完这个脚本后，在网页或者app中就可以看到发的信息了。 

这个是打了招呼之后表的变化。

没有什么难度，就是要分析，首先是分析账号信息在哪里，是ajax，返回的是json加了一些字符，把json提取出来，转成字典，比较好弄。

然后就是要分析发信接口是哪一个url，这个根据规律猜测下然后查看下post的值就能找到，然后就是找出接口中的关键字段，关键字段不多，uhash是决定发送给谁（尝试几下就可以），textfield是发送的内容。

本文的登录是headers中的cookie字段携带。