Java模拟登陆新浪微博抓取数据【转载】

package com.shiyimm.crawler.weibo;

import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.script.Invocable;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;

import net.sf.json.JSONObject;

import org.apache.commons.codec.binary.Base64;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;

import com.shiyimm.crawler.util.MyUrlUtil;
import com.shiyimm.crawler.util.UrlUtil;

public class SinaWeibo {
    private HttpClient client;
    private String username;    //登录帐号(明文)
    private String password;    //登录密码(明文)
    private String su;            //登录帐号(Base64加密)
    private String sp;            //登录密码(各种参数RSA加密后的密文)
    private long servertime;    //初始登录时，服务器返回的时间戳,用以密码加密以及登录用
    private String nonce;        //初始登录时，服务器返回的一串字符，用以密码加密以及登录用
    private String rsakv;        //初始登录时，服务器返回的一串字符，用以密码加密以及登录用
    private String pubkey;        //初始登录时，服务器返回的RSA公钥
    
    private String errInfo;        //登录失败时的错误信息
    private String location;    //登录成功后的跳转连接
    
    private String url;
    
    public SinaWeibo(String username,String password){
        client = new DefaultHttpClient();
        this.username = username;
        this.password = password;
    }
    
    
    /**
     * 初始登录信息<br>
     * 返回false说明初始失败
     * @return
     */
    public boolean preLogin(){
        boolean flag = false;
        try {
            su = new String(Base64.encodeBase64(URLEncoder.encode(username, "UTF-8").getBytes()));
            String url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&rsakt=mod&checkpin=1&" +
                    "client=ssologin.js(v1.4.5)&_="+getTimestamp();
            url += "&su="+su;
            String content;
            content = HttpTools.getRequest(client, url);
            //System.out.println(content);
            System.out.println("content------------"+content);
            JSONObject json = JSONObject.fromObject(content);
            System.out.println(json);
            servertime = json.getLong("servertime");
            nonce = json.getString("nonce");
            rsakv = json.getString("rsakv");
            pubkey = json.getString("pubkey");
            flag = encodePwd();
        } catch (UnsupportedEncodingException e) {
            // TODO Auto-generated catch block
            //e.printStackTrace();
        } catch (ClientProtocolException e) {
            // TODO Auto-generated catch block
            //e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            //e.printStackTrace();
        }
        return flag;
    }
    
    /**
     * 登录
     * @return true:登录成功
     */
    public boolean login(){
        if(preLogin()){
            String url = "http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.5)";
            List<NameValuePair> parms = new ArrayList<NameValuePair>();
            parms.add(new BasicNameValuePair("entry", "weibo"));
            parms.add(new BasicNameValuePair("geteway", "1"));
            parms.add(new BasicNameValuePair("from", ""));
            parms.add(new BasicNameValuePair("savestate", "7"));
            parms.add(new BasicNameValuePair("useticket", "1"));
            parms.add(new BasicNameValuePair("pagerefer", "http://login.sina.com.cn/sso/logout.php?entry=miniblog&r=http%3A%2F%2Fweibo.com%2Flogout.php%3Fbackurl%3D%2F"));
            parms.add(new BasicNameValuePair("vsnf", "1"));
            parms.add(new BasicNameValuePair("su", su));
            parms.add(new BasicNameValuePair("service", "miniblog"));
            parms.add(new BasicNameValuePair("servertime", servertime+""));
            parms.add(new BasicNameValuePair("nonce", nonce));
            parms.add(new BasicNameValuePair("pwencode", "rsa2"));
            parms.add(new BasicNameValuePair("rsakv", rsakv));
            parms.add(new BasicNameValuePair("sp", sp));
            parms.add(new BasicNameValuePair("encoding", "UTF-8"));
            parms.add(new BasicNameValuePair("prelt", "182"));
            parms.add(new BasicNameValuePair("url", "http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack"));
            parms.add(new BasicNameValuePair("returntype", "META"));
            try {
                String content = HttpTools.postRequest(client, url, parms);
                System.out.println("content----------"+content);
                String regex = "location\.replace\("(.+?)"\);";
                Pattern p = Pattern.compile(regex);
                Matcher m = p.matcher(content);
                if(m.find()){
                    location = m.group(1);
                    if(location.contains("reason=")){
                        errInfo = location.substring(location.indexOf("reason=")+7);
                        errInfo = URLDecoder.decode(errInfo, "GBK");
                    }else{
                        String result = HttpTools.getRequest(client, location);
                        System.out.println("result--------------"+result);
                        return true;
                    }
                }
            } catch (ClientProtocolException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
//            url = "http://www.weibo.com/hm";
//            System.out.println(MyUrlUtil.getResource(url));
        }
        return false;
    }
    
    /**
     * 密码进行RSA加密&lt;br&gt;
     * 返回false说明加密失败
     * @return
     */
    private boolean encodePwd(){
        ScriptEngineManager sem = new ScriptEngineManager();
        ScriptEngine se = sem.getEngineByName("javascript");
        try {
            FileReader fr = new FileReader("E:\encoder.js");
            se.eval(fr);
            Invocable invocableEngine = (Invocable) se;
            String callbackvalue = (String) invocableEngine.invokeFunction("encodePwd",pubkey,servertime,nonce,password);
            sp = callbackvalue;
            return true;
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            System.out.println("加密脚本encoder.sj未找到");
        } catch (ScriptException e) {
            // TODO Auto-generated catch block
            //e.printStackTrace();
        } catch (NoSuchMethodException e) {
            // TODO Auto-generated catch block
            //e.printStackTrace();
        }
        errInfo = "密码加密失败!";
        return false;
    }
    
    public String getErrInfo() {
        return errInfo;
    }
    
    /**
     * 获取时间戳
     * @return
     */
    private long getTimestamp(){
        Date now = new Date();
        return now.getTime();
    }
    
    public static void main(String[] args) throws ClientProtocolException, IOException {
        SinaWeibo weibo = new SinaWeibo("账号", "密码");
        if(weibo.login()){
            System.out.println("登陆成功！");
            String url = "http://www.weibo.com/hm";
//            String source = MyUrlUtil.getResource(url);
//            System.out.println(source);
        }else{
            System.out.println("登录失败！");
        }
    }
}
<pre class="java; toolbar: true; auto-links: false;">package com.shiyimm.crawler.weibo;

import java.io.IOException;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.util.EntityUtils;

public class HttpTools {
    /**
     * 正常GET方式HTTP请求
     * @param client
     * @param url
     * @return
     * @throws ClientProtocolException
     * @throws IOException
     */
    public static String getRequest(HttpClient client,String url) throws ClientProtocolException, IOException{
        HttpGet get = new HttpGet(url);
        get.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11");
        //get.addHeader(&quot;Referer&quot;, &quot;http://2013.weibo.com/&quot;);
        HttpResponse response = client.execute(get);
        HttpEntity entity = response.getEntity();
        String content = EntityUtils.toString(entity,"GBK");
        //System.out.println(content);
        /*EntityUtils.consume(entity);*/
        return content;
    }
    
    /**
     * 正常POST方式HTTP请求
     * @param client
     * @param url
     * @param parms
     * @return
     * @throws ClientProtocolException
     * @throws IOException
     */
    public static String postRequest(HttpClient client,String url,List<NameValuePair> parms) throws ClientProtocolException, IOException{
        HttpPost post = new HttpPost(url);
        post.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11");
        post.addHeader("Content-Type", "application/x-www-form-urlencoded");
        //post.addHeader(&quot;Referer&quot;, &quot;http://2013.weibo.com/&quot;);
        UrlEncodedFormEntity postEntity = new UrlEncodedFormEntity(parms, "UTF-8");
        post.setEntity(postEntity);
        HttpResponse response = client.execute(post);
        HttpEntity entity = response.getEntity();
        String content = EntityUtils.toString(entity,"GBK");
        /*EntityUtils.consume(entity);*/
        return content;
    }
}