selenium 爬取空间说说

package cn.hb.util;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.Keys;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.firefox.FirefoxOptions;
import org.openqa.selenium.interactions.Actions;

/**
 * 爬取说说写入到txt中,爬取100条
 * 
 * @author tele
 *
 */
public class QZTwitterCrawler {
    static String url = "https://user.qzone.qq.com/1350560858";
    static int maxSize = 100;
    static int pageSize = 20;
    static String userName="qq"; 
    static String pwd = "密码";
    public static void main(String[] args) throws InterruptedException, IOException {
        login();
    }

    /**
     * 登录
     * 
     * @throws InterruptedException
     * @throws IOException
     */
    public static void login() throws InterruptedException, IOException {
        System.setProperty("webdriver.gecko.driver", "D:/browserdriver/geckodriver.exe");

        FirefoxOptions options = new FirefoxOptions();
        options.setBinary("F:/ff/firefox.exe");

        WebDriver driver = new FirefoxDriver(options);
        driver.manage().window().maximize();
        // 超时
        try {
            driver.manage().timeouts().pageLoadTimeout(3, TimeUnit.SECONDS);
            driver.manage().timeouts().setScriptTimeout(3, TimeUnit.SECONDS);
            driver.get(url);
        } catch (Exception e) {
            System.out.println("所需元素已出现,停止加载页面");
        } finally {
            // 切换到登录login
            driver.switchTo().frame("login_frame");

            WebElement switcher_plogin = driver.findElement(By.id("switcher_plogin"));
            System.out.println(switcher_plogin.getText());
            if (switcher_plogin.isDisplayed()) {
                switcher_plogin.click();
            }
            // 用户名
            driver.findElement(By.id("u")).clear();
            driver.findElement(By.id("u")).sendKeys(userName);

            // 密码
            driver.findElement(By.id("p")).clear();
            driver.findElement(By.id("p")).sendKeys(pwd);

            // 登录
            try {
                driver.findElement(By.id("login_button")).click();
                Thread.sleep(3000);
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if ("https://i.qq.com/".equals(driver.getCurrentUrl())) {
                    System.out.println("登录失败!5秒后再次尝试登录");
                    Thread.sleep(5000);
                    driver.findElement(By.id("login_button")).click();
                }
            }

            // 退出frame
            driver.switchTo().defaultContent();

            System.out.println(driver.getCurrentUrl());

            JavascriptExecutor jsExecutor = (JavascriptExecutor) driver;
            // 如果有亲密度提示
            
              try { WebElement fs_guide = driver.findElement(By.xpath(
              "//div[@id='friendship_promote_layer']/table[@class='tbl-fs-guide']//a"
             )); if(fs_guide != null && fs_guide.isDisplayed()) {
              fs_guide.click(); } } catch (Exception e) { e.printStackTrace();
              }finally {
             
              }
             

            // 点击说说
            driver.findElement(By.cssSelector("#menuContainer ul.head-nav-menu>li.menu_item_311>a")).click();

            Thread.sleep(2000);

            // 切换到frame
            driver.switchTo().frame(driver.findElement(By.className("app_canvas_frame")));

            Thread.sleep(5000);

            // 拼接cookie
        /*    StringBuilder builder = new StringBuilder();
            Set<Cookie> cookieSet = driver.manage().getCookies();
            cookieSet.forEach(c -> builder.append(c.getName()).append("=").append(c.getValue()).append("; "));
            cookies = builder.toString();*/

            // 保存
            saveTwitter(driver);

            System.out.println("内容提取完毕,退出浏览器");
            driver.quit();

        }
    }

    /**
     * 序列化
     * @param driver
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    public static void saveTwitter(WebDriver driver) throws InterruptedException, IOException {
        File file = new File("f:/qz/twitter.txt");

        // 文件夹检测
        if (!file.getParentFile().exists()) {
            file.mkdirs();
        } else {
            file.delete();
        }

        
        
        FileWriter fileWriter = new FileWriter(file, true);

        String xpath;
        // 模拟按键进行滚动
        Actions actions = new Actions(driver);

        
        //说说总量
        String totalNumStr = driver.findElement(By.xpath("//div[@class='feed_num']/a")).getText();
        int totalNum = Integer.parseInt(totalNumStr);
        
        // 计算页数
        int totalPage = (int) Math.ceil((double)Math.min(maxSize, totalNum) / (double) pageSize);

        // 构造xpath
        for (int i = 0; i < totalPage; i++) {

            for (int j = 0; j < pageSize; j++) {
                xpath = "//ol[@id='msgList']/li[" + (j + 1) + "]/div[3]/div[2]/pre[@class='content']";
                // 获取说说内容
                try {
                    WebElement element = driver.findElement(By.xpath(xpath));
                    String text = element.getText();
                    System.out.println("本页第" + (j + 1) + "条   :" + text);
                    fileWriter.write(text, 0, text.length());

                } catch (Exception e) {
                    e.printStackTrace();
                } finally {

                }
                if (j % 2 == 0) {
                    actions.sendKeys(Keys.ARROW_DOWN).perform();
                }
            }
            System.out.println("第" + (i + 1) + "页说说爬取完毕");
            // 分页
            if ((i + 2) <= totalPage) {
                driver.findElement(By.xpath("//a[@id='pager_num_" + i + "_" + (i + 2) + "']")).click();
                // 等待页面加载
                Thread.sleep(3000);
            }
        }

        if (fileWriter != null) {
            fileWriter.close();
        }
    }

}

比爬取相册简单点,唯一有点弯的是页码的构造了,我写的这个只支持获取文字,可以用来生成词云