网络爬虫速成指南 （四） URL判重

如果采集量比较小：
布隆过滤器详解原理

如果采集量比较大：
redis：主要是把URL转为md5，作为key来进行判重


关于布隆过滤器器的参数说明，简单点说：
private static BloomFilter<String> bloomFilter = new BloomFilter<String>(2X, X);
这样用就可以了

附：布隆过滤器实现，@author Magnus Skjegstad <magnus@skjegstad.com>

import java.io.Serializable;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.BitSet;
import java.util.Collection;

/**
 * Implementation of a Bloom-filter, as described here:
 * http://en.wikipedia.org/wiki/Bloom_filter
 * 
 * Inspired by the SimpleBloomFilter-class written by Ian Clarke. This
 * implementation provides a more evenly distributed Hash-function by using a
 * proper digest instead of the Java RNG. Many of the changes were proposed in
 * comments in his blog:
 * http://blog.locut.us/2008/01/12/a-decent-stand-alone-java
 * -bloom-filter-implementation/
 * 
 * @param <E>
 *            Object type that is to be inserted into the Bloom filter, e.g.
 *            String or Integer.
 * @author Magnus Skjegstad <magnus@skjegstad.com>
 */
public class BloomFilter<E> implements Serializable {
    private BitSet bitset;
    private int bitSetSize;
    private double bitsPerElement;
    private int expectedNumberOfFilterElements; // expected (maximum) number of
                                                // elements to be added
    private int numberOfAddedElements; // number of elements actually added to
                                        // the Bloom filter
    private int k; // number of hash functions

    static final Charset charset = Charset.forName("UTF-8"); // encoding used
                                                                // for storing
                                                                // hash values
                                                                // as strings

    static final String hashName = "MD5"; // MD5 gives good enough accuracy in
                                            // most circumstances. Change to
                                            // SHA1 if it's needed
    static final MessageDigest digestFunction;
    static { // The digest method is reused between instances
        MessageDigest tmp;
        try {
            tmp = java.security.MessageDigest.getInstance(hashName);
        } catch (NoSuchAlgorithmException e) {
            tmp = null;
        }
        digestFunction = tmp;
    }

    /**
     * Constructs an empty Bloom filter. The total length of the Bloom filter
     * will be c*n.
     * 
     * @param c
     *            is the number of bits used per element.
     * @param n
     *            is the expected number of elements the filter will contain.
     * @param k
     *            is the number of hash functions used.
     */
    public BloomFilter(double c, int n, int k) {
        this.expectedNumberOfFilterElements = n;
        this.k = k;
        this.bitsPerElement = c;
        this.bitSetSize = (int) Math.ceil(c * n);
        numberOfAddedElements = 0;
        this.bitset = new BitSet(bitSetSize);
    }

    /**
     * Constructs an empty Bloom filter. The optimal number of hash functions
     * (k) is estimated from the total size of the Bloom and the number of
     * expected elements.
     * 
     * @param bitSetSize
     *            defines how many bits should be used in total for the filter.
     * @param expectedNumberOElements
     *            defines the maximum number of elements the filter is expected
     *            to contain.
     */
    public BloomFilter(int bitSetSize, int expectedNumberOElements) {
        this(bitSetSize / (double) expectedNumberOElements,
                expectedNumberOElements, (int) Math
                        .round((bitSetSize / (double) expectedNumberOElements)
                                * Math.log(2.0)));
    }

    /**
     * Constructs an empty Bloom filter with a given false positive probability.
     * The number of bits per element and the number of hash functions is
     * estimated to match the false positive probability.
     * 
     * @param falsePositiveProbability
     *            is the desired false positive probability.
     * @param expectedNumberOfElements
     *            is the expected number of elements in the Bloom filter.
     */
    public BloomFilter(double falsePositiveProbability,
            int expectedNumberOfElements) {
        this(Math.ceil(-(Math.log(falsePositiveProbability) / Math.log(2)))
                / Math.log(2), // c = k / ln(2)
                expectedNumberOfElements, (int) Math.ceil(-(Math
                        .log(falsePositiveProbability) / Math.log(2)))); // k =
                                                                            // ceil
                                                                            // (
                                                                            // -
                                                                            // log_2
                                                                            // (
                                                                            // false
                                                                            // prob
                                                                            // .
                                                                            // )
                                                                            // )
    }

    /**
     * Construct a new Bloom filter based on existing Bloom filter data.
     * 
     * @param bitSetSize
     *            defines how many bits should be used for the filter.
     * @param expectedNumberOfFilterElements
     *            defines the maximum number of elements the filter is expected
     *            to contain.
     * @param actualNumberOfFilterElements
     *            specifies how many elements have been inserted into the
     *            <code>filterData</code> BitSet.
     * @param filterData
     *            a BitSet representing an existing Bloom filter.
     */
    public BloomFilter(int bitSetSize, int expectedNumberOfFilterElements,
            int actualNumberOfFilterElements, BitSet filterData) {
        this(bitSetSize, expectedNumberOfFilterElements);
        this.bitset = filterData;
        this.numberOfAddedElements = actualNumberOfFilterElements;
    }

    /**
     * Generates a digest based on the contents of a String.
     * 
     * @param val
     *            specifies the input data.
     * @param charset
     *            specifies the encoding of the input data.
     * @return digest as long.
     */
    public static long createHash(String val, Charset charset) {
        return createHash(val.getBytes(charset));
    }

    /**
     * Generates a digest based on the contents of a String.
     * 
     * @param val
     *            specifies the input data. The encoding is expected to be
     *            UTF-8.
     * @return digest as long.
     */
    public static long createHash(String val) {
        return createHash(val, charset);
    }

    /**
     * Generates a digest based on the contents of an array of bytes.
     * 
     * @param data
     *            specifies input data.
     * @return digest as long.
     */
    public static long createHash(byte[] data) {
        long h = 0;
        byte[] res;

        synchronized (digestFunction) {
            res = digestFunction.digest(data);
        }

        for (int i = 0; i < 4; i++) {
            h <<= 8;
            h |= ((int) res[i]) & 0xFF;
        }
        return h;
    }

    /**
     * Compares the contents of two instances to see if they are equal.
     * 
     * @param obj
     *            is the object to compare to.
     * @return True if the contents of the objects are equal.
     */
    @Override
    public boolean equals(Object obj) {
        if (obj == null) {
            return false;
        }
        if (getClass() != obj.getClass()) {
            return false;
        }
        final BloomFilter<E> other = (BloomFilter<E>) obj;
        if (this.expectedNumberOfFilterElements != other.expectedNumberOfFilterElements) {
            return false;
        }
        if (this.k != other.k) {
            return false;
        }
        if (this.bitSetSize != other.bitSetSize) {
            return false;
        }
        if (this.bitset != other.bitset
                && (this.bitset == null || !this.bitset.equals(other.bitset))) {
            return false;
        }
        return true;
    }

    /**
     * Calculates a hash code for this class.
     * 
     * @return hash code representing the contents of an instance of this class.
     */
    @Override
    public int hashCode() {
        int hash = 7;
        hash = 61 * hash + (this.bitset != null ? this.bitset.hashCode() : 0);
        hash = 61 * hash + this.expectedNumberOfFilterElements;
        hash = 61 * hash + this.bitSetSize;
        hash = 61 * hash + this.k;
        return hash;
    }

    /**
     * Calculates the expected probability of false positives based on the
     * number of expected filter elements and the size of the Bloom filter. <br
     * /><br /> The value returned by this method is the <i>expected</i> rate of
     * false positives, assuming the number of inserted elements equals the
     * number of expected elements. If the number of elements in the Bloom
     * filter is less than the expected value, the true probability of false
     * positives will be lower.
     * 
     * @return expected probability of false positives.
     */
    public double expectedFalsePositiveProbability() {
        return getFalsePositiveProbability(expectedNumberOfFilterElements);
    }

    /**
     * Calculate the probability of a false positive given the specified number
     * of inserted elements.
     * 
     * @param numberOfElements
     *            number of inserted elements.
     * @return probability of a false positive.
     */
    public double getFalsePositiveProbability(double numberOfElements) {
        // (1 - e^(-k * n / m)) ^ k
        return Math.pow((1 - Math.exp(-k * (double) numberOfElements
                / (double) bitSetSize)), k);

    }

    /**
     * Get the current probability of a false positive. The probability is
     * calculated from the size of the Bloom filter and the current number of
     * elements added to it.
     * 
     * @return probability of false positives.
     */
    public double getFalsePositiveProbability() {
        return getFalsePositiveProbability(numberOfAddedElements);
    }

    /**
     * Returns the value chosen for K.<br /> <br /> K is the optimal number of
     * hash functions based on the size of the Bloom filter and the expected
     * number of inserted elements.
     * 
     * @return optimal k.
     */
    public int getK() {
        return k;
    }

    /**
     * Sets all bits to false in the Bloom filter.
     */
    public void clear() {
        bitset.clear();
        numberOfAddedElements = 0;
    }

    /**
     * Adds an object to the Bloom filter. The output from the object's
     * toString() method is used as input to the hash functions.
     * 
     * @param element
     *            is an element to register in the Bloom filter.
     */
    public void add(E element) {
        long hash;
        String valString = element.toString();
        for (int x = 0; x < k; x++) {
            hash = createHash(valString + Integer.toString(x));
            hash = hash % (long) bitSetSize;
            bitset.set(Math.abs((int) hash), true);
        }
        numberOfAddedElements++;
    }

    /**
     * Adds all elements from a Collection to the Bloom filter.
     * 
     * @param c
     *            Collection of elements.
     */
    public void addAll(Collection<? extends E> c) {
        for (E element : c)
            add(element);
    }

    /**
     * Returns true if the element could have been inserted into the Bloom
     * filter. Use getFalsePositiveProbability() to calculate the probability of
     * this being correct.
     * 
     * @param element
     *            element to check.
     * @return true if the element could have been inserted into the Bloom
     *         filter.
     */
    public boolean contains(E element) {
        long hash;
        String valString = element.toString();
        for (int x = 0; x < k; x++) {
            hash = createHash(valString + Integer.toString(x));
            hash = hash % (long) bitSetSize;
            if (!bitset.get(Math.abs((int) hash)))
                return false;
        }
        return true;
    }

    /**
     * Returns true if all the elements of a Collection could have been inserted
     * into the Bloom filter. Use getFalsePositiveProbability() to calculate the
     * probability of this being correct.
     * 
     * @param c
     *            elements to check.
     * @return true if all the elements in c could have been inserted into the
     *         Bloom filter.
     */
    public boolean containsAll(Collection<? extends E> c) {
        for (E element : c)
            if (!contains(element))
                return false;
        return true;
    }

    /**
     * Read a single bit from the Bloom filter.
     * 
     * @param bit
     *            the bit to read.
     * @return true if the bit is set, false if it is not.
     */
    public boolean getBit(int bit) {
        return bitset.get(bit);
    }

    /**
     * Set a single bit in the Bloom filter.
     * 
     * @param bit
     *            is the bit to set.
     * @param value
     *            If true, the bit is set. If false, the bit is cleared.
     */
    public void setBit(int bit, boolean value) {
        bitset.set(bit, value);
    }

    /**
     * Return the bit set used to store the Bloom filter.
     * 
     * @return bit set representing the Bloom filter.
     */
    public BitSet getBitSet() {
        return bitset;
    }

    /**
     * Returns the number of bits in the Bloom filter. Use count() to retrieve
     * the number of inserted elements.
     * 
     * @return the size of the bitset used by the Bloom filter.
     */
    public int size() {
        return this.bitSetSize;
    }

    /**
     * Returns the number of elements added to the Bloom filter after it was
     * constructed or after clear() was called.
     * 
     * @return number of elements added to the Bloom filter.
     */
    public int count() {
        return this.numberOfAddedElements;
    }

    /**
     * Returns the expected number of elements to be inserted into the filter.
     * This value is the same value as the one passed to the constructor.
     * 
     * @return expected number of elements.
     */
    public int getExpectedNumberOfElements() {
        return expectedNumberOfFilterElements;
    }

    /**
     * Get expected number of bits per element when the Bloom filter is full.
     * This value is set by the constructor when the Bloom filter is created.
     * See also getBitsPerElement().
     * 
     * @return expected number of bits per element.
     */
    public double getExpectedBitsPerElement() {
        return this.bitsPerElement;
    }

    /**
     * Get actual number of bits per element based on the number of elements
     * that have currently been inserted and the length of the Bloom filter. See
     * also getExpectedBitsPerElement().
     * 
     * @return number of bits per element.
     */
    public double getBitsPerElement() {
        return this.bitSetSize / (double) numberOfAddedElements;
    }
}

package com.lietu.show;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Properties;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

//提取商品信息
public class ExtractProduct {

    static BloomFilter<String> urlSeen = new BloomFilter<String>(200000, 20000);

    /**
     * @param args
     * @throws IOException
     */
    public static void main(String[] args) throws IOException {
        String url = "http://www.xiu.com/about/classifymap.shtml";
        Document doc = Jsoup.connect(url).get();
        Elements links = doc.select("a[href]"); // 带有href属性的a标签
        for (Element link : links) { // 遍历每个链接
            String linkHref = link.attr("href"); // 得到href属性中的值，也就是url地址
            if (linkHref.startsWith("http://list.xiu.com/")
                    || linkHref.startsWith("http://brand.xiu.com/")) {
                linkHref = getListURL(linkHref);
                if (urlSeen.contains(linkHref))
                    continue;
                // System.out.println(linkHref); //输出url地址和锚点上的文字说明
                extractList(linkHref);
            }
        }
    }
    
    public static String getListURL(String oldURL){
        ParseURL splitURL = new ParseURL(oldURL);
        if(splitURL.searchparms==null){
            return splitURL.baseURL;
        }
        //System.out.println(splitURL.baseURL);
        String curPage = splitURL.searchparms.get("currentPage");
        // http://list.xiu.com/20069.html?currentPage=2
        if (curPage != null)
            //System.out.println(splitURL.baseURL + "?currentPage=" + curPage);
            return splitURL.baseURL + "?currentPage=" + curPage;
        else{
            //System.out.println(splitURL.baseURL);
            return splitURL.baseURL;
        }
    }

    //处理目录页
    // http://list.xiu.com/19757.html
    // http://brand.xiu.com/3117.html
    public static void extractList(String url) {
        System.out.println("List url:" + url);
        urlSeen.add(url);
        Document doc = null;
        try {
            doc = Jsoup.connect(url).get();
        } catch (Exception e) {
            try {
                doc = Jsoup.connect(url).get();
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
                return;
            }
        }

        Elements links = doc.select("a[href]"); // 带有href属性的a标签
        for (Element link : links) { // 遍历每个链接
            String linkHref = link.attr("href"); // 得到href属性中的值，也就是url地址
            if (linkHref.startsWith("http://list.xiu.com/")
                    || linkHref.startsWith("http://brand.xiu.com/")) {
                linkHref = getListURL(linkHref);
                if (urlSeen.contains(linkHref))
                    continue;
                extractList(linkHref); // 输出url地址和锚点上的文字说明
            } else if (linkHref.startsWith("http://item.xiu.com/product/")) { // "http://item.xiu.com/product/0359097.shtml"
                if (urlSeen.contains(linkHref))
                    continue;
                getProduct(linkHref);
            }
        }
    }

    //处理详细页
    public static void getProduct(String url) {
        System.out.println("Product url:" + url);
        urlSeen.add(url);
        // String url = "http://item.xiu.com/product/0359097.shtml";
        Document doc = null;
        try {
            doc = Jsoup.connect(url).get();
        } catch (Exception e) {
            try{
                doc = Jsoup.connect(url).get();
            }
            catch (Exception e2) {
                try {
                    doc = Jsoup.connect(url).get();
                } catch (IOException e1) {
                    // TODO Auto-generated catch block
                    e1.printStackTrace();
                    return;
                }
            }
        }
        Elements links = doc.select("div.p_title"); // 带有href属性的a标签
        String name = links.get(0).childNode(1).childNode(0).toString();
        // System.out.println(name);

        Element link = doc.select("div.conlist").get(2);
        String desc = link.text();
        // System.out.println(link.text());

        Element img = doc.select("#imgPic").first();
        String imgSrc = img.attr("src");
        // System.out.println(img.attr("src"));
        
        Element thumb = doc.select("img[onload]").first();
        String thumbSrc = thumb.attr("src");
        //System.out.println(thumbSrc);
        
        insertDatabase(url,name,desc,imgSrc,thumbSrc);
    }

    public static void insertDatabase(String url, String name, String desc,
            String img,String thumbSrc) {
        Connection conn = null;
        PreparedStatement stmt2 = null;
        try {
            Class.forName("sun.jdbc.odbc.JdbcOdbcDriver");
            Properties p = new Properties(); 
            p.put("charSet", "gb2312");
            conn = DriverManager
                    .getConnection("jdbc:odbc:driver={Microsoft Access Driver (*.mdb)};DBQ=good.mdb",p);

            stmt2 = conn
                    .prepareStatement("insert into good(url,productName,productDesc,img,thumb)values(?,?,?,?,?)");
            stmt2.setString(1, url);
            stmt2.setString(2, name);
            stmt2.setString(3, desc);
            stmt2.setString(4, img);
            stmt2.setString(5, thumbSrc);
            stmt2.executeUpdate();
        } catch (Exception ex) {
            ex.printStackTrace();
        } finally {
            try {
                if (stmt2 != null)
                    stmt2.close();
                if (conn != null)
                    conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }

}

  给定误算率p，插入元素个数n，那么最优位数组大小为：

 

Hash函数个数选取最优数目为：

　　

python 实现：

import math

N = 30000000000.0
P = 0.001

def get_len_arr(n, p):
    return -(n * math.log(p) / (math.log(2) ** 2))


def get_k(p):
    return - (math.log(p) / math.log(2))

m = get_len_arr(N, P)

print m, 'bit'

g = m / (8 * 1024 * 1024 * 1024)

print g, 'GB'

k = get_k(P)

print k, 'ge hash hanshu'
=========================
4.31327626982e+11 bit
50.2131445079 GB       300亿，50G内存。
9.96578428466 ge hash hanshu