Heritrix 3.1.0 源码解析（三十二）

本文要分析的是FetchDNS处理器，该处理器的功能是解析CrawlURI curi对象的DNS地址，该处理器是采用dnsjava-2.0.3.jar组件进行解析DNS的（我们可以参考本文代码采用dnsjava-2.0.3.jar组件API解析DNS）

FetchDNS处理器的重要成员变量

// Defaults.
    private short ClassType = DClass.IN;
    private short TypeType = Type.A;
    protected InetAddress serverInetAddr = null;

 /**
     * Used to do DNS lookups.
     */
    protected ServerCache serverCache;
    public ServerCache getServerCache() {
        return this.serverCache;
    }
    @Autowired
    public void setServerCache(ServerCache serverCache) {
        this.serverCache = serverCache;
    }
    
    /**
     * Whether or not to perform an on-the-fly digest hash of retrieved
     * content-bodies.
     */
    {
        setDigestContent(true);
    }
    public boolean getDigestContent() {
        return (Boolean) kp.get("digestContent");
    }
    public void setDigestContent(boolean digest) {
        kp.put("digestContent",digest);
    }

    /**
     * Which algorithm (for example MD5 or SHA-1) to use to perform an 
     * on-the-fly digest hash of retrieved content-bodies.
     */
    String digestAlgorithm = "sha1"; 
    public String getDigestAlgorithm() {
        return digestAlgorithm;
    }
    public void setDigestAlgorithm(String digestAlgorithm) {
        this.digestAlgorithm = digestAlgorithm;
    }

处理器void innerProcess(CrawlURI curi)方法

protected void innerProcess(CrawlURI curi) {
        Record[] rrecordSet = null; // Retrieved dns records
        String dnsName = null;
        try {
            dnsName = curi.getUURI().getReferencedHost();
        } catch (URIException e) {
            logger.log(Level.SEVERE, "Failed parse of dns record " + curi, e);
        }
        
        if(dnsName == null) {
            curi.setFetchStatus(S_UNFETCHABLE_URI);
            return;
        }

        CrawlHost targetHost = getServerCache().getHostFor(dnsName);
        //IP地址转换为InetAddress类型
        if (isQuadAddress(curi, dnsName, targetHost)) {
            // We're done processing.
            return;
        }
        
        // Do actual DNS lookup.
        curi.setFetchBeginTime(System.currentTimeMillis());

        // Try to get the records for this host (assume domain name)
        // TODO: Bug #935119 concerns potential hang here
        String lookupName = dnsName.endsWith(".") ? dnsName : dnsName + ".";
        try {
            //DNS解析
            rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run();
        } catch (TextParseException e) {
            rrecordSet = null;
        }
        curi.setContentType("text/dns");
        if (rrecordSet != null) {
            if (logger.isLoggable(Level.FINE)) {
                logger.fine("Found recordset for " + lookupName);
            }
            //设置CrawlHost targetHost对象的IP属性； CrawlURI curi对象的Recorder httpRecorder属性
            storeDNSRecord(curi, dnsName, targetHost, rrecordSet);
        } else {
            if (logger.isLoggable(Level.FINE)) {
                logger.fine("Failed find of recordset for " + lookupName);
            }
            if (getAcceptNonDnsResolves()||"localhost".equals(dnsName)) {
                // Do lookup that bypasses javadns.
                InetAddress address = null;
                try {
                    address = InetAddress.getByName(dnsName);
                } catch (UnknownHostException e1) {
                    address = null;
                }
                if (address != null) {
                    targetHost.setIP(address, DEFAULT_TTL_FOR_NON_DNS_RESOLVES);
                    curi.setFetchStatus(S_GETBYNAME_SUCCESS);
                    if (logger.isLoggable(Level.FINE)) {
                        logger.fine("Found address for " + dnsName +
                            " using native dns.");
                    }
                } else {
                    if (logger.isLoggable(Level.FINE)) {
                        logger.fine("Failed find of address for " + dnsName +
                            " using native dns.");
                    }
                    setUnresolvable(curi, targetHost);
                }
            } else {
                setUnresolvable(curi, targetHost);
            }
        }
        curi.setFetchCompletedTime(System.currentTimeMillis());
    }

相关调用方法如下（dnsjava-2.0.3.jar组件的API） 

/**
     * 设置CrawlHost targetHost对象的IP属性； CrawlURI curi对象的Recorder httpRecorder属性
     * @param curi
     * @param dnsName
     * @param targetHost
     * @param rrecordSet
     */
    protected void storeDNSRecord(final CrawlURI curi, final String dnsName,
            final CrawlHost targetHost, final Record[] rrecordSet) {
        // Get TTL and IP info from the first A record (there may be
        // multiple, e.g. www.washington.edu) then update the CrawlServer
        ARecord arecord = getFirstARecord(rrecordSet);
        if (arecord == null) {
            throw new NullPointerException("Got null arecord for " +
                dnsName);
        }
        //设置CrawlHost targetHost对象IP属性
        targetHost.setIP(arecord.getAddress(), arecord.getTTL());
        try {
            //CrawlURI curi对象的Recorder httpRecorder属性
            recordDNS(curi, rrecordSet);
            curi.setFetchStatus(S_DNS_SUCCESS);
            curi.setDNSServerIPLabel(ResolverConfig.getCurrentConfig().server());
        } catch (IOException e) {
            logger.log(Level.SEVERE, "Failed store of DNS Record for " +
                curi.toString(), e);
            setUnresolvable(curi, targetHost);
        }
    }
    /**
     * IP地址转换为InetAddress
     * @param curi
     * @param dnsName
     * @param targetHost
     * @return
     */
    protected boolean isQuadAddress(final CrawlURI curi, final String dnsName,
            final CrawlHost targetHost) {
        boolean result = false;
        Matcher matcher = InetAddressUtil.IPV4_QUADS.matcher(dnsName);
        // If it's an ip no need to do a lookup
        if (matcher == null || !matcher.matches()) {
            return result;
        }
        
        result = true;
        // Ideally this branch would never be reached: no CrawlURI
        // would be created for numerical IPs
        if (logger.isLoggable(Level.WARNING)) {
            logger.warning("Unnecessary DNS CrawlURI created: " + curi);
        }
        try {
            targetHost.setIP(InetAddress.getByAddress(dnsName, new byte[] {
                    (byte) (new Integer(matcher.group(1)).intValue()),
                    (byte) (new Integer(matcher.group(2)).intValue()),
                    (byte) (new Integer(matcher.group(3)).intValue()),
                    (byte) (new Integer(matcher.group(4)).intValue()) }),
                    CrawlHost.IP_NEVER_EXPIRES); // Never expire numeric IPs
            curi.setFetchStatus(S_DNS_SUCCESS);
        } catch (UnknownHostException e) {
            logger.log(Level.SEVERE, "Should never be " + e.getMessage(), e);
            setUnresolvable(curi, targetHost);
        }
        return result;
    }
    /**
     * 封装到CrawlURI curi对象的Recorder httpRecorder属性
     * @param curi
     * @param rrecordSet
     * @throws IOException
     */
    protected void recordDNS(final CrawlURI curi, final Record[] rrecordSet)
            throws IOException {
        //转换为byte[]
        final byte[] dnsRecord = getDNSRecord(curi.getFetchBeginTime(),
                rrecordSet);

        Recorder rec = curi.getRecorder();
        // Shall we get a digest on the content downloaded?
        boolean digestContent = getDigestContent();
        String algorithm = null;
        if (digestContent) {
            algorithm = getDigestAlgorithm();
            rec.getRecordedInput().setDigest(algorithm);
        } else {
            rec.getRecordedInput().setDigest((MessageDigest)null);
        }
        //byte[]转换为InputStream，封装到CrawlURI curi对象的Recorder httpRecorder属性
        InputStream is = curi.getRecorder().inputWrap(
                new ByteArrayInputStream(dnsRecord));

        if (digestContent) {
            rec.getRecordedInput().startDigest();
        }

        // Reading from the wrapped stream, behind the scenes, will write
        // files into scratch space
        try {
            while (is.read(this.reusableBuffer) != -1) {
                continue;
            }
        } finally {
            is.close();
            rec.closeRecorders();
        }
        curi.setContentSize(dnsRecord.length);

        if (digestContent) {
            curi.setContentDigest(algorithm,
                rec.getRecordedInput().getDigestValue());
        }
    }
    /**
     * 转换为byte[]
     * @param fetchStart
     * @param rrecordSet
     * @return
     * @throws IOException
     */
    protected byte [] getDNSRecord(final long fetchStart,
            final Record[] rrecordSet)
    throws IOException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        // Start the record with a 14-digit date per RFC 2540
        byte[] fetchDate = ArchiveUtils.get14DigitDate(fetchStart).getBytes();
        baos.write(fetchDate);
        // Don't forget the newline
        baos.write("\n".getBytes());
        int recordLength = fetchDate.length + 1;
        if (rrecordSet != null) {
            for (int i = 0; i < rrecordSet.length; i++) {
                byte[] record = rrecordSet[i].toString().getBytes();
                recordLength += record.length;
                baos.write(record);
                // Add the newline between records back in
                baos.write("\n".getBytes());
                recordLength += 1;
            }
        }
        return baos.toByteArray();
    }
    
    protected void setUnresolvable(CrawlURI curi, CrawlHost host) {
        host.setIP(null, 0);
        curi.setFetchStatus(S_DOMAIN_UNRESOLVABLE); 
    }
    /**
     * 返回Record[] rrecordSet数组Type.A类型的Record元素
     * @param rrecordSet
     * @return
     */
    protected ARecord getFirstARecord(Record[] rrecordSet) {
        ARecord arecord = null;
        if (rrecordSet == null || rrecordSet.length == 0) {
            if (logger.isLoggable(Level.FINEST)) {
                logger.finest("rrecordSet is null or zero length: " +
                    rrecordSet);
            }
            return arecord;
        }
        for (int i = 0; i < rrecordSet.length; i++) {
            if (rrecordSet[i].getType() != Type.A) {
                if (logger.isLoggable(Level.FINEST)) {
                    logger.finest("Record " + Integer.toString(i) +
                        " is not A type but " + rrecordSet[i].getType());
                }
                continue;
            }
            arecord = (ARecord) rrecordSet[i];
            break;
        }
        return arecord;
    }

FetchDNS处理器和后面的FetchHTTP处理器涉及到消息摘要算法MessageDigest digest 对象，我这里转自网上的一篇文章供参考 

转自 http://huangyunbin.iteye.com/blog/1123442

MessageDigest的功能及用法

MessageDigest 类为应用程序提供信息摘要算法的功能，如 MD5 或 SHA 算法。信息摘要是安全的单向哈希函数，它接收任意大小的数据，并输出固定长度的哈希值。 

MessageDigest 对象开始被初始化。该对象通过使用 update（）方法处理数据。任何时候都可以调用 reset（）方法重置摘要。一旦所有需要更新的数据都已经被更新了，应该调用digest() 方法之一完成哈希计算。 

对于给定数量的更新数据，digest 方法只能被调用一次。在调用 digest 之后，MessageDigest 对象被重新设置成其初始状态。 

1、public static MessageDigest getInstance(String algorithm) 
                                 throws NoSuchAlgorithmException 

   返回实现指定摘要算法的 MessageDigest 对象。 

   algorithm - 所请求算法的名称 

2、public static MessageDigest getInstance(String algorithm, 
                                        String provider) 
                                 throws NoSuchAlgorithmException, 
                                        NoSuchProviderException 

  返回实现指定摘要算法的 MessageDigest 对象。 

  algorithm - 所请求算法的名称 

  provider - 提供者的名称。 

3、public void update(byte[] input) 

  使用指定的 byte 数组更新摘要。 

4、public byte[] digest() 

  通过执行诸如填充之类的最终操作完成哈希计算。在调用此方法之后，摘要被重置。 

5、public static boolean isEqual(byte[] digesta, 
                              byte[] digestb) 

    比较两个摘要的相等性。做简单的字节比较。 


注意：Provider可以通过 java.security.Security.getProviders() 方法获取已注册提供者列表。比较常用的有“SUN” 

SUN提供的常用的算法名称有：MD2 
MD5 
                        SHA-1 
                        SHA-256 
                        SHA-384 
                        SHA-512 

Code举例: 

import java.security.*; 
public class myDigest { 
  public static void main(String[] args)  { 
    myDigest my=new myDigest(); 
    my.testDigest(); 
  } 
  public void testDigest() 
  { 
   try { 
     String myinfo="我的测试信息"; 
    //java.security.MessageDigest alg=java.security.MessageDigest.getInstance("MD5"); 
      java.security.MessageDigest alga=java.security.MessageDigest.getInstance("SHA-1"); 
      alga.update(myinfo.getBytes()); 
      byte[] digesta=alga.digest(); 
      System.out.println("本信息摘要是:"+byte2hex(digesta)); 
      //通过某中方式传给其他人你的信息(myinfo)和摘要(digesta) 对方可以判断是否更改或传输正常 
      java.security.MessageDigest algb=java.security.MessageDigest.getInstance("SHA-1"); 
      algb.update(myinfo.getBytes()); 
      if (algb.isEqual(digesta,algb.digest())) { 
         System.out.println("信息检查正常"); 
       } 
       else 
        { 
          System.out.println("摘要不相同"); 
         } 
   } 
   catch (java.security.NoSuchAlgorithmException ex) { 
     System.out.println("非法摘要算法"); 
   } 
  } 
  public String byte2hex(byte[] b) //二行制转字符串 
    { 
     String hs=""; 
     String stmp=""; 
     for (int n=0;n<b.length;n++) 
      { 
       stmp=(java.lang.Integer.toHexString(b[n] & 0XFF)); 
       if (stmp.length()==1) hs=hs+"0"+stmp; 
       else hs=hs+stmp; 
       if (n<b.length-1)  hs=hs+":"; 
      } 
     return hs.toUpperCase(); 
    } 
} 

关于Java加密的更多信息：http://www.ibm.com/developerworks/cn/java/l-security/

--------------------------------------------------------------------------

本系列Heritrix 3.1.0 源码解析系本人原创

转载请注明出处 博客园 刺猬的温驯

本文链接 http://www.cnblogs.com/chenying99/archive/2013/04/30/3052411.html