爬虫

import java.io.IOException;
import java.util.Set;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;


public class wikiCrawler {
     private void initCrawlerWithSeeds(String[] seeds) {
        for(int i=0;i<seeds.length;i++){
            LinkQueue.addUnvisitedUrl(seeds[i]);
        }
    }
     public void crawling(String[] seeds) throws IOException, ParserException{
         initCrawlerWithSeeds(seeds);
         while(!LinkQueue.unVisitedUrlEmpty()&&LinkQueue.getVisitedUrlNum()<1000){
             String visitUrl=(String)LinkQueue.unVisitedUrlDequeue();
             
             TagNameFilter tagNameFilter=new TagNameFilter("title");
             
             
              DownLoadFile downLoadFile=new DownLoadFile("D://");
             String filepath=downLoadFile.downloadFile(visitUrl);
             System.out.println(filepath);
             if(filepath!=null){
             String contentString=HtmlContent.getHtml(filepath);
             
             NodeList list=new Parser(contentString).extractAllNodesThatMatch(tagNameFilter);
             String title=((Node)list.elementAt(0)).toPlainTextString();
             System.out.println(title);
             LinkQueue.addVisitedUrl(visitUrl);
             if(contentString!=null){
                 Set<String> linksSet=WikiParseHtml.extractLinkSet(contentString);
                 for(Object link:linksSet){
                     LinkQueue.addUnvisitedUrl((String) link);
                 }
             }
             }
         }
     }
     public static void main(String[] args) throws IOException, ParserException{
         wikiCrawler crawler=new wikiCrawler();
         crawler.crawling(new String[]{"http://free0007.iteye.com"});
     }
}

html content

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;


public class HtmlContent {

    public static String getHtml(String filepath) throws IOException {
        //File file = new File(filepath);

       /*StringBuffer sb = new StringBuffer();
       String s ="";
       BufferedReader br=new BufferedReader(new InputStreamReader(new FileInputStream(filepath),"UTF-8"));

       while( (s = br.readLine()) != null) {
         sb.append(s + "
");
         }

       br.close();
       String str = sb.toString();
      return str;*/

        try {  
            BufferedReader bis = new BufferedReader(new InputStreamReader(new FileInputStream( new File(filepath)),"UTF-8") );  
            StringBuffer stringBuffer=new StringBuffer(250000);
            String szTemp;  
              
            while ( (szTemp = bis.readLine()) != null) {  
                stringBuffer.append(szTemp+"
");  
            }  
            bis.close();  
            return stringBuffer.toString();  
        }  
        catch( Exception e ) {  
            return "";  
        }  
    }
    /*public static void main(String[] args){
        try {
            System.out.print(HtmlContent.getHtml("D://zh.wikipedia.org_wiki_Wikipedia_%E9%A6%96%E9%A1%B5.html"));
        } catch (IOException e) {
            System.out.print("error");
            e.printStackTrace();
        }
    }*/
}

downloadfile

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;


public class DownLoadFile {
    private String filepath="";
    public String getFileNameByUrl(String url, String contentType){
        url=url.substring(7);
        if(contentType.indexOf("html")!=-1){
            url=url.replaceAll("[\?/:*|<>"]", "_")+".html";
            return url;
        }
        //application/pdf
        else{
            return url.replaceAll("[\?/:*|<>"]", "_")+contentType.substring(contentType.indexOf("/")+1);
        }
    }
    public DownLoadFile(String filepath){
        this.filepath=filepath;
    }
    private void saveToLocal(InputStream is,String filePath) throws IOException{
        try {
            DataOutputStream outputStream=new DataOutputStream(new FileOutputStream(new File(filePath)));
            int len=0;
            byte[] buffer=new byte[1024];
            while((len=is.read(buffer))!=-1){
                outputStream.write(buffer, 0, len);
            }
            outputStream.flush();
            outputStream.close();
        } catch (FileNotFoundException e) {
            
            e.printStackTrace();
        }
    }
    public String downloadFile(String url) throws IOException{
        String filePathString=null;
        CloseableHttpClient httpClient=HttpClients.createDefault();
        HttpGet httpGet=new HttpGet(url);
        RequestConfig requestConfig=RequestConfig.copy(RequestConfig.custom().build()).setConnectTimeout(5000).build();
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response=httpClient.execute(httpGet);
        try {
            String statusCode=response.getStatusLine().toString();
            if(Integer.parseInt(statusCode.split(" ")[1])!=HttpStatus.SC_OK){
                System.err.println(url+" Failed:"+statusCode);
                filePathString=null;
            }
            else {
                HttpEntity entity=response.getEntity();
                InputStream input= entity.getContent();
                Header header= entity.getContentType();
                filePathString=filepath+getFileNameByUrl(url, header.getValue());
                
                saveToLocal(input, filePathString);
                
                
            }

        } catch (ClientProtocolException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            response.close();
            httpClient.close();
        }
        
        return filePathString;
    }

    
}

wikiparserhtml

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Set;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;


public class WikiParseHtml {
    public static Set<String> extractLinkSet(String content){
        Set<String> linksSet=new HashSet<String>();
        try {
            Parser parser= Parser.createParser(content, "utf-8");
            NodeClassFilter nodeClassFilter=new NodeClassFilter(LinkTag.class);
            NodeList list=parser.extractAllNodesThatMatch(nodeClassFilter);
            for(int i=0;i<list.size();i++){
                Node tagNode=list.elementAt(i);
                if(tagNode instanceof LinkTag){
                    LinkTag linkTag=(LinkTag)tagNode;
                    String urlString=linkTag.getLink();
                    if(urlString.startsWith("http://")){
                        linksSet.add(urlString);
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return linksSet;
        
    }
}

queue

import java.util.LinkedList;


public class Queue {
    private LinkedList<Object> queue=new LinkedList<Object>();
    public void enQueue(Object t){
        queue.addLast(t);
    }
    public Object deQueue(){
        return queue.removeFirst();
    }
    public boolean isQueueEmpty(){
        return queue.isEmpty();
    }
    public boolean contains(Object t){
        return queue.contains(t);
    }
    public boolean empty(){
        return queue.isEmpty();
    }
}

linkqueue

import java.util.HashSet;
import java.util.Set;


public class LinkQueue {
    private static Set visitedUrl=new HashSet();
    private static Queue unvisitedUrl=new Queue();
    public static Queue getUnvisitedUrl(){
        return unvisitedUrl;
    }
    public static void addVisitedUrl(String url){
        visitedUrl.add(url);
    }
    public static void removeVisitedUrl(String url){
        visitedUrl.remove(url);
    }
    public static Object unVisitedUrlDequeue(){
        return unvisitedUrl.deQueue();
    }
    public static void addUnvisitedUrl(String url){
        if(url!=null||!url.trim().equals("")||!visitedUrl.contains(url)||!unvisitedUrl.contains(url))
            unvisitedUrl.enQueue(url);
    }
    public static int getVisitedUrlNum(){
        return visitedUrl.size();
    }
    public static boolean unVisitedUrlEmpty(){
        return unvisitedUrl.isQueueEmpty();
    }
}