扫描FTP，保存文件

1、需求：某公司ftp服务器中一个文件夹中有30个文件（文件名字是不同的），每五分钟产生一个新的文件，同时删除这三十个文件中最早产生的文件，该文件夹中始终保持30个文件。

　　现在需要采集一周的数据做研究。

　　解决思路，用java扫描该文件夹，把所有产生的新文件保存到本地一个目录下，文件名持久化一个文本中，防止FTP出问题。一周后可以得到这一周的数据。

下面是代码：

package cim;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.SocketException;
import java.util.List;

import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.log4j.Logger;

public class FtpUtil {
    static Logger logger = Logger.getLogger(FtpUtil.class);
    /**
     * 获取FTPClient对象
     * 
     * @param ftpHost
     *            FTP主机服务器
     * @param ftpPassword
     *            FTP 登录密码
     * @param ftpUserName
     *            FTP登录用户名
     * @param ftpPort
     *            FTP端口 默认为21
     * @return
     */
    static FTPClient ftpClient = null;

    public static FTPClient getFTPClient(String ftpHost, String ftpUserName,
            String ftpPassword, int ftpPort) {
        try {
            ftpClient = new FTPClient();
            ftpClient.setControlEncoding("GBK");
            ftpClient.connect(ftpHost, ftpPort);// 连接FTP服务器
            ftpClient.login(ftpUserName, ftpPassword);// 登陆FTP服务器
            if (!FTPReply.isPositiveCompletion(ftpClient.getReplyCode())) {
                logger.info("未连接到FTP，用户名或密码错误。");
                close();
            } else {
                // logger.info("FTP连接成功。");
            }
        } catch (SocketException e) {
            e.printStackTrace();
            logger.info("FTP连接错误，请正确配置IP地址，账号和密码。");
        } catch (IOException e) {
            e.printStackTrace();
            logger.info("FTP的端口错误,请正确配置。");
        }
        return ftpClient;
    }

    /*
     * 从FTP服务器下载文件
     * 
     * @param ftpHost FTP IP地址
     * 
     * @param ftpUserName FTP 用户名
     * 
     * @param ftpPassword FTP用户名密码
     * 
     * @param ftpPort FTP端口
     * 
     * @param ftpPath FTP服务器中文件所在路径 格式： ftptest/aa
     * 
     * @param localPath 下载到本地的位置 格式：H:/download
     * 
     * @param txtAddress 本地文件名
     * 
     * @param time 扫描时间间隔
     */
    public static void downloadFtpFile(String ftpHost, String ftpUserName,
            String ftpPassword, int ftpPort, String ftpPath, String localPath,
            String txtAddress, int time) {
        while(ftpPort!=21){
            logger.info("检测到用户输入的端口号为:"+ftpPort+"，现已修改为默认值21.");
            ftpPort = 21;
        }
        int fs0 = 0;// 定义初始化文件的个数为零
        for (int a = 0; a < 500000; a++) {
            FTPClient ftpClient = null;
            try {
                ftpClient = getFTPClient(ftpHost, ftpUserName, ftpPassword,ftpPort);
                ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);// 设置文件类型为二进制否则可能导致乱码文件无法打开
                ftpClient.enterLocalPassiveMode();// 设置被动模式
                boolean dir = ftpClient.changeWorkingDirectory(ftpPath);
                // System.out.println("进入指定FTP文件夹==>"+dir);
                txtUtil txt = new txtUtil();

                List<String> listStrings = null;
                try {
                    listStrings = txt.readTxtFile(txtAddress);
                } catch (Exception e1) {
                    System.out.println("警告：发现问题读取文本的时候，程序错误");
                    e1.printStackTrace();
                }
                System.out.println("第" + (a + 1) + "次循环的listStrings长度为："+ listStrings.size());
                if (dir) {
                    File file2 = null;
                    FTPFile[] fs = ftpClient.listFiles();
                    if (a == 0) {
                        for (int i = 0; i < fs.length; i++) {
                            boolean bo = listStrings.contains(fs[i].getName());
                            if (!bo) {
                                if(fs[i].getName().endsWith("xml")){
                                    File file = new File(localPath);
                                    if (!file.exists() && !file.isDirectory()) {
                                        file.mkdir();
                                        logger.info("本地cim_download文件夹不存在，创建成功");
                                    }
                                    File localFile = new File(localPath
                                            + File.separatorChar + fs[i].getName());
                                    OutputStream os = new FileOutputStream(localFile);
                                    ftpClient.retrieveFile(fs[i].getName(), os);
                                    os.close();
                                    logger.info("首次启动文件复制成功：" + fs[i].getName()+ " 大小为：" + localFile.length() + "字节");
                                    txt.recordName(txtAddress, fs[i].getName());
                                    listStrings.add(fs[i].getName());
                                }
                            } else {
                                logger.info("首次启动发现：<" + fs[i].getName()+ ">，本地已存在，不再复制");
                            }
                        }
                    }else{
                        for (int i = 0; i < fs.length; i++) {
                            boolean bo = listStrings.contains(fs[i].getName());
                            if (!bo) {
                                if(fs[i].getName().endsWith("xml")){
                                    System.out.println("发现新文件：" + fs[i].getName());
                                    try {
                                        Thread.sleep(500);
                                    } catch (InterruptedException e) {
                                        e.printStackTrace();
                                    }
                                    file2 = new File(localPath);
                                    if (!file2.exists() && !file2.isDirectory()) {
                                        file2.mkdir();
                                        logger.info("本地cim_download文件夹可能被误删，现已创建成功");
                                    }
                                    File localFile = new File(localPath+ File.separatorChar + fs[i].getName());
                                    OutputStream os = new FileOutputStream(localFile);
                                    ftpClient.retrieveFile(fs[i].getName(), os);
                                    os.close();
                                    logger.info("新文件：" + fs[i].getName() + "  复制成功");
                                    // listStrings.add(fs[i].getName());
                                    txt.recordName(txtAddress, fs[i].getName());
                                }
                            }
                        }
                    }
                    
                    fs0 = fs.length;
                    try {
                        System.out.println("查询第" + (a + 1) + "次时，有" + fs0+ "个文件，" + "下次扫描时间为" + time + "毫秒后。");
                        Thread.sleep(time);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            } catch(NullPointerException e){
                logger.error("NullPointerException,创建连接为空。请检查FTP服务器"+"!!!"+e.getMessage());
            }catch (FileNotFoundException e) {
                logger.error("FileNotFoundException"+e.getMessage());
                e.printStackTrace();
            } catch (SocketException e) {
                logger.error("连接FTP失败.");
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
                logger.error("文件读取错误。");
                e.printStackTrace();
            } catch (Exception e) {
                logger.error("未知异常！！！");
            } finally {
                try {
                    close();
                } catch (Exception e) {
                    logger.error("关闭FTP错误！"+e.getMessage());
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 转码[GBK -> ISO-8859-1] 不同的平台需要不同的转码
     * 
     * @param obj
     * @return
     */
    private static String gbkToIso8859(Object obj) {
        try {
            if (obj == null)
                return "";
            else
                return new String(obj.toString().getBytes("GBK"), "iso-8859-1");
        } catch (Exception e) {
            return "";
        }
    }

    /**
     * 转码[ISO-8859-1 -> GBK] 不同的平台需要不同的转码
     * 
     * @param obj
     * @return
     */
    private static String iso8859ToGbk(Object obj) {
        try {
            if (obj == null)
                return "";
            else {
                String str = new String(obj.toString().getBytes("iso-8859-1"),
                        "GBK");
                return str;
            }
        } catch (Exception e) {
            return "";
        }
    }

    /**
     * 关闭当前连接
     */
    public static void close() {
        try {
            ftpClient.logout();
            ftpClient.disconnect();
        } catch (IOException e) {
            logger.error("ftp ftpserver close error : " + e.getMessage());
        }
    }
}

package cim;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;

public class txtUtil {
    
    public void recordName(String txtAddress,String name) throws IOException{
        File file = new File(txtAddress);
        if(!file.exists()){
            file.createNewFile();
        }
        FileWriter fw = new FileWriter(file, true);
        PrintWriter pw = new PrintWriter(fw);
        pw.print(name);
        pw.print(",");
        pw.flush();
        try {
            fw.flush();
            pw.close();
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    public  List<String> readTxtFile(String txtAddress)throws Exception{  
        List<String> list = new ArrayList<>();
        try {
            String encoding="UTF-8";
            File file=new File(txtAddress);
            if(!file.exists()){
                file.createNewFile();
            }
            if(file.isFile() && file.exists()){ //判断文件是否存在
                InputStreamReader read = new InputStreamReader(new FileInputStream(file),encoding);//考虑到编码格式
                BufferedReader bufferedReader = new BufferedReader(read);
                String lineTxt = null;
                while((lineTxt = bufferedReader.readLine()) != null){
                    String[] sourceStrArray =lineTxt.split(",");
                    for(int i = 0 ;i<sourceStrArray.length;i++){
                        list.add(sourceStrArray[i]);
                    }
                }
                read.close();
            }else{ 
                System.out.println("读取文件名集合出错");
            }
        }catch (Exception e) {
            System.out.println("读取文件内容出错");
            e.printStackTrace();
        }
        
        return list;
    }
}

package cim;

import java.io.File;
import java.util.HashMap;
import java.util.Map;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class ftpTest {

    public static void main(String[] args) {
        
        ftpTest test = new ftpTest();
        Map<String, Object> map = test.XMLReader();
        String ftpHost = (String) map.get("ftpHost");  
        String ftpUserName = (String) map.get("ftpUserName");  
        String ftpPassword = (String) map.get("ftpPassword");  
        int ftpPort = Integer.parseInt((String) map.get("ftpPort")) ;  
        String ftpPath = (String) map.get("ftpPath");  
        String localPath = (String) map.get("localPath");  
        String txtAddress = (String) map.get("txtAddress");  
        int time = Integer.parseInt((String)map.get("time"));  
        
        FtpUtil.downloadFtpFile(ftpHost, ftpUserName, ftpPassword, ftpPort, ftpPath, localPath, txtAddress,time);  
    }
    
    public Map<String,Object> XMLReader(){
        Element element = null;
        File f = new File("test.xml");
        DocumentBuilder db = null;
        DocumentBuilderFactory dbf = null;
        Map<String, Object> map = new HashMap<>();
        try {
            dbf = DocumentBuilderFactory.newInstance();
            db = dbf.newDocumentBuilder();
            Document dt = db.parse(f);
            element = dt.getDocumentElement();
            NodeList childNodes = element.getChildNodes();
            for (int i = 0; i < childNodes.getLength(); i++) {
                Node node1 = childNodes.item(i);
                if ("Account".equals(node1.getNodeName())) {
                    NodeList nodeDetail = node1.getChildNodes();
                    for (int j = 0; j < nodeDetail.getLength(); j++) {
                        Node detail = nodeDetail.item(j);
                        if ("ftpHost".equals(detail.getNodeName())) 
                            map.put("ftpHost", detail.getTextContent());
                        else if ("ftpUserName".equals(detail.getNodeName())) 
                            map.put("ftpUserName", detail.getTextContent());
                        else if ("ftpPassword".equals(detail.getNodeName()))
                            map.put("ftpPassword", detail.getTextContent());
                        else if ("ftpPort".equals(detail.getNodeName())) 
                            map.put("ftpPort", detail.getTextContent());
                        else if ("ftpPath".equals(detail.getNodeName())) 
                            map.put("ftpPath", detail.getTextContent());
                        else if ("localPath".equals(detail.getNodeName())) 
                            map.put("localPath", detail.getTextContent());
                        else if ("txtAddress".equals(detail.getNodeName())) 
                            map.put("txtAddress", detail.getTextContent());
                        else if ("time".equals(detail.getNodeName())) 
                            map.put("time", detail.getTextContent());
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return map;
    }
}

log4j.rootLogger=info,A1,A2
# u8F93u51FAu5230u63A7u5236u53F0
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [u4fe1u606f] %m%n
# u8F93u51FAu5230u6587u4EF6u5F53u4E2D
log4j.appender.A2=org.apache.log4j.FileAppender
log4j.appender.A2.File=cim_logging.log
log4j.appender.A2.Append=true
log4j.appender.A2.layout=org.apache.log4j.PatternLayout
log4j.appender.A2.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [u4fe1u606f] %m%n

<?xml version="1.0" encoding="UTF-8"?>
<Accounts>
 <Account type="type1">
<!--  FTP端口号 -->
  <ftpHost>127.0.0.1</ftpHost>
<!--   FTP账号 -->
  <ftpUserName>1</ftpUserName>
<!--   FTP密码 -->
  <ftpPassword>1</ftpPassword>
<!--   FTP端口号 -->
  <ftpPort>22</ftpPort>
<!--   FTP路径 -->
  <ftpPath>users/ems/open2000e/data/xmldat/nari/all/</ftpPath>
<!--   本地保存文件路径 -->
  <localPath>cim_download</localPath>
<!--   本地保存文件名集合路径 -->
  <txtAddress>cim_name.txt</txtAddress>
<!--   设置扫描文件的时间间隔,单位：毫秒 -->
  <time>5000</time>
 </Account>
</Accounts>

是用的main方法启动。

Fat Jar打包插件方法    http://jingyan.baidu.com/article/da1091fbd7dae1027849d63b.html

下面是运行jar包