上传到云和删除的简单应用

这段时间刚刚接触hadoop这个东西，简单的应用了一下，具体的应该是模仿应用，看代码：

spring中配置扫描配置文件：

applicationContext.xml

<bean id="propertyConfigurer"
        class="cn.edu.hbcf.vo.CustomizedPropertyConfigurer">
        <property name="locations">
            <list>
                <value>classpath:system.properties</value>
            </list>
        </property>
    </bean>

配置文件代码如下：

system.properties:

controller如下：

DFSController如下：

package cn.edu.hbcf.cloud.hadoop.dfs.controller;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.UUID;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;

import cn.edu.hbcf.cloud.hadoop.dfs.service.DFSService;
import cn.edu.hbcf.cloud.hadoop.dfs.vo.DfsInfo;
import cn.edu.hbcf.common.constants.WebConstants;
import cn.edu.hbcf.common.hadoop.CFile;
import cn.edu.hbcf.common.hadoop.FileManager;
import cn.edu.hbcf.common.hadoop.FileSysFactory;
import cn.edu.hbcf.common.hadoop.FileSysType;
import cn.edu.hbcf.common.vo.Criteria;
import cn.edu.hbcf.common.vo.ExceptionReturn;
import cn.edu.hbcf.common.vo.ExtFormReturn;
import cn.edu.hbcf.plugin.oa.pojo.NoteBook;
import cn.edu.hbcf.privilege.pojo.BaseUsers;

@Controller
@RequestMapping("/dfs")
public class DFSController {
    @Autowired
    private DFSService dfsService;
    
    @RequestMapping(method=RequestMethod.GET)
    public String index(HttpServletRequest request,HttpSession session,Model model){
        
        return "plugins/cloud/web/views/hadoop/dfs/dfs";
    }
    
    @RequestMapping(value = "/play",method=RequestMethod.GET)
    public String play(HttpServletRequest request,HttpSession session,Model model){
        
        return "plugins/cloud/web/views/hadoop/dfs/play";
    }
    
    @RequestMapping("/upload")
    public void upload(@RequestParam MultipartFile file, HttpServletRequest request, HttpServletResponse response){
        String uploadFileName = file.getOriginalFilename();
        FileManager fileMgr=FileSysFactory.getInstance(FileSysType.HDFS);
        String path=request.getParameter("path");
        HttpSession session=request.getSession(true);
        BaseUsers u = (BaseUsers) session.getAttribute(WebConstants.CURRENT_USER);
        // 获取文件后缀名 
        String fileType = StringUtils.substringAfterLast(uploadFileName, ".");
        String saveName = UUID.randomUUID().toString().replace("-", "")+("".equals(fileType) ? "" : "." + fileType);
        String filePath="/filesharesystem/"+u.getAccount()+path;
        try {
            fileMgr.putFile(file.getInputStream(), filePath, saveName);
            System.out.println("path="+filePath+" uploadFileName="+uploadFileName);
            StringBuffer buffer = new StringBuffer();
            buffer.append("{success:true,fileInfo:{fileName:'").append(uploadFileName).append("',");
            buffer.append("filePath:'").append(filePath+saveName).append("',");            
            buffer.append("projectPath:'").append(filePath+saveName).append("',");
            buffer.append("storeName:'").append(saveName).append("',");
            buffer.append("fileSize:").append((float)file.getSize()/1024);
            buffer.append("}}");
            response.setContentType("text/html;charset=utf-8;");
            response.getWriter().write(buffer.toString());
            response.getWriter().flush();
            response.getWriter().close();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
    @RequestMapping(value="/downloadFile", method = RequestMethod.GET)
    public void download(HttpServletRequest request, HttpServletResponse response,String filePath,String fileName) {
         //String fileName=request.getParameter("fileName");
         try {
             fileName=new String(fileName.getBytes("ISO-8859-1"),"UTF-8");
             filePath=new String(filePath.getBytes("ISO-8859-1"),"UTF-8");
         } catch (UnsupportedEncodingException e1) {
             // TODO Auto-generated catch block
             e1.printStackTrace();
         }

         OutputStream output = null;
         BufferedInputStream bis=null;
         BufferedOutputStream bos=null;
          FileManager fileMgr=FileSysFactory.getInstance(FileSysType.HDFS);
         //response.setContentType("application/x-msdownload");  
         //response.setCharacterEncoding("UTF-8");
         try {
             output = response.getOutputStream();
             response.setHeader("Content-Disposition","attachment;filename="+new String(fileName.getBytes("gbk"),"iso-8859-1"));
             bis=new BufferedInputStream(fileMgr.getFile(filePath));
             bos=new BufferedOutputStream(output);
             byte[] buff=new byte[2048];
             int bytesRead;
             while(-1!=(bytesRead=bis.read(buff,0,buff.length))){
                 bos.write(buff,0,bytesRead);
             }
         } catch (UnsupportedEncodingException e) {
             // TODO Auto-generated catch block
             e.printStackTrace();
         } catch (IOException e) {
             // TODO Auto-generated catch block
             e.printStackTrace();
         } finally{
             if(bis!=null)
                 try {
                     bis.close();
                 } catch (IOException e) {
                     // TODO Auto-generated catch block
                     e.printStackTrace();
                 }
             if(bos!=null)
                 try {
                     bos.close();
                 } catch (IOException e) {
                     // TODO Auto-generated catch block
                     e.printStackTrace();
                 }
         }
         
     }
    @RequestMapping("/delete")
    public void delete(String path, HttpServletResponse response){
        FileManager fileMgr=FileSysFactory.getInstance(FileSysType.HDFS);
        try {
            fileMgr.deleteFile(path);
            StringBuffer buffer = new StringBuffer();
            buffer.append("{success:true}");
            response.setContentType("text/html;charset=utf-8;");
            response.getWriter().write(buffer.toString());
            response.getWriter().flush();
            response.getWriter().close();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
    @RequestMapping(value ="/queryMovie",method=RequestMethod.GET)
    public void queryMovie( HttpServletRequest request, HttpServletResponse response){
        response.setContentType("application/octet-stream");
        FileManager fileMgr = FileSysFactory.getInstance(FileSysType.HDFS);
        FSDataInputStream fsInput = fileMgr.getStreamFile("/filesharesystem/admin/test/a.swf");
        OutputStream os;
        try {
            os = response.getOutputStream();
            IOUtils.copyBytes(fsInput, os, 4090, false);
            os.flush();
            os.close();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } 

    }
    
    /**
     * 根据提交的路径，从云上获取swf格式和picture文档内容
     */
    @RequestMapping(value = "/queryResource", method = RequestMethod.GET)
    public void queryResource(HttpServletRequest request,
            HttpServletResponse response, String resourcePath) {
        response.setContentType("application/octet-stream");
        FileManager fileMgr = FileSysFactory.getInstance(FileSysType.HDFS);
        FSDataInputStream fsInput = fileMgr.getStreamFile(resourcePath);
        OutputStream os = null;
        try {
            os = response.getOutputStream();
            IOUtils.copyBytes(fsInput, os, 4090, false);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } finally{
            
            try {
                if(os != null){
                    os.flush();
                    os.close();
                }
                if(fsInput!= null){
                    fsInput.close();
                }
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    }

    @RequestMapping(value = "/queryListForTree", method = RequestMethod.POST)
    @ResponseBody
    public List<CFile> queryListForTree(HttpServletRequest request) {
        HttpSession session=request.getSession(true);
        BaseUsers u = (BaseUsers) session.getAttribute(WebConstants.CURRENT_USER);
        Criteria criteria = new Criteria();
        criteria.put("username", u.getAccount());
        return dfsService.queryListForTree(criteria);
    }
    
 
    @RequestMapping(value="/status",method=RequestMethod.GET)
    public String dfsIndex(HttpServletRequest request,HttpSession session,Model model){
        
        return "plugins/cloud/web/views/hadoop/dfs/dfsStatus";
    }
    
    @RequestMapping(value="/status", method = RequestMethod.POST)
    @ResponseBody
    public Object get(HttpSession session){
        try{
            DfsInfo dfs = dfsService.getDfsInfo();
            if(dfs != null){
                return new ExtFormReturn(true,dfs);
            }else{
                return new ExtFormReturn(false);
            }
            
        }catch(Exception e){
            
            e.printStackTrace();
            return new ExceptionReturn(e);
            
        }
    
    }
    
    @RequestMapping(value="/dataNode", method = RequestMethod.POST)
    @ResponseBody
    public Object dataNode(){
        try{
            return dfsService.getDataNodeList();
            
        }catch(Exception e){
            
            e.printStackTrace();
            return new ExceptionReturn(e);
            
        }
    
    }
}

重点应用了，上传，删除，查询云操作

其中，hadoop的几个实体类如下：

CFile.java

package cn.edu.hbcf.common.hadoop;

import java.util.List;

public class CFile {
    private String name;
    private long size;
    private int leaf;
    private String createTime;
    private List<CFile> children;
    private String fullPath;
    
    public String getFullPath() {
        return fullPath;
    }
    public void setFullPath(String fullPath) {
        this.fullPath = fullPath;
    }
    public String getName() {
        return name;
    }
    public void setName(String name) {
        this.name = name;
    }
    public long getSize() {
        return size;
    }
    public void setSize(long size) {
        this.size = size;
    }
    public int getLeaf() {
        return leaf;
    }
    public void setLeaf(int leaf) {
        this.leaf = leaf;
    }
    public List<CFile> getChildren() {
        return children;
    }
    public void setChildren(List<CFile> children) {
        this.children = children;
    }
    public String getCreateTime() {
        return createTime;
    }
    public void setCreateTime(String createTime) {
        this.createTime = createTime;
    }
}

FileManager.java

package cn.edu.hbcf.common.hadoop;

import java.io.DataInputStream;
import java.io.InputStream;
import java.util.List;

import org.apache.hadoop.fs.FSDataInputStream;

public interface FileManager {
    public void createDirForUser(String dirName);
    public boolean isDirExistForUser(String uname);
    public void deleteFile(String fileName);
    public void getFile(String userName,String remoteFileName,String localFileName);
    public void putFile(String directoryName,String remoteFileName,String localFileName);
    public List<CFile> listFileForUser(String uname,String filePrefix);
    public void putFile(InputStream input,String path,String remoteFileName);
    public DataInputStream getFile(String remoteFileName);
    public FSDataInputStream getStreamFile(String remoteFileName);
    
    public boolean isFileExist(String filePath);
}

FileSysFactory.java

package cn.edu.hbcf.common.hadoop;

public class FileSysFactory {
    private static FileManager instanceApacheHDFS=null;
    private final static FileManager instanceNormalFileSys=null;
    public static synchronized FileManager getInstance(FileSysType fileSysType){
        switch(fileSysType){
        case HDFS:
            if(instanceApacheHDFS==null){
                instanceApacheHDFS=new HDFSFileManager();
            }
            
            return instanceApacheHDFS;
        case NormalFileSys:
            
        }
        return null;
    }
}

FileSysType.java

package cn.edu.hbcf.common.hadoop;

public enum FileSysType {
    HDFS,NormalFileSys
}

HDFSFileManager.java

package cn.edu.hbcf.common.hadoop;

import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import cn.edu.hbcf.common.springmvc.CustomizedPropertyConfigurer;

public class HDFSFileManager implements FileManager {
    private final static String root_path="/filesharesystem/";
    private static FileSystem fs=null;
    public HDFSFileManager(){
        if(fs==null){
            try{
                Configuration conf=new Configuration(true);
                String url =(String)CustomizedPropertyConfigurer.getContextProperty("hdfs.URL");
                //conf.set("fs.default.name", url);
                conf.set("fs.defaultFS", url);
                fs=FileSystem.get(conf);
            }catch(IOException e){
                e.printStackTrace();
            }
        }
    }
    public void createDirForUser(String dirName) {
        // TODO Auto-generated method stub
        try {
            Path f=new Path(root_path);
            fs.mkdirs(new Path(f,dirName));
            
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    public void deleteFile(String fileName) {
        // TODO Auto-generated method stub
        try{
            Path f=new Path(fileName);
            fs.delete(f);
        }catch(IOException e){
            e.printStackTrace();
        }
    }

    public void getFile(String userName, String remoteFileName,
            String localFileName) {
        // TODO Auto-generated method stub

    }

    public DataInputStream getFile(String remoteFileName) {
        // TODO Auto-generated method stub
        try {
            return fs.open(new Path(remoteFileName));
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return null;
    }

    public FSDataInputStream getStreamFile(String remoteFileName) {
        // TODO Auto-generated method stub
        try {
            return fs.open(new Path(remoteFileName));
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return null;
    }

    public boolean isDirExistForUser(String uname) {
        // TODO Auto-generated method stub
        Path f=new Path(root_path+uname);
        try {
            return fs.exists(f);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return false;
    }

    public List<CFile> listFileForUser(String uname, String filePrefix) {
        List<CFile> list=new ArrayList<CFile>();
        String fp=root_path+uname+"/"+filePrefix;
        Path path=new Path(fp);  
        try {
            FileStatus[] stats = fs.listStatus(path);
            for(int i=0;i<stats.length;i++){
                CFile file=new CFile();
                int len=stats[i].getPath().toString().lastIndexOf("/");
                String fileName=stats[i].getPath().toString().substring(len+1);
                java.text.Format formatter=new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 
                String createTime=formatter.format(stats[i].getModificationTime());
                if(stats[i].isDir()){
                    file.setLeaf(0);
                    file.setSize(stats[i].getBlockSize());
                    file.setName(fileName);
                    file.setCreateTime(createTime);
                    String newpath=filePrefix+"/"+fileName;
                    file.setChildren(listFileForUser(uname,newpath));
                    file.setFullPath(stats[i].getPath().toString().replace("hdfs://master:9000", ""));
                    list.add(file);
                    
                }else{
                    file.setLeaf(1);
                    file.setName(fileName);
                    file.setCreateTime(createTime);
                    file.setFullPath(stats[i].getPath().toString().replace("hdfs://master:9000", ""));
                    list.add(file);
                }
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }  

        return list;
    }

    public void putFile(String directoryName, String remoteFileName,
            String localFileName) {
        // TODO Auto-generated method stub

    }

    public void putFile(InputStream input, String path,
            String remoteFileName) {
        // TODO Auto-generated method stub
        final int taskSize=1024;
        BufferedInputStream istream=null;
        DataOutputStream ostream=null;
        istream=new BufferedInputStream(input);
        try {
            ostream=fs.create(new Path(path+"/"+remoteFileName));
            int bytes;
            byte[] buffer=new byte[taskSize];
            while((bytes=istream.read(buffer))>=0){
                ostream.write(buffer,0,bytes);
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            try {
                if(istream!=null)
                istream.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            try {
                if(ostream!=null)
                ostream.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    }
    @Override
    public boolean isFileExist(String filePath) {
        // TODO Auto-generated method stub
        Path f=new Path(filePath);
        try {
            return fs.exists(f);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return false;
    }

}

用到的几个jar包：

<dependency>
            <groupId>org.apache.solr</groupId>
            <artifactId>solr-solrj</artifactId>
            <version>4.10.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.4.0</version>
            <exclusions>
                <exclusion>
                    <artifactId>jasper-runtime</artifactId>
                    <groupId>tomcat</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>jasper-compiler</artifactId>
                    <groupId>tomcat</groupId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>  
            <groupId>org.apache.hadoop</groupId>  
            <artifactId>hadoop-hdfs</artifactId>  
            <!-- <version>2.0.0-cdh4.4.0</version>   -->
            <version>2.2.0</version>
            <exclusions>  
                <exclusion>  
                    <artifactId>jdk.tools</artifactId>  
                    <groupId>jdk.tools</groupId>  
                </exclusion>  
            </exclusions>  
        </dependency>  

 最后，需要配置一下环境变量：

HADOOP_HOME
C:UsersPromiseDesktophadoop-common-2.2.0-bin-master