zoukankan      html  css  js  c++  java
  • 读写hdfs文件(工作笔记)

    import java.io.IOException;
    import java.io.InputStream;
    import java.util.List;
    import java.util.Map;
    
    import org.apache.commons.io.IOUtils;
    import org.apache.commons.lang3.StringUtils;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    import com.google.common.collect.Maps;
    
    
    public class LoadHdfsTemplate {
        static final Logger logger = LoggerFactory
                .getLogger(LoadHdfsTemplate.class);
        
        protected FileSystem fileSystem ;
        
        private String hdfsPath = "/template/template.txt";
        
        public final boolean checkFile(String filePath) {
            boolean exists = false;
            try {
                Path path = new Path(filePath);
                exists = fileSystem.exists(path);
            } catch (IOException e) {
                logger.error("接口模板文件不存在!", e);
            } catch (Exception e) {
                logger.error("", e);
            }
            return exists;
        }
        
        public Map<String,Object> readHdfsFile() throws IOException{
            fileSystem = FileSystem.get(new Configuration());
            Path path = new Path(hdfsPath);
            InputStream in = fileSystem.open(path);
            List<String> lines = IOUtils.readLines(in);
            if(null == lines || lines.isEmpty()){
                return null;
            }
            Map<String,Object> map =  Maps.newConcurrentMap();
            int rowNum = 0;
            for(String line : lines){
                rowNum++;
                String[] content = line.split("    ");
                String code = content[0].trim();
                String template = content[1].trim();
                if(StringUtils.isEmpty(line) || StringUtils.isEmpty(template)){
                    logger.error("第{}条模板格式错误!内容为:{}",rowNum,line);
                    continue;
                }
                map.put(code, template);
            }
            return map;
            
        }
    
        public void setHdfsPath(String hdfsPath) {
            this.hdfsPath = hdfsPath;
        }
    }

     应用场景为,读取hdfs上的文件,并转化成<k,v>形式存入内存。

  • 相关阅读:
    c盘清理
    Uploading source images to prepare for transformations HTTP Error 400: POST https://res.ionic.io/api/v1/upload
    maven install 与install:install 的区别
    mysql 、pgsql、oracle 常见分页
    Java 动态代理的原理
    浅谈EC和多副本
    工信部ICP备案后,是否还需要公安备案
    RHEL6.5、RHEL7.2忘记ROOT密码恢复小结
    域名恶意解析的原因是什么
    HTTP 请求头中的 X-Forwarded-For
  • 原文地址:https://www.cnblogs.com/yangsy0915/p/5881679.html
Copyright © 2011-2022 走看看