zoukankan      html  css  js  c++  java
  • 转 Scala更改文件目录结构

    原文链接:http://blog.csdn.net/gavin_chun/article/details/78652638

    原文件目录结构

    hdfs://192.168.187.111:8020/spark/emp/temp/201711112025/d=20171111/h=20/part-r-00000-6ba69620-ba52-4cb1-9ea0-6634ae0e16bc.txt
    • 1

    更改后的目录结构

    hdfs://192.168.187.111:8020/spark/emp/data/d=20171111/h=20/201711112025-1.txt
    • 1
    package com.sparkopHDFS
    
    import java.net.URI
    import org.apache.hadoop.conf.Configuration
    import org.apache.hadoop.fs.{FileSystem, Path}
    import org.apache.spark.deploy.SparkHadoopUtil
    
    /**
      * Created with IntelliJ IDEA.
      * Description: 
      * Author: A_ChunUnique
      * Date: 2017/11/27
      * Time: 17:41
      *
      **/
    object MvPath {
      def main(args: Array[String]): Unit = {
        val conf = new Configuration()
        conf.set("fs.default.name", "hdfs://192.168.187.111:8020")
        val fileSystem = FileSystem.get(URI.create("hdfs://192.168.187.111:8020/spark/emp/temp/201711112025/d=20171111/h=20"), conf)
        val outputPath = "hdfs://192.168.187.111:8020/spark/emp/"
        val loadTime = "201711112025"
        val partition = "/d=20171111/h=20"
        changeFileName(fileSystem, outputPath, loadTime, partition)
      }
    
      def changeFileName(fileSystem: FileSystem, outputPath: String, loadTime: String, partition: String): Unit = {
        //返回一个Path序列(hdfs://192.168.187.111:8020/spark/emp/temp/201711112025/d=20171111/h=20/*.txt)
        val paths = SparkHadoopUtil.get.globPath(new Path(outputPath + "temp/" + loadTime + partition + "/*.txt"))
        var times = 0
        paths.map(x => {
          //将hdfs://192.168.187.111:8020/spark/emp/temp/201711112025替换成hdfs://192.168.187.111:8020/spark/emp/data/
          //hdfs://192.168.187.111:8020/spark/emp/data/d=20171111/h=20/part-r-00002-6ba69620-ba52-4cb1-9ea0-6634ae0e16bc.txt
          var newLocation = x.toString.replace(outputPath + "temp/" + loadTime, outputPath + "data/")
          println("1:" + newLocation)
          //hdfs://192.168.187.111:8020/spark/emp/data/d=20171111/h=20/00002-6ba69620-ba52-4cb1-9ea0-6634ae0e16bc.txt
          newLocation = newLocation.replace("part-r-", "")
          //获取最后一个"/"的索引
          val index = newLocation.lastIndexOf("/")
          times += 1
          //hdfs://192.168.187.111:8020/spark/emp/data/d=20171111/h=20/201711112025-times.txt
          newLocation = newLocation.substring(0, index + 1) + loadTime + "-" + times + ".txt"
          println("2:" + newLocation)
          val officialPath = new Path(newLocation)
          //如果hdfs://192.168.187.111:8020/spark/emp/data/d=20171111/h=20/不存在,就创建它
          if (!fileSystem.exists(officialPath.getParent)) {
            fileSystem.mkdirs(officialPath.getParent)
          }
          fileSystem.rename(x, officialPath)
        })
      }
    }
  • 相关阅读:
    BZOJ2241 [SDOI2011]打地鼠 【模拟】
    K-D tree入门
    BZOJ1924 [Sdoi2010]所驼门王的宝藏 【建图 + tarjan】
    BZOJ1925 [Sdoi2010]地精部落 【dp】
    BZOJ1926 [Sdoi2010]粟粟的书架 【主席树 + 二分 + 前缀和】
    BZOJ2457 [BeiJing2011]双端队列 【贪心】
    二进制集合枚举子集
    BZOJ1923 [Sdoi2010]外星千足虫 【高斯消元】
    BZOJ1922 [Sdoi2010]大陆争霸 【最短路】
    HDU3157:Crazy Circuits——题解
  • 原文地址:https://www.cnblogs.com/kxgdby/p/7956576.html
Copyright © 2011-2022 走看看