zoukankan      html  css  js  c++  java
  • spark parquet 从hdfs 上读 和写 scala 版本

    import org.apache.spark.SparkConf
    import org.apache.spark.SparkContext
    import org.apache.spark.sql.SQLContext
    import org.apache.spark.sql.DataFrame
    import org.apache.spark.sql.SaveMode
    
    
    object GenericLoadSave {
      
      def main(args: Array[String]): Unit = {
        val conf = new SparkConf()
            .setAppName("GenericLoadSave")
            .setMaster("local")
        val sc = new SparkContext(conf)
        val sqlContext = new SQLContext(sc)
      
        //读取一个parquet文件
        val usersDF = sqlContext.read.format("parquet").load("hdfs://hadoop1:9000/input/users.parquet")
    
        usersDF.write.mode(SaveMode.Overwrite).format("parquet").save("hdfs://hadoop1:9000/output/namesAndFavColors_scala")  
        
        val tDF = sqlContext.read.format("parquet").load("hdfs://hadoop1:9000/output/namesAndFavColors_scala")
        tDF.show()
        
      }
    }

  • 相关阅读:
    ASP.NET Core 静态资源的打包与压缩
    算法
    字符串反转
    js 获取随机数
    AspNetCore MVC 跨域
    add digits
    1-bit and 2-bit Characters
    删除字符串中出现次数最少的字符
    洗牌
    哈夫曼编码
  • 原文地址:https://www.cnblogs.com/TendToBigData/p/10501299.html
Copyright © 2011-2022 走看看