zoukankan      html  css  js  c++  java
  • Scala_Load csv data to hive via spark2.1

    code:

    package com.liupu
    import org.apache.spark.{ SparkContext, SparkConf }
    import org.apache.spark.sql.hive.HiveContext
    import org.apache.spark.sql.hive.orc._
    object HiveContextLoadCsv {
      def main(args: Array[String]) {
        var sc = new SparkContext()
        val hiveContext = new org.apache.spark.sql.hive.HiveContext(sc)
        val df = hiveContext.read
          .format("com.databricks.spark.csv")
          .option("header", "true")
          .option("inferSchema", "true")
          .load("/tmp/cars.csv")
        val selectedData = df.select("year", "model")
        selectedData.write.format("orc").option("header", "true").save("/tmp/hive_cars")
        hiveContext.sql("create external table hive_cars(year int,model string) stored as orc location '/tmp/hive_cars'")
        hiveContext.sql("show tables").collect().foreach(println)
        hiveContext.sql("select * from hive_cars").collect().foreach(println)
        sc.stop()
      }
    }

    spark submit:

    ./spark-submit 
    --class com.liupu.HiveContextLoadCsv 
    --master local[*] 
    /home/pl62716/scalaTest.jar
  • 相关阅读:
    Nginx的访问控制
    远程登录
    Linux的网络命令
    laravel-collect
    laravel-model
    laravel-Macroable
    laravel-容器
    机器学习-Logisitic回归
    机器学习-多变量线性回归
    算法笔记-分支界限法
  • 原文地址:https://www.cnblogs.com/liupuLearning/p/6566908.html
Copyright © 2011-2022 走看看