zoukankan      html  css  js  c++  java
  • Scala_Load csv data to hive via spark2.1_via pass parameters_HiveAllType

    prepare CSV data

    NT,col_SMALLINT,col_BIGINT,col_INT,col_FLOAT,col_DOUBLE,col_DECIMAL,col_TIMESTAMP,col_DATE,col_INTERVAL,col_STRING,col_VARCHAR,col_CHAR,col_BOOLEAN,col_BINARY
    100Y,100S,100L,100,1.11,2.22,4.44,2013­01-01 01:50:50,2013­01-01,2 SECONDS,"""123""",ab,a,TRUE,111

    create CSV file

    [Dev root @ sd-9c1f-2eac /tmp/pl62716]
    # vi CsvLoadToHive.csv
    col_TINYINT,col_SMALLINT,col_BIGINT,col_INT,col_FLOAT,col_DOUBLE,col_DECIMAL,col_TIMESTAMP,col_DATE,col_INTERVAL,col_STRIN
    G,col_VARCHAR,col_CHAR,col_BOOLEAN,col_BINARY
    100Y,100S,100L,100,1.11,2.22,4.44,2013­01-01 01:50:50,2013­01-01,2 SECONDS,"""123""",ab,a,TRUE,111

    scala test code

    package com.liupu
    import org.apache.spark.{ SparkContext, SparkConf }
    import org.apache.spark.sql.hive.HiveContext
    import org.apache.spark.sql.hive.orc._
    object LoadCsv3 {
      def main(args: Array[String]) {
        var sourceCsvPath = args(0)
        var targetPath = args(1)
        var hiveTableName = args(2)
    
        var sc = new SparkContext()
        val hiveContext = new org.apache.spark.sql.hive.HiveContext(sc)
        val df = hiveContext.read
          .format("com.databricks.spark.csv")
          .option("header", "true")
          .option("inferSchema", "true")
          .load(sourceCsvPath)
        val selectedData = df.select("col_TINYINT","col_SMALLINT","col_BIGINT","col_INT","col_FLOAT","col_DOUBLE","col_DECIMAL","col_TIMESTAMP","col_DATE","col_INTERVAL","col_STRING","col_VARCHAR","col_CHAR","col_BOOLEAN","col_BINARY")
        selectedData.write.format("orc").option("header", "true").save(targetPath)
        hiveContext.sql(s"create external table $hiveTableName(col_TINYINT TINYINT,col_SMALLINT SMALLINT,col_BIGINT BIGINT,col_INT INT,col_FLOAT FLOAT,col_DOUBLE DOUBLE,col_DECIMAL DECIMAL,col_TIMESTAMP TIMESTAMP,col_DATE DATE,col_INTERVAL INTERVAL,col_STRING STRING,col_VARCHAR VARCHAR,col_CHAR CHAR,col_BOOLEAN BOOLEAN,col_BINARY BINARY) stored as orc location '$targetPath'")
        hiveContext.sql("show tables").collect().foreach(println)
        sc.stop()
      }
    }

    spark test

  • 相关阅读:
    Git 学习小问题记录
    Spring缓存源码剖析:(一)工具选择
    最佳线程数
    Python 装饰器备忘
    使用SCSS扩展Bootstrap4
    Flask 路由相关操作
    Flask开发环境搭建
    Python数据分析开发环境
    Python中的矩阵操作
    Windows 安装 MySQL 8.0.11
  • 原文地址:https://www.cnblogs.com/liupuLearning/p/6567822.html
Copyright © 2011-2022 走看看