zoukankan      html  css  js  c++  java
  • 验证一下spark Row getAS类型以及控制问题

    package com.javartisan.demo
    
    import org.apache.spark.sql.SparkSession
    
    object SparkLocal {
    
      def main(args: Array[String]): Unit = {
    
        val spark = SparkSession.builder().master("local[*]").appName("spark test").getOrCreate()
        import spark.implicits._
    
        val sc = spark.sparkContext
        val a: Int = 1
        val b: Int = 1
        val c: String = "1"
        val d: String = "1"
    
        val arr1 = Array[(Int, Int, String, String)]((a, b, c, d))
        val arr2 = Array[(Int, Int, String, String)]((2, b, c, d))
        val rdd1 = sc.parallelize[(Int, Int, String, String)](arr1)
        val rdd2 = sc.parallelize[(Int, Int, String, String)](arr2)
        val df1 = rdd1.toDF("a", "b", "c", "d")
        val df2 = rdd2.toDF("a1", "b1", "c1", "d1")
        df1.printSchema()
        df2.printSchema()
    
        val full = df1.join(df2, $"a" === $"a1", "full")
        val newFull = full.rdd.map(row => {
          //GenericRowWithSchema
          println("row class " + row.getClass)
          row
        })
        println(newFull.count())
        full.show(false)
        println(df1.count())
        println(df2.count())
        spark.stop()
      }
    }
    

      

  • 相关阅读:
    kubernetes构架及组件介绍
    二进制部署k8s
    Git
    Redis sentinel
    redis主从复制
    k8s安装
    基于Jenkins实现可腹部回滚的cicd平台
    Redis基础命令和持久化
    构建自动发现的Docker服务架构
    Redis
  • 原文地址:https://www.cnblogs.com/leodaxin/p/10948959.html
Copyright © 2011-2022 走看看