zoukankan      html  css  js  c++  java
  • SparkSQL小例子

    详情请看:http://www.ibm.com/developerworks/cn/opensource/os-cn-spark-practice3/

    import org.apache.spark.SparkConf
    import org.apache.spark.SparkContext
    import org.apache.spark.sql.SQLContext
    import org.apache.spark.sql.types._
    import org.apache.spark.sql.Row
    import org.apache.spark.rdd.RDD
    
    object PeopleDataStatisticSparkSQL {
      def main(args: Array[String]): Unit = {
        val conf = new SparkConf().setAppName("SparkSQL").setMaster("local");
        val sc = new SparkContext(conf)
        val peopleDataRDD = sc.textFile("/Users/lihu/Desktop/crawle/xingbie.txt")
        val sqlCtx = new SQLContext(sc)
        import sqlCtx.implicits._
        val shemaArray = Array("id", "gender", "height")
        val schema = StructType(shemaArray.map(StructField(_, StringType, true)))
        val rowRDD:RDD[Row] = peopleDataRDD.map(_.split(" ")).map(eachRow => Row(eachRow(0), eachRow(1), eachRow(2)))
        val peopleDF = sqlCtx.createDataFrame(rowRDD, schema)
        peopleDF.registerTempTable("people")
        println(sqlCtx.sql("select id from people where height > 180 and gender = 'M'").count())
        println(sqlCtx.sql("select id from people where height > 170 and gender = 'F'").count())
        println(peopleDF.filter(peopleDF("gender").equalTo("M")).filter(peopleDF("height") > 165).count())
        peopleDF.groupBy(peopleDF("gender")).count().show()
        peopleDF.filter(peopleDF("gender").equalTo("M")).filter(peopleDF("height") > 165).show(2)
        peopleDF.sort($"height".desc).take(3).foreach{row => println(row(0) + " " + row(1) + " " + row(2))}
        peopleDF.groupBy(peopleDF("gender")).agg(Map("height" -> "avg")).show()
        peopleDF.groupBy(peopleDF("gender")).agg(Map("height" -> "max")).show()
        peopleDF.groupBy(peopleDF("gender")).agg(Map("height" -> "min")).show()
      }
    }
    
  • 相关阅读:
    Vue 框架怎么实现对象和数组的监听?
    能说下 vue-router 中常用的 hash 和 history 路由模式实现原理吗?
    vue-router 路由模式有几种?
    Vue 组件间通信有哪几种方式?
    v-model 的原理?
    华硕笔记本修复
    linux下制作u盘启动盘
    virtualbox不能启动虚拟机
    ubuntu14.04建立wifi热点
    git中文文件名和中文目录显示乱码
  • 原文地址:https://www.cnblogs.com/sunyaxue/p/6373456.html
Copyright © 2011-2022 走看看