zoukankan      html  css  js  c++  java
  • sparkStreaming(2.1.0)示范代码

    package cn.piesat

    import org.apache.spark.SparkConf
    import org.apache.spark.sql.SparkSession
    import org.apache.spark.streaming.{Seconds, StreamingContext}

    object SparkApp {
    def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount")
    val ssc = new StreamingContext(conf, Seconds(5))
    val lines = ssc.socketTextStream("localhost", 9999)
    val words=lines.flatMap(_.split(" "))
    words.foreachRDD(rdd=>{
    val spark=SparkSession.builder().config(rdd.sparkContext.getConf).getOrCreate()
    import spark.implicits._
    val worldDataFrame=rdd.map(w=>{
    Record(w)
    }).toDF()
    worldDataFrame.createOrReplaceTempView("words")
    val wordCountsDataFram=spark.sql("select word,count(*) as total from words group by word")
    wordCountsDataFram.show()
    })
    ssc.start()
    ssc.awaitTermination()
    }
    }

    case class Record(val word:String) {

    }
  • 相关阅读:
    Park Visit
    1894: 985的方格难题
    985的数字难题
    Highways
    最短路
    SQL内容补充
    8.前端资源优化
    7.CSRF攻击和文件上传漏洞攻击
    6.XSS攻击方式及防御措施
    5.避免重复提交表单
  • 原文地址:https://www.cnblogs.com/runnerjack/p/9515032.html
Copyright © 2011-2022 走看看