zoukankan      html  css  js  c++  java
  • Spark往Elasticsearch读写数据

    def main(args: Array[String]): Unit = {
    
        val sparkConf = new SparkConf().setAppName("DecisionTree1").setMaster("local[2]")
        sparkConf.set("es.index.auto.create", "true")
        sparkConf.set("es.nodes", "10.3.162.202")
        sparkConf.set("es.port", "9200")
        val sc = new SparkContext(sparkConf)
        //write2Es(sc)
        read4Es(sc);
      }
    
      def write2Es(sc: SparkContext) = {
        val numbers = Map("one" -> 1, "two" -> 2, "three" -> 3)
        val airports = Map("OTP" -> "Otopeni", "SFO" -> "San Fran")
        var rdd = sc.makeRDD(Seq(numbers, airports))
        EsSpark.saveToEs(rdd, "spark/docs")
        println("--------------------End-----------------")
      }
    
      def read4Es(sc: SparkContext) {
        val rdd = EsSpark.esRDD(sc, "spark/docs")
        rdd.foreach(line => {
          val key = line._1
          val value = line._2
          println("------------------key:" + key)
          for (tmp <- value) {
            val key1 = tmp._1
            val value1 = tmp._2
            println("------------------key1:" + key1)
            println("------------------value1:" + value1)
          }
        })
      }
    

    例子依赖jar:elasticsearch-spark_2.10-2.1.0.jar

  • 相关阅读:
    max_element( )
    dp
    dfs
    dp
    区间dp
    树形dp
    dp-最长回文串
    go 结构体函数
    go 结构体初始化
    Golang数组和切片的区别
  • 原文地址:https://www.cnblogs.com/qq27271609/p/4689976.html
Copyright © 2011-2022 走看看