zoukankan      html  css  js  c++  java
  • Spark往Elasticsearch读写数据

    def main(args: Array[String]): Unit = {
    
        val sparkConf = new SparkConf().setAppName("DecisionTree1").setMaster("local[2]")
        sparkConf.set("es.index.auto.create", "true")
        sparkConf.set("es.nodes", "10.3.162.202")
        sparkConf.set("es.port", "9200")
        val sc = new SparkContext(sparkConf)
        //write2Es(sc)
        read4Es(sc);
      }
    
      def write2Es(sc: SparkContext) = {
        val numbers = Map("one" -> 1, "two" -> 2, "three" -> 3)
        val airports = Map("OTP" -> "Otopeni", "SFO" -> "San Fran")
        var rdd = sc.makeRDD(Seq(numbers, airports))
        EsSpark.saveToEs(rdd, "spark/docs")
        println("--------------------End-----------------")
      }
    
      def read4Es(sc: SparkContext) {
        val rdd = EsSpark.esRDD(sc, "spark/docs")
        rdd.foreach(line => {
          val key = line._1
          val value = line._2
          println("------------------key:" + key)
          for (tmp <- value) {
            val key1 = tmp._1
            val value1 = tmp._2
            println("------------------key1:" + key1)
            println("------------------value1:" + value1)
          }
        })
      }
    

    例子依赖jar:elasticsearch-spark_2.10-2.1.0.jar

  • 相关阅读:
    ....
    排序相关的问题(jq,java)_1123
    Spring aop 记录操作日志
    vue -element ui 自定义验证规则,封装在公共的文件里
    vue
    ES6 新特性
    正则表达式
    面向对象基础--继承(2)
    面向对象基础(1)
    安装vue环境
  • 原文地址:https://www.cnblogs.com/qq27271609/p/4689976.html
Copyright © 2011-2022 走看看