zoukankan      html  css  js  c++  java
  • intellij 调试spark scala 程序 报错

    spark用的是cdh spark-2.0.1
    
    package main.scala
     import org.apache.spark.rdd.RDD
     import org.apache.spark.{SparkConf, SparkContext}
     /* Created by tomtang on 2016/12/16.
      */
    object tomhelloworld {
    
      def FILE_NAME:String = "word_count_results_"
      def main(args: Array[String]) {
    
    //    if (args.length < 1) {
    //      println("Usage:SparkWordCount FileName");
    //      System.exit(1);
    //    }
        val conf = {
          new SparkConf().setMaster("local[*]").setAppName("spark_test")
        }
        val sc = new SparkContext(conf)
        val textFile = sc.textFile("D:\IdeaProjects\first_scala\data\testdata.txt")
        val wordCounts: RDD[(String, Int)] = textFile.flatMap(line => line split " ").map(
          word => (word, 1)).reduceByKey((a, b) => a + b)
        //print the results,for debug use.
        //println("Word Count program running results:")
        //wordCounts.collect().foreach(e => {
        //val (k,v) = e
        //println(k+"="+v)
        //});
        wordCounts.saveAsTextFile(FILE_NAME+System.currentTimeMillis())
        println("Word Count program running results are successfully saved.")
    
        println("Hello World")
      }
    }
    
    报错:
    Exception in thread "main" java.lang.NoSuchMethodError: scala.Predef$.refArrayOps([Ljava/lang/Object;)Lscala/collection/mutable/ArrayOps;
    at org.apache.spark.util.Utils$.getCallSite(Utils.scala:1410)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:77)
    at sparkTraining.SimpleApp$.main(testScala.scala:9)
    at sparkTraining.SimpleApp.main(testScala.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
    
    参考http://blog.csdn.net/u013887486/article/details/53201900
    因为没有按照文档里面的Scala版本号下载对应的Scala,下载对应的版本2.11.7
    
    发现是scala的版本问题。
    
    
    


    选ivy2.10.4

  • 相关阅读:
    BISDN上收集到的SAP BI的极好文章的链接
    如何设置'REUSE_ALV_GRID_DISPLAY'的单个单元格的颜色
    如何设置REUSE_ALV_GRID_DISPLAY'的单个单元格的是否可以输入
    BWABAP to copy aggregates from one cube to another
    SDva01的屏幕增强
    js鼠标悬停效果
    MySQL更新UPDATA的使用
    使用mysql C语言API编写程序—MYSQL数据库查询操作
    MySQL的部分基础语句
    MySQLdelete某个元组||、&&操作
  • 原文地址:https://www.cnblogs.com/tangtianfly/p/6206961.html
Copyright © 2011-2022 走看看