zoukankan      html  css  js  c++  java
  • intellij 调试spark scala 程序 报错

    spark用的是cdh spark-2.0.1
    
    package main.scala
     import org.apache.spark.rdd.RDD
     import org.apache.spark.{SparkConf, SparkContext}
     /* Created by tomtang on 2016/12/16.
      */
    object tomhelloworld {
    
      def FILE_NAME:String = "word_count_results_"
      def main(args: Array[String]) {
    
    //    if (args.length < 1) {
    //      println("Usage:SparkWordCount FileName");
    //      System.exit(1);
    //    }
        val conf = {
          new SparkConf().setMaster("local[*]").setAppName("spark_test")
        }
        val sc = new SparkContext(conf)
        val textFile = sc.textFile("D:\IdeaProjects\first_scala\data\testdata.txt")
        val wordCounts: RDD[(String, Int)] = textFile.flatMap(line => line split " ").map(
          word => (word, 1)).reduceByKey((a, b) => a + b)
        //print the results,for debug use.
        //println("Word Count program running results:")
        //wordCounts.collect().foreach(e => {
        //val (k,v) = e
        //println(k+"="+v)
        //});
        wordCounts.saveAsTextFile(FILE_NAME+System.currentTimeMillis())
        println("Word Count program running results are successfully saved.")
    
        println("Hello World")
      }
    }
    
    报错:
    Exception in thread "main" java.lang.NoSuchMethodError: scala.Predef$.refArrayOps([Ljava/lang/Object;)Lscala/collection/mutable/ArrayOps;
    at org.apache.spark.util.Utils$.getCallSite(Utils.scala:1410)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:77)
    at sparkTraining.SimpleApp$.main(testScala.scala:9)
    at sparkTraining.SimpleApp.main(testScala.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
    
    参考http://blog.csdn.net/u013887486/article/details/53201900
    因为没有按照文档里面的Scala版本号下载对应的Scala,下载对应的版本2.11.7
    
    发现是scala的版本问题。
    
    
    


    选ivy2.10.4

  • 相关阅读:
    JDBC_批量处理语句提高处理速度
    JDBC_获取插入记录的主键值
    JDBC_获取数据库连接
    SmartSprites 智能批量合并 CSS 雪碧图
    移动前端开发技巧摘录
    将已有项目提交到github/从github上pull到本地
    网页设计创新式布局与交互
    如何打好前端游击战
    jQuery Mobile十大常用技巧
    Javascript图片的懒加载与预加载
  • 原文地址:https://www.cnblogs.com/tangtianfly/p/6206961.html
Copyright © 2011-2022 走看看