zoukankan      html  css  js  c++  java
  • sparkSQL报错org.apache.spark.sql.AnalysisException: Unable to generate an encoder for inner class `cn.itcast.spark.sql.Intro$People` without access to the scope that this class was defined in.

    学习sparkSQL报错

    org.apache.spark.sql.AnalysisException: Unable to generate an encoder for inner class `cn.itcast.spark.sql.Intro$People` without access to the scope that this class was defined in.
    Try moving this class out of its parent class.;
    
    	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance$$anonfun$apply$34$$anonfun$applyOrElse$14.applyOrElse(Analyzer.scala:2347)
    	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance$$anonfun$apply$34$$anonfun$applyOrElse$14.applyOrElse(Analyzer.scala:2343)
    	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267)
    	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267)
    	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
    	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:266)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsDown$1.apply(QueryPlan.scala:258)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsDown$1.apply(QueryPlan.scala:258)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:279)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1(QueryPlan.scala:289)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$6.apply(QueryPlan.scala:298)
    	at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:298)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDown(QueryPlan.scala:258)
    	at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressions(QueryPlan.scala:249)
    	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance$$anonfun$apply$34.applyOrElse(Analyzer.scala:2343)
    	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance$$anonfun$apply$34.applyOrElse(Analyzer.scala:2339)
    	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$resolveOperators$1.apply(LogicalPlan.scala:62)
    	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$resolveOperators$1.apply(LogicalPlan.scala:62)
    	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
    	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperators(LogicalPlan.scala:61)
    	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance$.apply(Analyzer.scala:2339)
    	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance$.apply(Analyzer.scala:2338)
    	at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85)
    	at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82)
    	at scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124)
    	at scala.collection.immutable.List.foldLeft(List.scala:84)
    	at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:82)
    	at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:74)
    	at scala.collection.immutable.List.foreach(List.scala:392)
    	at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:74)
    	at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.resolveAndBind(ExpressionEncoder.scala:255)
    	at org.apache.spark.sql.Dataset.<init>(Dataset.scala:206)
    	at org.apache.spark.sql.Dataset.<init>(Dataset.scala:170)
    	at org.apache.spark.sql.Dataset$.apply(Dataset.scala:61)
    	at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:472)
    	at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:392)
    	at org.apache.spark.sql.SQLImplicits.rddToDatasetHolder(SQLImplicits.scala:205)
    	at cn.itcast.spark.sql.Intro.sqlIntro(Intro.scala:42)
    	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    	at java.lang.reflect.Method.invoke(Method.java:498)
    	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
    	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
    	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
    	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
    	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
    	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
    	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
    	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
    	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
    	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
    	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
    	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
    	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
    	at org.junit.runner.JUnitCore.run(JUnitCore.java:157)
    	at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
    	at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
    	at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
    	at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
    

      代码内容

    class Intro {
    
      case class People(name: String, age: Int)
    
      @Test
      def sqlIntro(): Unit ={
        val spark = new SparkSession.Builder()
          .master("local[6]")
          .appName("sql_intro")
          .getOrCreate()
    
        import spark.implicits._
    
        val peopleRdd = spark.sparkContext.parallelize(Seq(People("zs", 9), People("ls", 15)))
        val peopleDS = peopleRdd.toDS()
    
        val teenagers = peopleDS.where('age > 10)
          .where('age < 20)
          .select('name)
          .as[String]
    
        teenagers.show()
    
        spark.stop()
      }
    
    }
    

      解决方法:将 People  放到  Intro  外

    class Intro {
    
      @Test
      def sqlIntro(): Unit ={
        val spark = new SparkSession.Builder()
          .master("local[6]")
          .appName("sql_intro")
          .getOrCreate()
    
        import spark.implicits._
    
        val peopleRdd = spark.sparkContext.parallelize(Seq(People("zs", 9), People("ls", 15)))
        val peopleDS = peopleRdd.toDS()
    
        val teenagers = peopleDS.where('age > 10)
          .where('age < 20)
          .select('name)
          .as[String]
    
        teenagers.show()
    
        spark.stop()
      }
    
    }
    
    case class People(name: String, age: Int)
    

      

    
    
    spark
  • 相关阅读:
    好玩夫妻
    笔记整理MS SQL2005 中查询表的字段信息,
    庆幸也与你逛过那一段旅程
    PureMVC
    简单工厂模式
    工厂方法模式
    UML类图
    PureMVC
    oracle双机热备
    一个不错的免费网络硬盘
  • 原文地址:https://www.cnblogs.com/TimeWater/p/14001249.html
Copyright © 2011-2022 走看看