zoukankan      html  css  js  c++  java
  • SparkException: Could not find CoarseGrainedScheduler or it has been stopped.

    org.apache.spark.SparkException: Could not find CoarseGrainedScheduler or it has been stopped.
            at org.apache.spark.rpc.netty.Dispatcher.postMessage(Dispatcher.scala:163)
            at org.apache.spark.rpc.netty.Dispatcher.postOneWayMessage(Dispatcher.scala:133)
            at org.apache.spark.rpc.netty.NettyRpcEnv.send(NettyRpcEnv.scala:192)
            at org.apache.spark.rpc.netty.NettyRpcEndpointRef.send(NettyRpcEnv.scala:516)
            at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.reviveOffers(CoarseGrainedSchedulerBackend.scala:356)
            at org.apache.spark.scheduler.TaskSchedulerImpl.executorLost(TaskSchedulerImpl.scala:494)
            at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend$DriverEndpoint.disableExecutor(CoarseGrainedSchedulerBackend.scala:301)
            at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:121)
            at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:120)
            at scala.Option.foreach(Option.scala:236)
            at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint.onDisconnected(YarnSchedulerBackend.scala:120)
            at org.apache.spark.rpc.netty.Inbox$$anonfun$process$1.apply$mcV$sp(Inbox.scala:142)
            at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:204)
            at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100)
            at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:217)
            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
            at java.lang.Thread.run(Thread.java:745)
    18/10/14 22:23:26 ERROR netty.Inbox: Ignoring error
    org.apache.spark.SparkException: Could not find CoarseGrainedScheduler or it has been stopped.
            at org.apache.spark.rpc.netty.Dispatcher.postMessage(Dispatcher.scala:163)
            at org.apache.spark.rpc.netty.Dispatcher.postOneWayMessage(Dispatcher.scala:133)
            at org.apache.spark.rpc.netty.NettyRpcEnv.send(NettyRpcEnv.scala:192)
            at org.apache.spark.rpc.netty.NettyRpcEndpointRef.send(NettyRpcEnv.scala:516)
            at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.reviveOffers(CoarseGrainedSchedulerBackend.scala:356)
            at org.apache.spark.scheduler.TaskSchedulerImpl.executorLost(TaskSchedulerImpl.scala:494)
            at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend$DriverEndpoint.disableExecutor(CoarseGrainedSchedulerBackend.scala:301)
            at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:121)
            at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:120)
            at scala.Option.foreach(Option.scala:236)
            at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint.onDisconnected(YarnSchedulerBackend.scala:120)
            at org.apache.spark.rpc.netty.Inbox$$anonfun$process$1.apply$mcV$sp(Inbox.scala:142)
            at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:204)
            at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100)
            at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:217)
            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
            at java.lang.Thread.run(Thread.java:745)
    
    ......
    
    java.util.concurrent.RejectedExecutionException: Task scala.concurrent.impl.CallbackRunnable@708dfce7 rejected from java.util.concurrent.ThreadPoolExecutor@346be0ef[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 224]
            at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
            at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
            at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
            at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:122)
            at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)
            at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)
            at scala.concurrent.Promise$class.complete(Promise.scala:55)
            at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)
            at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324)
            at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324)
            at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)
            at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293)
            at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133)
            at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)
            at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)
            at scala.concurrent.Promise$class.complete(Promise.scala:55)
            at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)
            at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:235)
            at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:235)
            at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)
            at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.processBatch$1(Future.scala:643)
            at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply$mcV$sp(Future.scala:658)
            at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply(Future.scala:635)
            at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply(Future.scala:635)
            at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
            at scala.concurrent.Future$InternalCallbackExecutor$Batch.run(Future.scala:634)
            at scala.concurrent.Future$InternalCallbackExecutor$.scala$concurrent$Future$InternalCallbackExecutor$$unbatchedExecute(Future.scala:694)
            at scala.concurrent.Future$InternalCallbackExecutor$.execute(Future.scala:685)
            at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)
            at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)
            at scala.concurrent.Promise$class.tryFailure(Promise.scala:112)
            at scala.concurrent.impl.Promise$DefaultPromise.tryFailure(Promise.scala:153)
            at org.apache.spark.rpc.netty.NettyRpcEnv.org$apache$spark$rpc$netty$NettyRpcEnv$$onFailure$1(NettyRpcEnv.scala:208)
            at org.apache.spark.rpc.netty.NettyRpcEnv$$anonfun$2.apply(NettyRpcEnv.scala:230)
            at org.apache.spark.rpc.netty.NettyRpcEnv$$anonfun$2.apply(NettyRpcEnv.scala:230)
            at org.apache.spark.rpc.netty.RpcOutboxMessage.onFailure(Outbox.scala:71)
            at org.apache.spark.network.client.TransportResponseHandler.failOutstandingRequests(TransportResponseHandler.java:110)
            at org.apache.spark.network.client.TransportResponseHandler.channelUnregistered(TransportResponseHandler.java:124)
            at org.apache.spark.network.server.TransportChannelHandler.channelUnregistered(TransportChannelHandler.java:94)
            at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)
            at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)
            at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:53)
            at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)
            at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)
            at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:53)
            at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)
            at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)
            at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:53)
            at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)
            at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)
            at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:739)
            at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:659)
            at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:328)
            at io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:627)
            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:362)

    解决办法:

    1. 在提交代码的时候添加配置spark-submit  ...  –conf spark.dynamicAllocation.enabled=false
    2. 也可以在代码中通过SparkConf设置:conf.set(“spark.dynamicAllocation.enabled”,”false”)
  • 相关阅读:
    你不是在拯救世界就是在拯救世界的路上
    你可以去当程序员了
    郭美美是个好姑娘
    据说有个老太太
    生命的尽头
    有关程序的50个至理名言
    程序员是这样的
    新买移动硬盘
    如果有天你看到我疯了,其实就是你疯了
    写字楼里写字间
  • 原文地址:https://www.cnblogs.com/slymonkey/p/10005395.html
Copyright © 2011-2022 走看看