zoukankan      html  css  js  c++  java
  • hive executeTask被interrupt处理

    异常信息如下:

    java.io.IOException: Failed on local exception: java.nio.channels.ClosedByInterruptException; Host Details : local host is: "hadoop008/192.168.28.77"; destination host is: "hadoop004":8020; 

    at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:764)
    at org.apache.hadoop.ipc.Client.call(Client.java:1415)
    at org.apache.hadoop.ipc.Client.call(Client.java:1364)
    at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
    at com.sun.proxy.$Proxy17.setReplication(Unknown Source)
    at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.setReplication(ClientNamenodeProtocolTranslatorPB.java:322)
    at sun.reflect.GeneratedMethodAccessor23.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
    at com.sun.proxy.$Proxy18.setReplication(Unknown Source)
    at org.apache.hadoop.hdfs.DFSClient.setReplication(DFSClient.java:1768)
    at org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:465)
    at org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:461)
    at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
    at org.apache.hadoop.hdfs.DistributedFileSystem.setReplication(DistributedFileSystem.java:461)
    at org.apache.hadoop.mapreduce.JobSubmitter.copyRemoteFiles(JobSubmitter.java:142)
    at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:214)
    at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:388)
    at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:481)
    at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1295)
    at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1292)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:415)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1642)
    at org.apache.hadoop.mapreduce.Job.submit(Job.java:1292)
    at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:564)
    at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:559)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:415)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1642)
    at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:559)
    at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:550)
    at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:420)
    at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:136)
    at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:153)
    at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:85)
    at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1554)
    at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1321)
    at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1139)
    at org.apache.hadoop.hive.ql.Driver.run(Driver.java:962)
    at org.apache.hadoop.hive.ql.Driver.run(Driver.java:957)
    at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:145)
    at org.apache.hive.service.cli.operation.SQLOperation.access$000(SQLOperation.java:69)
    at org.apache.hive.service.cli.operation.SQLOperation$1$1.run(SQLOperation.java:200)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:415)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1642)
    at org.apache.hadoop.hive.shims.HadoopShimsSecure.doAs(HadoopShimsSecure.java:502)
    at org.apache.hive.service.cli.operation.SQLOperation$1.run(SQLOperation.java:213)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
    at java.util.concurrent.FutureTask.run(FutureTask.java:262)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
    Caused by: java.nio.channels.ClosedByInterruptException
    at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
    at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:681)
    at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:192)
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:529)
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:493)
    at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:606)
    at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:700)
    at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:367)
    at org.apache.hadoop.ipc.Client.getConnection(Client.java:1463)
    at org.apache.hadoop.ipc.Client.call(Client.java:1382)

    ... 54 more



    解决办法:

    为HiveOptions.callTimeout设置合适的rpc timeout  


    参考:

    https://issues.apache.org/jira/browse/FLUME-1748

    http://stackoverflow.com/questions/32186212/hive-is-failing-to-write-data-in-hdfs

  • 相关阅读:
    Mysql千万级访问量架构
    [JAVA]《Java 核心技术》(一)
    [Linux 命令]获取当前目录下某种格式文件个数
    [项目bug收集整理3]get页面,post页面的url 参数预防问题
    [项目bug收集整理2] Map的key,如果是非内置对象,需要自己实现hashCode和equals方法
    [项目bug收集整理1]Integer 对象的相等比较
    锂电池保护原理
    (转载)mos管电压规格是什么,什么是VMOS管栅极
    (转载) MOS管区分NP沟道
    Idea中使用Lombok 编译报找不到符号
  • 原文地址:https://www.cnblogs.com/dailidong/p/7571152.html
Copyright © 2011-2022 走看看