zoukankan      html  css  js  c++  java
  • Hbase常见异常

    1.启动hbase后,在hbase shell中执行 status和create时报错

    ERROR: org.apache.hadoop.hbase.PleaseHoldException: Master is initializing
    	at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3064)
    	at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2039)
    	at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:651)
    	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
    	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
    	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:132)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304)


    master端报:

         

    2019-09-02 09:43:21,534 WARN [master/s128:16000:becomeActiveMaster] master.HMaster: hbase:meta,,1.1588230740 is NOT online; state={1588230740 state=OPEN, ts=1567388594985, server=s130,16020,1567155264196}; ServerCrashProcedures=true. Master startup cannot progress, in holding-pattern until region onlined.
    2019-09-02 09:43:22,535 WARN [master/s128:16000:becomeActiveMaster] master.HMaster: hbase:meta,,1.1588230740 is NOT online; state={1588230740 state=OPEN, ts=1567388594985, server=s130,16020,1567155264196}; ServerCrashProcedures=true. Master startup cannot progress, in holding-pattern until region onlined.
    2019-09-02 09:43:24,536 WARN [master/s128:16000:becomeActiveMaster] master.HMaster: hbase:meta,,1.1588230740 is NOT online; state={1588230740 state=OPEN, ts=1567388594985, server=s130,16020,1567155264196}; ServerCrashProcedures=true. Master startup cannot progress, in holding-pattern until region onlined.
    2019-09-02 09:43:28,537 WARN [master/s128:16000:becomeActiveMaster] master.HMaster: hbase:meta,,1.1588230740 is NOT online; state={1588230740 state=OPEN, ts=1567388594985, server=s130,16020,1567155264196}; ServerCrashProcedures=true. Master startup cannot progress, in holding-pattern until region onlined.

      在zookeeper中删除/hbase即可

    rmr /hbase
    

    2.输入start-hbase.sh启动集群时,regionServer起不来,包ClockOutOfSyncException错:

    2019-09-04 14:56:50,992 ERROR [regionserver/s130:16020] regionserver.HRegionServer: Master rejected startup because clock is out of sync
    org.apache.hadoop.hbase.ClockOutOfSyncException: org.apache.hadoop.hbase.ClockOutOfSyncException: Server s130,16020,1567580190826 has been rejected; Reported time is too far out of sync with master.  Time difference of 41898747ms > max allowed of 120000ms
    	at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:342)
    	at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:199)
    	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:542)
    	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:13298)
    	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
    	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:132)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304)
    
    	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
    	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:99)
    	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:89)
    	at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:363)
    	at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:340)
    	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2660)
    	at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:982)
    	at java.lang.Thread.run(Thread.java:748)
    Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.ClockOutOfSyncException): org.apache.hadoop.hbase.ClockOutOfSyncException: Server s130,16020,1567580190826 has been rejected; Reported time is too far out of sync with master.  Time difference of 41898747ms > max allowed of 120000ms
    	at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:342)
    	at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:199)
    	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:542)
    	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:13298)
    	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
    	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:132)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304)
    
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:389)
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97)
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423)
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419)
    	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:103)
    	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:118)
    	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:162)
    	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:192)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337)
    	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323)
    	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337)
    	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337)
    	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930)
    	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:796)
    	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:427)
    	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328)
    	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905)
    	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    	... 1 more
    2019-09-04 14:56:51,025 ERROR [regionserver/s130:16020] regionserver.HRegionServer: region server s130,16020,1567580190826: Unhandled: org.apache.hadoop.hbase.ClockOutOfSyncException: Server s130,16020,1567580190826 has been rejected; Reported time is too far out of sync with master.  Time difference of 41898747ms > max allowed of 120000ms
    	at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:342)
    	at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:199)
    	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:542)
    	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:13298)
    	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
    	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:132)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304)
     *****
    org.apache.hadoop.hbase.ClockOutOfSyncException: org.apache.hadoop.hbase.ClockOutOfSyncException: Server s130,16020,1567580190826 has been rejected; Reported time is too far out of sync with master.  Time difference of 41898747ms > max allowed of 120000ms
    	at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:342)
    	at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:199)
    	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:542)
    	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:13298)
    	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
    	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:132)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304)
    
    	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
    	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:99)
    	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:89)
    	at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:363)
    	at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:340)
    	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2660)
    	at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:982)
    	at java.lang.Thread.run(Thread.java:748)
    Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.ClockOutOfSyncException): org.apache.hadoop.hbase.ClockOutOfSyncException: Server s130,16020,1567580190826 has been rejected; Reported time is too far out of sync with master.  Time difference of 41898747ms > max allowed of 120000ms
    	at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:342)
    	at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:199)
    	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:542)
    	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:13298)
    	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)
    	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:132)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324)
    	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304)
    
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:389)
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97)
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423)
    	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419)
    	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:103)
    	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:118)
    	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:162)
    	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:192)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337)
    	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323)
    	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337)
    	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337)
    	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359)
    	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
    	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930)
    	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:796)
    	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:427)
    	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328)
    	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905)
    	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    	... 1 more
    2019-09-04 14:56:51,026 ERROR [regionserver/s130:16020] regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: []
    

     解决方案:

        1.一般都是由于超时引起的,可以在hbase-site.xml中添加超时配置:

     <property>
        <name>hbase.master.maxclockskew</name>
        <value>120000</value>
     </property>
    

        2.也可能是由于master和slave时钟不一致引起的,如图:

     可见s128和其他机器的时钟明显不一样,其他机器时钟由于顺序执行命令,稍微误差,属正常

    可以将s128的时钟调整正确,启动hbase一切正常

  • 相关阅读:
    [转载]IT名企的薪资情况
    [转载]基于Oracle的高性能动态SQL程序开发(一)
    [转载]基于Oracle的高性能动态SQL程序开发(二)
    [转载]从Google身上可以学到的14个东西
    两篇励志故事
    《RedHat9系统管理》读书笔记 下载
    Linux各种版本下载
    Hibernate介绍, 语法 和 Hibernate的基本元素
    android中系统日期时间的获取
    关于Struts框架简介
  • 原文地址:https://www.cnblogs.com/qixing/p/11446110.html
Copyright © 2011-2022 走看看