发现java的日志里面老是报错如下日志，还有kafka的消费者收不到消息，看来docker logs kafka日志如下：
原因估计是12月23日那天磁盘空间满了，导致redis和kafka异常
解决方法：
重启了redis的这个节点172.18.144.94:7006
重启了kafka就ok了
如下是redis cluster的报错
2020-01-16 17:10:00.098 pool-8-thread-3 ERROR com.hp.nova.common.todayuseronline.UserOnlineScheduleJob.userOnlineScheduleJob(UserOnlineScheduleJob.java:73) - java.lang.NumberFormatException: For input string: "redis.clients.jedis.exceptions.JedisMovedDataException: MOVED 9139 172.18.144.94:7006"
at sun.misc.FloatingDecimal.readJavaFormatString(FloatingDecimal.java:2043) ~[?:1.8.0_191]
at sun.misc.FloatingDecimal.parseDouble(FloatingDecimal.java:110) ~[?:1.8.0_191]
at java.lang.Double.parseDouble(Double.java:538) ~[?:1.8.0_191]
at com.hp.nova.common.todayuseronline.UserOnlineScheduleJob.userOnlineScheduleJob(UserOnlineScheduleJob.java:66) [classes!/:0.0.1-SNAPSHOT]
at sun.reflect.GeneratedMethodAccessor187.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191]
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:65) [spring-context-4.3.20.RELEASE.jar!/:4.3.20.RELEASE]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) [spring-context-4.3.20.RELEASE.jar!/:4.3.20.RELEASE]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-4.3.20.RELEASE.jar!/:4.3.20.RELEASE]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_191]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_191]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_191]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
2020-01-16 17:10:00.100 pool-8-thread-3 ERROR com.hp.nova.common.todayuseronline.UserOnlineScheduleJob.userOnlineScheduleJob(UserOnlineScheduleJob.java:73) - java.lang.NumberFormatException: For input string: "redis.clients.jedis.exceptions.JedisMovedDataException: MOVED 5856 172.18.144.94:7006"
at sun.misc.FloatingDecimal.readJavaFormatString(FloatingDecimal.java:2043) ~[?:1.8.0_191]
at sun.misc.FloatingDecimal.parseDouble(FloatingDecimal.java:110) ~[?:1.8.0_191]
at java.lang.Double.parseDouble(Double.java:538) ~[?:1.8.0_191]
at com.hp.nova.common.todayuseronline.UserOnlineScheduleJob.userOnlineScheduleJob(UserOnlineScheduleJob.java:66) [classes!/:0.0.1-SNAPSHOT]
at sun.reflect.GeneratedMethodAccessor187.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191]
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:65) [spring-context-4.3.20.RELEASE.jar!/:4.3.20.RELEASE]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) [spring-context-4.3.20.RELEASE.jar!/:4.3.20.RELEASE]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-4.3.20.RELEASE.jar!/:4.3.20.RELEASE]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_191]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_191]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_191]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
如下是kafka的报错
[2019-12-23 15:01:53,041] ERROR Error while writing to checkpoint file /kafka/kafka-logs-6c380161c0a2/__consumer_offsets-20/leader-epoch-checkpoint (kafka.server.LogDirFailureChannel)
java.io.FileNotFoundException: /kafka/kafka-logs-6c380161c0a2/__consumer_offsets-20/leader-epoch-checkpoint.tmp (No space left on device)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:162)
at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:52)
at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50)
at kafka.server.checkpoints.LeaderEpochCheckpointFile.write(LeaderEpochCheckpointFile.scala:64)
at kafka.server.epoch.LeaderEpochFileCache.flush(LeaderEpochFileCache.scala:219)
at kafka.server.epoch.LeaderEpochFileCache.$anonfun$assign$1(LeaderEpochFileCache.scala:62)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251)
at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259)
at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:52)
at kafka.cluster.Partition.$anonfun$makeLeader$8(Partition.scala:395)
at kafka.cluster.Partition.$anonfun$makeLeader$8$adapted(Partition.scala:394)
at scala.Option.foreach(Option.scala:257)
at kafka.cluster.Partition.$anonfun$makeLeader$1(Partition.scala:394)
at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251)
at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259)
at kafka.cluster.Partition.makeLeader(Partition.scala:367)
at kafka.server.ReplicaManager.$anonfun$makeLeaders$5(ReplicaManager.scala:1162)
at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:145)
at scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:235)
at scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:228)
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
at scala.collection.mutable.HashMap.foreach(HashMap.scala:145)
at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1160)
at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1072)
at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185)
at kafka.server.KafkaApis.handle(KafkaApis.scala:110)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69)
at java.lang.Thread.run(Thread.java:748)
log4j:ERROR Failed to flush writer,
java.io.IOException: No space left on device
at java.io.FileOutputStream.writeBytes(Native Method)
at java.io.FileOutputStream.write(FileOutputStream.java:326)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
at org.apache.log4j.DailyRollingFileAppender.subAppend(DailyRollingFileAppender.java:369)
at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
at org.apache.log4j.Category.callAppenders(Category.java:206)
at org.apache.log4j.Category.forcedLog(Category.java:391)
at org.apache.log4j.Category.log(Category.java:856)
at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
advertised.listeners = PLAINTEXT://172.18.144.94:9092
advertised.port = null
alter.config.policy.class.name = null
alter.log.dirs.replication.quota.window.num = 11
alter.log.dirs.replication.quota.window.size.seconds = 1