er[file_system_exception: /elk/elasticsearch/data/es_cluster/nodes/0/indices/logstash-zjzc-frontend-access-2016.08.13/2/index/_d_Lucene54_0.dvm: Too many open files];
Caused by: [logstash-zjzc-frontend-access-2016.08.13][[logstash-zjzc-frontend-access-2016.08.13][2]] EngineCreationFailureException[failed to open reader on writer]; nested: NotSerializableExceptionWrapper[file_system_exception: /elk/elasticsearch/data/es_cluster/nodes/0/indices/logstash-zjzc-frontend-access-2016.08.13/2/index/_d_Lucene54_0.dvm: Too many open files];
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:295)
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:166)
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1509)
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1493)
at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:966)
at org.elasticsearch.index.shard.IndexShard.skipTranslogRecovery(IndexShard.java:977)
at org.elasticsearch.indices.recovery.RecoveryTarget$PrepareForTranslogOperationsRequestHandler.messageReceived(RecoveryTarget.java:275)
at org.elasticsearch.indices.recovery.RecoveryTarget$PrepareForTranslogOperationsRequestHandler.messageReceived(RecoveryTarget.java:268)
at org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:75)
at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:300)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: NotSerializableExceptionWrapper[file_system_exception: /elk/elasticsearch/data/es_cluster/nodes/0/indices/logstash-zjzc-frontend-access-2016.08.13/2/index/_d_Lucene54_0.dvm: Too many open files]
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
at java.nio.channels.FileChannel.open(FileChannel.java:287)
at java.nio.channels.FileChannel.open(FileChannel.java:335)
at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
at org.apache.lucene.store.FileSwitchDirectory.openInput(FileSwitchDirectory.java:186)
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:89)
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:89)
at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:109)
at org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer.<init>(Lucene54DocValuesProducer.java:114)
at org.apache.lucene.codecs.lucene54.Lucene54DocValuesFormat.fieldsProducer(Lucene54DocValuesFormat.java:113)
at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsReader.<init>(PerFieldDocValuesFormat.java:268)
at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat.fieldsProducer(PerFieldDocValuesFormat.java:358)
at org.apache.lucene.index.SegmentDocValues.newDocValuesProducer(SegmentDocValues.java:51)
at org.apache.lucene.index.SegmentDocValues.getDocValuesProducer(SegmentDocValues.java:67)
at org.apache.lucene.index.SegmentReader.initDocValuesProducer(SegmentReader.java:147)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:81)
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:99)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:435)
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:100)
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:283)
修改ulimit:
* soft nofile 65535
* hard nofile 65535
* soft nproc 65535
* hard nproc 65535