ELK 安装笔记

logstash

-noverify
-javaagent:E:svnother njrebel6.0.0+crackjrebel6.0.0-crackjrebel.jar
-Drebel.dirs=E:svnlkStoreServer runkwebappWEB-INFclasses
-Drebel.spring_plugin=true
-Drebel.struts2-plugin=true

termvectors
termvectors - 必应词典
网络词向量；检索词向量

percolate - 必应词典
美['pɜrkə.leɪt]英['pɜː(r)kəleɪt]
v.渗入；渗透；渗漏；逐渐流传
n.渗出液
网络过滤；过滤出；渗滤液

https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz

curl -XPUT "http://192.168.4.140:9200/myindex/emp/3" -d '{"name":"lk", "age":23}'

[root@guosen ~]# curl -XGET "192.168.4.140:9200/logstash-2016.09.20/_search?pretty&q=response=200&geoip.city_name=Buffalo"
{
"error" : {
"root_cause" : [ {
"type" : "cluster_block_exception",
"reason" : "blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];"
} ],
"type" : "cluster_block_exception",
"reason" : "blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];"
},
"status" : 503
}

disconnected

Caused by: java.lang.IllegalArgumentException: No custom metadata prototype registered for type [licenses], node like missing plugin
s
at org.elasticsearch.cluster.metadata.MetaData.lookupPrototypeSafe(MetaData.java:119)
at org.elasticsearch.cluster.metadata.MetaData.readFrom(MetaData.java:691)
at org.elasticsearch.cluster.metadata.MetaData$Builder.readFrom(MetaData.java:1212)
at org.elasticsearch.gateway.TransportNodesListGatewayMetaState$NodeGatewayMetaState.readFrom(TransportNodesListGatewayMetaS
tate.java:220)
at org.elasticsearch.transport.netty.MessageChannelHandler.handleResponse(MessageChannelHandler.java:150)
... 23 more
[2016-09-20 13:49:55,364][WARN ][transport.netty ] [node-2] exception caught on transport layer [[id: 0x201a20e1, /192.168.
4.140:52209 => /192.168.4.49:9300]], closing connection
java.lang.IllegalStateException: Message not fully read (response) for requestId [1188], handler [org.elasticsearch.action.support.n
odes.TransportNodesAction$AsyncAction$2@33184d8e], error [false]; resetting
at org.elasticsearch.transport.netty.MessageChannelHandler$Marker.validateResponse(MessageChannelHandler.java:403)
at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:126)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:310)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337)
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)

    at java.lang.Thread.run(Thread.java:745)

[2016-09-20 13:49:55,372][INFO ][gateway ] [node-2] metadata state not restored, reason: found [1] metadata states,
required [2]
[2016-09-20 13:49:55,373][WARN ][discovery.zen ] [node-2] not enough master nodes, current nodes: {{node-2}{oTsjC4pyS72RK
-FbeqZMOw}{192.168.4.140}{192.168.4.140:9300}{rack=r2},}
[2016-09-20 13:49:55,373][INFO ][cluster.service ] [node-2] removed {{node-1}{lcnX80ztQ3WLGJgh5SW2Zw}{192.168.4.49}{192.168
.4.49:9300}{rack=r1},}, reason: zen-disco-node-failed({node-1}{lcnX80ztQ3WLGJgh5SW2Zw}{192.168.4.49}{192.168.4.49:9300}{rack=r1}), reason(transport disconnected)
[2016-09-20 13:49:59,302][INFO ][cluster.service ] [node-2] detected_master {node-1}{lcnX80ztQ3WLGJgh5SW2Zw}{192.168.4.49}{192.168.4.49:9300}{rack=r1}, added {{node-1}{lcnX80ztQ3WLGJgh5SW2Zw}{192.168.4.49}{192.168.4.49:9300}{rack=r1},}, reason: zen-disco-receive(from master [{node-1}{lcnX80ztQ3WLGJgh5SW2Zw}{192.168.4.49}{192.168.4.49:9300}{rack=r1}])
[2016-09-20 13:50:25,262][INFO ][cluster.routing.allocation.decider] [node-2] low disk watermark [85%] exceeded on [oTsjC4pyS72RK-FbeqZMOw][node-2][/var/lib/elasticsearch/my-application/nodes/0] free: 3.6gb[11.7%], replicas will not be assigned to this node
[2016-09-20 13:54:27,695][WARN ][rest.suppressed ] path: /_search, params: {pretty=}

    at java.lang.Thread.run(Thread.java:745)

[2016-09-20 13:57:15,116][WARN ][transport ] [node-2] Transport response handler not found of id [1653]
[2016-09-20 13:57:19,241][WARN ][transport ] [node-2] Transport response handler not found of id [1658]

https://discuss.elastic.co/t/shield-problems-transport-response-handler-not-found/60854/2 ---- 好像是没有 license 。。。。

Download and install the public signing key:

rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

Add the following in your /etc/yum.repos.d/ directory in a file with a .repo suffix, for example elasticsearch.repo

[elasticsearch-2.x]
name=Elasticsearch repository for 2.x packages
baseurl=https://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1

And your repository is ready for use. You can install it with:

yum install elasticsearch

elasticsearch
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"

++++

vi /etc/init.d/elasticsearch

00000

+++++ 插件安装 http://blog.csdn.net/whxaing2011/article/details/18237733 ++++++++++++++++++++

head ：

cd /usr/share/elasticsearch/

./bin/plugin install mobz/elasticsearch-head

http://192.168.4.49:9200/_plugin/bigdesk/

+++++

[root@guosen es]# service elasticsearch restart
停止 elasticsearch：[失败]
正在启动 elasticsearch：Exception in thread "main" java.lang.IllegalStateException: Failed to created node environment
Likely root cause: java.nio.file.AccessDeniedException: /var/lib/elasticsearch/elasticsearch/nodes/1
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:84)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:383)
at java.nio.file.Files.createDirectory(Files.java:630)
at java.nio.file.Files.createAndCheckIsDirectory(Files.java:734)
at java.nio.file.Files.createDirectories(Files.java:720)
at org.elasticsearch.env.NodeEnvironment.(NodeEnvironment.java:169)
at org.elasticsearch.node.Node.(Node.java:165)
at org.elasticsearch.node.Node.(Node.java:140)
at org.elasticsearch.node.NodeBuilder.build(NodeBuilder.java:143)
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:194)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:286)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:35)
Refer to the log for complete error details.
[失败]

http://stackoverflow.com/questions/34564625/how-to-solve-accessdeniedexception-in-elasticsearch

chown -R elasticsearch:elasticsearch /var/lib/elasticsearch/

正在启动 elasticsearch：[确定]
[root@guosen es]# Exception in thread "main" BindTransportException[Failed to bind to [9300-9400]]; nested: ChannelException[Failed to bind to: /192.168.1.140:9400]; nested: BindException[无法指定被请求的地址];
Likely root cause: java.net.BindException: 无法指定被请求的地址
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:444)
at sun.nio.ch.Net.bind(Net.java:436)
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:214)
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
at org.jboss.netty.channel.socket.nio.NioServerBoss$RegisterTask.run(NioServerBoss.java:193)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315)
at org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Refer to the log for complete error details.

network.host: 192.168.1.140 写错了！！！ 应该是 network.host: 192.168.4.140

++++++++++++ repo 作怪。。

[root@guosen es]# yum localinstall elasticsearch-2.4.0.rpm
已加载插件：product-id, refresh-packagekit, security, subscription-manager
This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register.
设置本地安装进程
诊断 elasticsearch-2.4.0.rpm: elasticsearch-2.4.0-1.noarch
elasticsearch-2.4.0.rpm 将被安装
elasticsearch-2.x | 2.9 kB 00:00
elasticsearch-2.x/primary_db | 7.3 kB 00:00
zabbix | 951 B 00:00
zabbix/primary | 9.6 kB 00:00
段错误 (core dumped)
[root@guosen es]#
[root@guosen es]#
[root@guosen es]# cd /etc/yum.repos.d
[root@guosen yum.repos.d]#
[root@guosen yum.repos.d]#
[root@guosen yum.repos.d]# mv zabbix.repo zabbix.repo.bak
[root@guosen yum.repos.d]#
[root@guosen yum.repos.d]#
[root@guosen yum.repos.d]# yum clean all
已加载插件：product-id, refresh-packagekit, security, subscription-manager

This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register.
Cleaning repos: elasticsearch-2.x
清理一切
[root@guosen yum.repos.d]#
[root@guosen yum.repos.d]# cd /es
[root@guosen es]#
[root@guosen es]# yum localinstall elasticsearch-2.4.0.rpm
已加载插件：product-id, refresh-packagekit, security, subscription-manager
This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register.
设置本地安装进程
诊断 elasticsearch-2.4.0.rpm: elasticsearch-2.4.0-1.noarch
elasticsearch-2.4.0.rpm 将被安装
elasticsearch-2.x | 2.9 kB 00:00
elasticsearch-2.x/primary_db | 7.3 kB 00:00
解决依赖关系
--> 执行事务检查
---> Package elasticsearch.noarch 0:2.4.0-1 will be 安装
--> 完成依赖关系计算

+++++++++++++++

+++++++++++ 集群： 下面的必现开启。 若要取消集群， 下面的必须注释

discovery.zen.minimum_master_nodes: 2

++++++++++++

curl -XGET 'http://192.168.4.49:9200/_license?pretty'

curl -XGET "192.168.4.49:9200/_cluster/health?pretty"

[root@guosen es]# yum localinstall jdk-7u67-linux-x64.rpm
已加载插件：product-id, refresh-packagekit, security, subscription-manager
This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register.
设置本地安装进程
诊断 jdk-7u67-linux-x64.rpm: 2000:jdk-1.7.0_67-fcs.x86_64
jdk-7u67-linux-x64.rpm 将作为 2000:jdk-1.6.0_33-fcs.x86_64 的更新
段错误 (core dumped)
[root@guosen es]#
[root@guosen es]# rpm -ivh jdk-7u67-linux-x64.rpm
Preparing... ########################################### [100%]
file /etc/init.d/jexec from install of jdk-2000:1.7.0_67-fcs.x86_64 conflicts with file from package jdk-2000:1.6.0_33-fcs.x86_64

  有台机器开发说要用jdk1.7，因为他们开发环境、测试环境全部都是1.7的。郁闷的是我把服务器都装上1.6版本了，那就改吧。

   1.下载1.7新版本的jdk：http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html  
   2.把下载的rpm文件放到/usr/local下去了；
   3.然后开始安装，可是：
  [root@server-bj001 local]# rpm -ivh jdk-7u67-linux-x64.rpm 

Preparing... ########################################### [100%]
file /etc/init.d/jexec from install of jdk-2000:1.7.0_67-fcs.x86_64 conflicts with file from package jdk-2000:1.6.0_29-fcs.x86_64

   报错了，把系统默认的openjdk卸掉试试。

    4. 卸载系统默认jdk：

[root@server-bj001 local]# rpm -qa |grep java
sun-javadb-demo-10.6.2-1.1.i386
java-1.6.0-openjdk-devel-1.6.0.0-1.50.1.11.5.el6_3.x86_64
sun-javadb-common-10.6.2-1.1.i386
sun-javadb-core-10.6.2-1.1.i386
sun-javadb-javadoc-10.6.2-1.1.i386
tzdata-java-2012j-1.el6.noarch
java_cup-0.10k-5.el6.x86_64
sun-javadb-client-10.6.2-1.1.i386
gcc-java-4.4.7-4.el6.x86_64
java-1.5.0-gcj-1.5.0.0-29.1.el6.x86_64
sun-javadb-docs-10.6.2-1.1.i386
[root@server-bj001 local]# rpm -e --nodeps sun-javadb-demo-10.6.2-1.1.i386
[root@server-bj001 local]# rpm -e --nodeps java-1.6.0-openjdk-devel-1.6.0.0-1.50.1.11.5.el6_3.x86_64
[root@server-bj001 local]# rpm -e --nodeps sun-javadb-common-10.6.2-1.1.i386
[root@server-bj001 local]# rpm -e --nodeps sun-javadb-core-10.6.2-1.1.i386
。。。。。。

5.再装还是那个错，忘了把自己安装的1.6卸掉了，继续来。
[root@server-bj001 local]# rpm -qa |grep jdk
jdk-1.6.0_29-fcs.x86_64
[root@server-bj001 local]# rpm -e --nodeps jdk-1.6.0_29-fcs.x86_64
[root@server-bj001 local]# rpm -ivh jdk-7u67-linux-x64.rpm
Preparing... ########################################### [100%]
1:jdk ########################################### [100%]
Unpacking JAR files...
rt.jar...
jsse.jar...
charsets.jar...
tools.jar...
localedata.jar...
jfxrt.jar...
这回可以安装了。

6.修改配置文件/etc/profile，最后添加下列：
export JAVA_HOME=/usr/lib/jdk/jdk1.7.0_67
export JRE_HOME=/usr/lib/jdk/jdk1.7.0_67/jre
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
export CLASSPATH=$CLASSPATH:.:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
PATH=$PATH:/var/lib/mysql/bin
export PATH

7. source /etc/profile 执行这个让配置文件生效

8. 验证下吧：
   [root@server-bj001 jdk1.7.0_67]# java -version
   java version "1.7.0_67"
   Java(TM) SE Runtime Environment (build 1.7.0_67-b01)
   Java HotSpot(TM) 64-Bit Server VM (build 24.65-b04, mixed mode)

终于搞定了，哎。饶了一个大圈，不过想想估计把自己安装的JDK1.6卸载掉应该就可以安装1.7的了，以后试试先。