Attempted to send a bulk request to Elasticsearch configured at '["http://192.168.32.152:9200"]', but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine
using the configuration provided? {:error_message=>"[503] {"error":{"root_cause":[{"type":"cluster_block_exception","reason":"blocked by: [SERVICE_UNAVAILABLE/2/no master];"}],"type
":"cluster_block_exception","reason":"blocked by: [SERVICE_UNAVAILABLE/2/no master];"},"status":503}", :error_class=>"Elasticsearch::Transport::Transport::Errors::ServiceUnavailable",
:backtrace=>["/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:201:in `__raise_transport_error'",
"/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:312:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/http/manticore.rb:67:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/client.rb:128:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-api-1.0.18/lib/elasticsearch/api/actions/bulk.rb:90:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:53:in `non_threadsafe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1
-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "org/jruby/ext/thread/Mutex.java:149:in `synchronize'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-
output-elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:172:in `safe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:101:in `submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:86:in `retrying_submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:29:in `multi_receive'", "org/jruby/RubyArray.java:1653:in `each_slice'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/common.rb:28:in `multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-
java/lib/logstash/output_delegator.rb:130:in `worker_multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:114:in
`multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "org/jruby/RubyHash.java:1342:in `each'",
"/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-
core-2.3.4-java/lib/logstash/pipeline.rb:232:in `worker_loop'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:201:in `start_workers'"],
:level=>:error}
[503] {"error":{"root_cause":[{"type":"cluster_block_exception","reason":"blocked by: [SERVICE_UNAVAILABLE/2/no master];"}],"type":"cluster_block_exception","reason":"blocked by:
[SERVICE_UNAVAILABLE/2/no master];"},"status":503} {:class=>"Elasticsearch::Transport::Transport::Errors::ServiceUnavailable", :backtrace=>["/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:201:in `__raise_transport_error'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:312:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/http/manticore.rb:67:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/client.rb:128:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-api-1.0.18/lib/elasticsearch/api/actions/bulk.rb:90:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:53:in `non_threadsafe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1
-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "org/jruby/ext/thread/Mutex.java:149:in `synchronize'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-
output-elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:172:in `safe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:101:in `submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:86:in `retrying_submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:29:in `multi_receive'", "org/jruby/RubyArray.java:1653:in `each_slice'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/common.rb:28:in `multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-
java/lib/logstash/output_delegator.rb:130:in `worker_multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:114:in
`multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "org/jruby/RubyHash.java:1342:in `each'",
"/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-
core-2.3.4-java/lib/logstash/pipeline.rb:232:in `worker_loop'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:201:in `start_workers'"],
:level=>:warn}
}
Attempted to send a bulk request to Elasticsearch configured at '["http://192.168.32.152:9200"]', but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine
using the configuration provided? {:error_message=>"[503] {"error":{"root_cause":[{"type":"cluster_block_exception","reason":"blocked by: [SERVICE_UNAVAILABLE/2/no master];"}],"type
":"cluster_block_exception","reason":"blocked by: [SERVICE_UNAVAILABLE/2/no master];"},"status":503}", :error_class=>"Elasticsearch::Transport::Transport::Errors::ServiceUnavailable",
:backtrace=>["/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:201:in `__raise_transport_error'",
"/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:312:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/http/manticore.rb:67:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/client.rb:128:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-api-1.0.18/lib/elasticsearch/api/actions/bulk.rb:90:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:53:in `non_threadsafe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1
-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "org/jruby/ext/thread/Mutex.java:149:in `synchronize'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-
output-elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:172:in `safe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:101:in `submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:86:in `retrying_submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:29:in `multi_receive'", "org/jruby/RubyArray.java:1653:in `each_slice'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/common.rb:28:in `multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-
java/lib/logstash/output_delegator.rb:130:in `worker_multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:129:in
`worker_multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:114:in `multi_receive'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "org/jruby/RubyHash.java:1342:in `each'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-
java/lib/logstash/pipeline.rb:232:in `worker_loop'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:201:in `start_workers'"],
:level=>:error}
[503] {"error":{"root_cause":[{"type":"cluster_block_exception","reason":"blocked by: [SERVICE_UNAVAILABLE/2/no master];"}],"type":"cluster_block_exception","reason":"blocked by:
[SERVICE_UNAVAILABLE/2/no master];"},"status":503} {:class=>"Elasticsearch::Transport::Transport::Errors::ServiceUnavailable", :backtrace=>["/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:201:in `__raise_transport_error'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/base.rb:312:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/transport/http/manticore.rb:67:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.18/lib/elasticsearch/transport/client.rb:128:in `perform_request'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/elasticsearch-api-1.0.18/lib/elasticsearch/api/actions/bulk.rb:90:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:53:in `non_threadsafe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1
-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "org/jruby/ext/thread/Mutex.java:149:in `synchronize'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-
output-elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:38:in `bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:172:in `safe_bulk'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:101:in `submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:86:in `retrying_submit'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.7.1-
java/lib/logstash/outputs/elasticsearch/common.rb:29:in `multi_receive'", "org/jruby/RubyArray.java:1653:in `each_slice'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-output-
elasticsearch-2.7.1-java/lib/logstash/outputs/elasticsearch/common.rb:28:in `multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-
java/lib/logstash/output_delegator.rb:130:in `worker_multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:129:in
`worker_multi_receive'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:114:in `multi_receive'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "org/jruby/RubyHash.java:1342:in `each'", "/usr/local/logstash-
2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:301:in `output_batch'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-
java/lib/logstash/pipeline.rb:232:in `worker_loop'", "/usr/local/logstash-2.3.4/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:201:in `start_workers'"],
:level=>:warn}
{
"message" => " 114.215.172.206 [14/Sep/2016:10:06:23 +0800] "GET /elk/ HTTP/1.1" - 200 15 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.21 Basic ECC zlib/1.2.3
libidn/1.18 libssh2/1.4.2" 0.000 -",
"@version" => "1",
"@timestamp" => "2016-09-14T02:08:52.784Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-09-14",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"tags" => [
[0] "_grokparsefailure"
]
}
{
"message" => " 114.215.172.206 [14/Sep/2016:10:06:25 +0800] "GET /elk/ HTTP/1.1" - 200 15 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.21 Basic ECC zlib/1.2.3
libidn/1.18 libssh2/1.4.2" 0.000 -",
"@version" => "1",
"@timestamp" => "2016-09-14T02:08:54.790Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-09-14",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"tags" => [
[0] "_grokparsefailure"
]
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
{
"message" => " 114.215.172.206 [14/Sep/2016:10:06:27 +0800] "GET /elk/ HTTP/1.1" - 200 15 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.21 Basic ECC zlib/1.2.3
libidn/1.18 libssh2/1.4.2" 0.000 -",
"@version" => "1",
"@timestamp" => "2016-09-14T02:08:56.793Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-09-14",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"tags" => [
haproxy 配置:
frontend www
bind *:9200
default_backend eshttp_server
backend eshttp_server
mode http
balance roundrobin
server ela01 192.168.32.80:9200 check inter 2000 fall 3
server ela02 192.168.32.81:9200 check inter 2000 fall 3
server ela03 192.168.32.82:9200 check inter 2000 fall 3
/*************
option redispatch：此参数用于cookie保持的环境中，在默认请况下，HAproxy会将其请求的后端服务器的serverID插入到cookie中，以保持会话的seesion持久性，而如果后端的服务器出现故障，客户端的cookie是不会刷新
的，这就会出现问题，此时如果设置此参数，就会将客户的请求强制定向到别外一台健康的后端服务器上，以保证服务正常。
option redispatch #当serverId对应的服务器挂掉后，强制定向到其他健康的服务器