java调用Elasticserach的API实现日志查询

前言：

    前面讲了ELK+Filebeat搭建，点击查看，那么如何在java程序中通过API查询日志呢？我们知道，ELK的流程是FileBeat扫描日志文件，将日志发送到Logstash，Logstash将日志进行筛选和分组，发送给Elasticsearch，最终Kibnan通过调用Elasticsearch的接口，查询日志，并展示出来。在这里，我的java程序就相当于替代了Kibana，由java调用es的接口，实现索引创建、索引查询、索引删除、日志数量查询、日志列表查询、日志分页查询等。

1，java项目引入Elasticsearch依赖

    这里使用的是maven，版本和es全家桶版本一致

<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>7.14.0</version>
</dependency>
<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.14.0</version>
</dependency>

2，yml文件中加入es依赖

elasticsearch:
  host: 192.168.2.129
  port: 9200
  connTimeout: 3000
  socketTimeout: 5000
  connectionRequestTimeout: 500
  maxConnectNum: 100
  maxConnectPerRoute: 100

3，es的configuration文件

package com.test.elk;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 *
 * @author ：wdl
 * ==============================================================================
 * @文件名称 ：ElasticsearchConfiguration.java
 * ==============================================================================
 * @本类描述 : ElsaticSearch配置类
 * @功能概述 ：
 * @创建日期 ：2021/10/12
 */
@Configuration
public class ElasticsearchConfiguration {
    @Value("${elasticsearch.host}")
    private String host;

    @Value("${elasticsearch.port}")
    private int port;

    @Value("${elasticsearch.connTimeout}")
    public int connectTimeOut;

    @Value("${elasticsearch.socketTimeout}")
    private int socketTimeOut;

    @Value("${elasticsearch.connectionRequestTimeout}")
    private int connectionRequestTimeOut;

    @Value("${elasticsearch.maxConnectNum}")
    private int maxConnectNum;

    @Value("${elasticsearch.maxConnectPerRoute}")
    private int maxConnectPerRoute;

    @Bean(destroyMethod = "close", name = "client")
    public RestHighLevelClient initRestClient() {
        // 异步httpclient连接延时配置
        RestClientBuilder builder = RestClient.builder(new HttpHost(host, port));
        builder.setRequestConfigCallback(builder1 -> {
                    builder1.setConnectTimeout(connectTimeOut);
                    builder1.setSocketTimeout(socketTimeOut);
                    builder1.setConnectionRequestTimeout(connectionRequestTimeOut);
                    return builder1;
                });
        // 异步httpclient连接数配置
        builder.setHttpClientConfigCallback(httpClientBuilder -> {
            httpClientBuilder.setMaxConnTotal(maxConnectNum);
            httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute);
            return httpClientBuilder;
        });
        return new RestHighLevelClient(builder);
    }
}

4，controller类

package com.test.elk;

import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;

import javax.annotation.Resource;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * @author ：wdl
 * ==============================================================================
 * @文件名称 ：ElasticSearchController.java
 * ==============================================================================
 * @本类描述 : 操作es控制类
 * @功能概述 ：
 * @创建日期 ：2021/10/12
 */
@Controller
@RequestMapping(value = "system/test")
public class ElasticSearchController {
    @Resource
    private RestHighLevelClient client;

    @RequestMapping(value = "existsIndex", method = RequestMethod.POST)
    @ResponseBody
    public boolean existsIndex(String index) throws IOException {
        // 声明GetIndexRequest时要把索引信息带进去，可以带入字符串也可以带入String[],如果传入多个索引，则全部存在才会返回true,只要有一个不存在,就会返回false
        String[] indices = {index};
        GetIndexRequest request = new GetIndexRequest(indices);
        boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
        System.out.println("existsIndex: " + exists);
        return exists;
    }
    @RequestMapping(value = "createIndex", method = RequestMethod.POST)
    @ResponseBody
    public boolean createIndex (String index){
        try {
            if(!existsIndex(index)){
                CreateIndexRequest request = new CreateIndexRequest(index);
                this.client.indices().create(request, RequestOptions.DEFAULT);
                return Boolean.TRUE ;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return Boolean.FALSE;
    }
    @RequestMapping(value = "deleteIndex", method = RequestMethod.POST)
    @ResponseBody
    public boolean deleteIndex(String index) {
        try {
            if(existsIndex(index)){
                DeleteIndexRequest request = new DeleteIndexRequest(index);
                AcknowledgedResponse response = client.indices().delete(request, RequestOptions.DEFAULT);
                return response.isAcknowledged();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return Boolean.FALSE;
    }

    /**
     * 查询总数
     */
    @RequestMapping(value = "count", method = RequestMethod.POST)
    @ResponseBody
    public Long count(String index){
        // 这里的条件和分页查询的条件一直，可以在分页查询中调用count方法，查询出日志总数
        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
        // 匹配message字段中包含hello关键字的
        queryBuilder.must(QueryBuilders.matchPhraseQuery("message", "hello"));
        // 时间范围筛选
        queryBuilder.filter(QueryBuilders.rangeQuery("@timestamp").from(1634086800000L).to(1634087400000L));
        CountRequest countRequest = new CountRequest(index);
        countRequest.query(queryBuilder);
        try {
            CountResponse countResponse = client.count(countRequest, RequestOptions.DEFAULT);
            return countResponse.getCount();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return 0L;
    }

    /**
     * 查询集合
     */
    @RequestMapping(value = "list", method = RequestMethod.POST)
    @ResponseBody
    public List<Map<String,Object>> list (String index) {
        // RangeQueryBuilder 可以给字段指定时间范围，但是不能加关键字查询，如果需要多条件组合查询，请看page方法
        RangeQueryBuilder rangequerybuilder = QueryBuilders
                .rangeQuery("@timestamp")
                .from(1634086800000L).to(1634087400000L);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(rangequerybuilder);
        SearchRequest searchRequest = new SearchRequest(index);
        searchRequest.source(sourceBuilder);
        try {
            SearchResponse searchResp = client.search(searchRequest, RequestOptions.DEFAULT);
            List<Map<String,Object>> data = new ArrayList<>() ;
            SearchHit[] searchHitArr = searchResp.getHits().getHits();
            for (SearchHit searchHit:searchHitArr){
                Map<String,Object> res = new HashMap<>();
                Map<String,Object> temp = searchHit.getSourceAsMap();
                res.put("id",searchHit.getId());
                res.put("time",temp.get("@timestamp"));
                res.put("message",temp.get("message"));
                data.add(res);
            }
            return data;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    /**
     * 分页查询
     */
    @RequestMapping(value = "page", method = RequestMethod.POST)
    @ResponseBody
    public List<Map<String,Object>> page (String index) {
        // SearchSourceBuilder可以指定复杂条件，from：分页查询起始偏移量；size：每页条数；sort：排序；query：查询条件；可以声明QueryBuilder的一个实现类，将条件加入queryBuilder，再放入sourceBuilder；
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.from(0);
        sourceBuilder.size(10);
        sourceBuilder.sort("@timestamp", SortOrder.DESC);
        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
        queryBuilder.must(QueryBuilders.matchPhraseQuery("message", "hello"));
        queryBuilder.filter(QueryBuilders.rangeQuery("@timestamp").from(1634086800000L).to(1634087400000L));
        sourceBuilder.query(queryBuilder);
        SearchRequest searchRequest = new SearchRequest(index);
        // 将sourceBulider放入searchRequest
        searchRequest.source(sourceBuilder);
        try {
            SearchResponse searchResp = client.search(searchRequest, RequestOptions.DEFAULT);
            List<Map<String,Object>> data = new ArrayList<>() ;
            SearchHit[] searchHitArr = searchResp.getHits().getHits();
            for (SearchHit searchHit:searchHitArr){
                Map<String,Object> res = new HashMap<>();
                Map<String,Object> temp = searchHit.getSourceAsMap();
                res.put("time",transTime(temp.get("@timestamp")));
                res.put("message",temp.get("message"));
                data.add(res);
            }
            return data;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null ;
    }
    //将Z时区时间转换成UTC时间
    private Date transTime(Object time){
        Date date = new Date();
        String dateStr = String.valueOf(time);
        dateStr = dateStr.replace("Z", " UTC");
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
        try {
            date = format.parse(dateStr);
        } catch (ParseException e) {
            e.printStackTrace();
        }
        return date;
    }
}