使用java操作elasticsearch(1)

1.安装elasticsearch

　　这儿用的是5.6.9的版本，下载安装过程较为简单,在官网上下载好后解压到文件夹。需要注意的是在elasticsearch-5.6.9config下的elasticsearch.yml文件夹中

　　配置data与log的地址，自己创建文件夹后如下配置

　　

# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: D:/java/soft/elasticsearch/dev/data
#
# Path to log files:
#
path.logs: D:/java/soft/elasticsearch/dev/log
#

　启动elasticsearch   点击elasticksearch.bat即可启动  注意电脑上要安装1.8以上的jdk

出现以下消息则说明启动成功

上面红圈的9300端口则是与es进行数据交互时的端口，下面的9200则是查看数据的端口，ip为安装es的ip

启动项目后，访问http://ip:9200   ip是安装elasticsearch服务的机器ip，如果出现以下信息即代表安装成功

　　

{
    "name": "kSs9lbx",
    "cluster_name": "elasticsearch",
    "cluster_uuid": "Wu6n-WzvQuecDEMWspm_aA",
    "version": {
        "number": "5.6.9",
        "build_hash": "877a590",
        "build_date": "2018-04-12T16:25:14.838Z",
        "build_snapshot": false,
        "lucene_version": "6.6.1"
    },
    "tagline": "You Know, for Search"
}

2.创建java的springboot工程

　　2.1pom.xml文件依赖

　　

<parent>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-parent</artifactId>
    <version>1.5.2.RELEASE</version>
    <relativePath/> <!-- lookup parent from repository -->
</parent>

<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <java.version>1.8</java.version>
    <fastjson.version>1.2.31</fastjson.version>
    <jackson.version>2.8.7</jackson.version>
</properties>

<dependencies>
    <!-- set jetty server -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
        <exclusions>
            <exclusion>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-tomcat</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-configuration-processor</artifactId>
        <optional>true</optional>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-jetty</artifactId>
    </dependency>
    <!-- aop -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-aop</artifactId>
    </dependency>
    <!-- json处理 -->
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>${fastjson.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/net.sourceforge.javacsv/javacsv -->
    <dependency>
        <groupId>net.sourceforge.javacsv</groupId>
        <artifactId>javacsv</artifactId>
        <version>2.0</version>
    </dependency>
    <!-- jackson json begin -->
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-core</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-annotations</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.module</groupId>
        <artifactId>jackson-module-jaxb-annotations</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <!-- jackson json end -->
    <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
    <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-lang3</artifactId>
        <version>3.3.2</version>
    </dependency>
    <!-- druid datasource -->
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>druid</artifactId>
        <version>1.0.27</version>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-validation</artifactId>
    </dependency>
    <dependency>
        <groupId>commons-io</groupId>
        <artifactId>commons-io</artifactId>
        <version>2.4</version>
    </dependency>
    <dependency>
        <groupId>com.github.theborakompanioni</groupId>
        <artifactId>thymeleaf-extras-shiro</artifactId>
        <version>${thymeleaf-shiro.version}</version>
    </dependency>
    <!-- mysql驱动 -->
    <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <scope>runtime</scope>
    </dependency>
    <!-- spring boot 开发工具 -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-devtools</artifactId>
        <optional>true</optional>
    </dependency>
    <!-- 测试 -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-test</artifactId>
        <scope>test</scope>
    </dependency>
    <!--elasticsearrch-->
    <dependency>
        <groupId>org.elasticsearch</groupId>
        <artifactId>elasticsearch</artifactId>
        <version>5.6.9</version>
    </dependency>

    <dependency>
        <groupId>org.elasticsearch.client</groupId>
        <artifactId>transport</artifactId>
        <version>5.6.9</version>
    </dependency>

    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-data-jpa</artifactId>
    </dependency>

</dependencies>

<repositories>
    <repository>
        <id>aliyun</id>
        <name>aliyun</name>
        <url>http://maven.aliyun.com/nexus/content/groups/public</url>
    </repository>
</repositories>

<build>
    <plugins>
        <plugin>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-maven-plugin</artifactId>
            <configuration>
                <includeSystemScope>true</includeSystemScope>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <version>2.18.1</version>
            <configuration>
                <skipTests>true</skipTests>
            </configuration>
        </plugin>
    </plugins>
</build>

　　2.2application.yml文件配置

　　

server:
  port: 9999
es:
  host: 127.0.0.1
  port: 9300
spring:
  datasource:
    initialize: false #false时不进行数据库表结构和数据初始化
    username: root
    #password: root
    password: root
    url: jdbc:mysql://localhost:3306/estest?createDatabaseIfNotExist=true&autoReconnect=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true
    type: com.alibaba.druid.pool.DruidDataSource
    #连接池配置
    driverClassName: com.mysql.jdbc.Driver
    # 初始化大小，最小，最大
    initialSize: 5
    minIdle: 5
    maxActive: 50
    # 配置获取连接等待超时的时间
    maxWait: 60000
    # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
    timeBetweenEvictionRunsMillis: 60000
    # 配置一个连接在池中最小生存的时间，单位是毫秒
    minEvictableIdleTimeMillis: 300000
    validationQuery: SELECT 1 FROM DUAL
    testWhileIdle: true
    testOnBorrow: false
    testOnReturn: false
    # 打开PSCache，并且指定每个连接上PSCache的大小
    poolPreparedStatements: true
    maxPoolPreparedStatementPerConnectionSize: 20
    # 配置监控统计拦截的filters，去掉后监控界面sql无法统计，'wall'用于防火墙
    filters: stat,log4j
    # 通过connectProperties属性来打开mergeSql功能；慢SQL记录
    connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000

　　2.3 创建elasticsearch的配置类

　　

package spakt.demo.config;

import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.net.InetAddress;
import java.net.UnknownHostException;

@Configuration
public class EsConfiguration {


    private String clusterName = "elasticsearch";
    @Value("${es.host}")
    private String esHost;

    @Value("${es.port}")
    private String esPort;

    @Bean
    public TransportClient getTransportClient() throws UnknownHostException {

        Settings sets = Settings.builder().put("cluster.name", clusterName) //集群名字
                .put("client.transport.sniff", true)
                .build();
        InetSocketTransportAddress inet = new InetSocketTransportAddress(InetAddress.getByName(esHost), Integer.parseInt(esPort));
        TransportClient transportClient = new PreBuiltTransportClient(sets).addTransportAddress(inet);
        return transportClient;
    }
}

 　　这儿注意 如果是es6.x以后的版本 ，在引入es包之后，还要引入netty插件包，这儿以6.7.2为例

　　

        <!-- https://mvnrepository.com/artifact/org.elasticsearch.plugin/transport-netty4-client -->
        <dependency>
            <groupId>org.elasticsearch.plugin</groupId>
            <artifactId>transport-netty4-client</artifactId>
            <version>6.7.2</version>
        </dependency>

　　并且上面的配置中InetSocketTransportAddress 修改为TransportAddress即可 具体如下

　　

　　 @Bean
    public TransportClient initTransportClient() throws UnknownHostException {
        Settings build = Settings.builder().put("cluster.name", clusterName).put("client.transport.sniff", true).build();
        TransportAddress transportAddress = new TransportAddress(InetAddress.getByName(address),port);
        TransportClient transportClient = new PreBuiltTransportClient(build).addTransportAddress(transportAddress);
        return transportClient;
    }

　　

　　2.4 创建测试的controller类  包含添加，查询，删除方法

　　

package spakt.demo.elasticsearch;

import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import spakt.demo.bean.Prisoner;

import javax.annotation.Resource;
import java.text.DecimalFormat;
import java.util.*;

/**
 *
 */
@Controller
public class EsController {

//    @Resource(name = "jdbc2")
    @Autowired
    private JdbcTemplate jdbcTemplate;

    @Autowired
    private TransportClient transportClient;

    /**
     * 插入数据方法
     *
     * @return
     */
    @RequestMapping("/es")
    @ResponseBody
    public Object addEs() {

        //要插入的数据
        List<Map<String, Object>> prisonerListList = jdbcTemplate.
                queryForList("select id,name,fact,family_address as address from da_prisoner ");
        //批处理类
        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
        int count = 0;
        for (Map<String, Object> prisoner : prisonerListList) {

            //插入方法
            //jyplatform 代表index   理解为数据库中的库
            //prisoner   代表type   理解为数据库总的表
            //UUID  个人理解是为数据库中的主键
            String uuid = UUID.randomUUID().toString().replace("-", "");
            IndexRequestBuilder indexRequestBuilder = transportClient.
                    prepareIndex("jyplatform", "prisoner",uuid)
                    .setSource(prisoner);
            //单个执行
            //IndexResponse indexResponse = indexRequestBuilder.get();

            //批量执行 将添加器加入批处理中
            bulkRequestBuilder.add(indexRequestBuilder);
            if (count % 5000 == 0) {
                //每隔5000条执行一次批量插入的操作
                BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();
                System.err.println(bulkItemResponses.getTook());
                float num = (float) count / prisonerListList.size();
                DecimalFormat df = new DecimalFormat("0.00");
                String s = df.format(num * 100);
                System.err.println("================================" + s + "%");
            }
            count++;
        }
        return prisonerListList;
    }

    /**
     * 查询数据
     *
     * @return
     */
    @RequestMapping("/get")
    @ResponseBody
    public Object getMap(String content) {
        List<Prisoner> prisonerList = new ArrayList<>();
        // 构造搜索条件
        BoolQueryBuilder query = QueryBuilders.boolQuery();
        //fact 即是field  理解为表中的字段
        query.filter(QueryBuilders.matchPhraseQuery("fact", content));
        //开始查询
        SearchRequestBuilder searchRequestBuilder = transportClient
                .prepareSearch("jyplatform")   //确定index
                .setTypes("prisoner")   //确定type
                .setFrom(0)  //从多少条开始  分页使用
                .setSize(10000) //查多少条  分页使用 size-from>10000会报异常
                .setQuery(query); //加入查询条件
        //结果高亮显示  
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field("fact");
        highlightBuilder.preTags("<span style="color:red">");
        highlightBuilder.postTags("</span>");
        searchRequestBuilder.highlighter(highlightBuilder);
        //获取查询结果
        SearchResponse searchResponse = searchRequestBuilder.get();
        if (searchResponse.status() != RestStatus.OK) {
            System.err.println("查询结果错误");
            return "error";
        }
        //得到查询的结果
        SearchHits hits = searchResponse.getHits();
        for (SearchHit hit : hits) {
            //进行高亮处理
            String fact = "";
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            HighlightField field = highlightFields.get("fact");
            if(field!=null){
                Text[] fragments = field.fragments();
                for (Text fragment : fragments) {
                    fact+=fragment.toString();
                }
            }

            prisonerList.add(new Prisoner(hit.getSource().get("id").toString(),
                                          hit.getSource().get("name").toString(),
                                          StringUtils.isBlank(fact)?hit.getSource().get("fact").toString():fact,
                                          hit.getSource().get("address").toString()));
        }
        return prisonerList;
    }

    /**
     * 删除数据
     */
    @RequestMapping("/delete")
    @ResponseBody
    public String delete() {
        //开始查询
        SearchRequestBuilder searchRequestBuilder = transportClient
                .prepareSearch("jyplatform") //确定index
                .setTypes("prisoner") //确定type
                .setFrom(0)  // begin
                .setSize(10000); //size  size-from>10000会报异常
        //获取查询结果
        SearchHits hits = searchRequestBuilder.get().getHits();
        //批量操作
        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
        for (SearchHit hit : hits) {
            //删除方法
            //jyplatform 代表index   理解为数据库中的库
            //prisoner   代表type   理解为数据库总的表
            // hit.getId()  这条数据的id
            DeleteRequestBuilder deleteRequestBuilder = transportClient.
                    prepareDelete("jyplatform", "prisoner", hit.getId());
            //批量删除
//                bulkRequestBuilder.add(deleteRequestBuilder);
//                if(i%5000 ==0){
//                    bulkRequestBuilder.execute();
//                }
            //单个操作
            deleteRequestBuilder.get();
        }
        return "OK";
    }


}

　　2.5 创建启动类启动测试

　　

package spakt.demo;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ImportResource;

@SpringBootApplication
@ImportResource(value = "classpath:spring-*.xml")
public class DemoApplication {

    public static void main(String[] args) {
        SpringApplication.run(DemoApplication.class, args);
    }
}

总结

　　目前只学习了下初步的使用，复杂的应用待后面继续挖掘。集群也，还有分词器一大堆东西待待填坑