SpringBoot2.0+ElasticSearch网盘搜索实现

1、ES是如何实现分布式高并发全文检索
2、简单介绍ES分片Shards分片技术
3、为什么ES主分片对应的备分片不在同一台节点存放
4、索引的主分片定义好后为什么不能做修改
5、ES如何实现高可用容错方案
6、搭建Linux上环境三台ES高可用集群环境
7、基于ES网盘搜索引擎实现

网盘搜索引擎，写个Job 去网上爬虫 存储起来  以JSON的格式 存放在 ES服务器

网盘搜索引擎的原理： 

  采用ES存储抓取的数据 抓取服务器

   测试数据：

##先创建索引

PUT /clouddisk
 
##测试数据maping

POST /clouddisk/_mapping/disk
{
  "disk":{
    "properties":{
       "name":{
         "type":"text",
         "analyzer":"ik_smart",
         "search_analyzer":"ik_smart"
       },
       "source":{
         "type":"keyword"
       },
      "describe":{
         "type":"text",
         "analyzer":"ik_smart",
         "search_analyzer":"ik_smart"
       },
       "shartime":{
         "type":"date"
       },
       "browsetimes":{
         "type":"long"
       },
      "filesize":{
         "type":"float"
       },
      "sharpeople":{
         "type":"keyword"
       },
       "collectiontime":{
         "type":"date"
       },
        "baiduaddres":{
         "type":"keyword"
       }
    }
  }
  
}
 
 
 
 
 POST /clouddisk/disk
{
    "name": "2018史上最全美剧",
    "source": "百度云盘",
    "describe": "该课程由小明提供",
    "shartime": "2018-10-10",
    "browsetimes": 100000,
    "filesize": 4.35,
    "sharpeople": "美国东部",
    "collectiontime": "2018-11-24",
    "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"

}

POST /clouddisk/disk
{
    "name": "2018史上最全韩剧",
    "source": "百度云盘",
    "describe": "该课程小丽提供",
    "shartime": "2018-10-12",
    "browsetimes": 100000,
    "filesize": 6.35,
    "sharpeople": "韩国釜山",
    "collectiontime": "2018-11-24",
    "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"

}

POST /clouddisk/disk
{
    "name": "老友记",
    "source": "百度云盘",
    "describe": "该课程由张三提供",
    "shartime": "2018-10-10",
    "browsetimes": 100000,
    "filesize": 1.35,
    "sharpeople": "美国",
    "collectiontime": "2018-11-24",
    "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"

}

POST /clouddisk/disk
{
    "name": "英语雅思",
    "source": "百度云盘",
    "describe": "该课程由大海提供",
    "shartime": "2018-10-10",
    "browsetimes": 100000,
    "filesize": 1.35,
    "sharpeople": "大海",
    "collectiontime": "2018-11-24",
    "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"

}

POST /clouddisk/disk
{
    "name": "小猪佩奇",
    "source": "百度云盘",
    "describe": "该课程由朱佩琦出品",
    "shartime": "2018-10-10",
    "browsetimes": 100000,
    "filesize": 1.35,
    "sharpeople": "佩奇",
    "collectiontime": "2018-11-24",
    "baiduaddres": "https://pan.baidu.com/s/1VQxFq6JnKh0KP-5aMq-WpA#list/path=%2F"

}
 
 

maven:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.toov5</groupId>
  <artifactId>springboot-esPan</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  
      <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.0.0.RELEASE</version>
        <relativePath /> <!-- lookup parent from repository -->
    </parent>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>

        <dependency>
            <groupId>com.google.collections</groupId>
            <artifactId>google-collections</artifactId>
            <version>1.0-rc2</version>
        </dependency>
        <!-- springboot整合freemarker -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-freemarker</artifactId>
        </dependency>

    </dependencies>
  
</project>

Dao：

package com.toov5.dao;

import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;

import com.toov5.entity.CloudDiskEntity;

public interface CloudDiskDao extends ElasticsearchRepository<CloudDiskEntity, String> {

}

Entity:

package com.toov5.entity;

import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;

import lombok.Data;

@Data
@Document(indexName = "clouddisk", type = "disk")
public class CloudDiskEntity {

    @Id
    private String id;
    // 名称
    private String name;
    // 来源
    private String source;
    // 描述
    private String describe;
    // 分享时间
    private String shartime;
    // 浏览次数
    private Long browsetimes;
    // 文件大小
    private Double filesize;
    // 分享人
    private String sharpeople;
    // 收录时间
    private String collectiontime;
    // 地址
    private String baiduaddres;

}

Controller:

package com.toov5.controller;

import java.util.List;
import java.util.Optional;

import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import com.google.common.collect.Lists;
import com.toov5.dao.CloudDiskDao;
import com.toov5.entity.CloudDiskEntity;

//SpringBoot 整合 ES

@RestController
public class CloudDiskController {
    @Autowired
    private CloudDiskDao cloudDiskDao;
    
    @RequestMapping("/findById/{id}")
    public Optional<CloudDiskEntity> findById(@PathVariable String id) {
        Optional<CloudDiskEntity> findById = cloudDiskDao.findById(id);
        return  findById;
    }
    @RequestMapping("/search")
    public List<CloudDiskEntity> search(String keyWord,String describe) {
        //创建查询   查询所有的
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
        if (!StringUtils.isEmpty(keyWord)) {
            //模糊查询  一定要用ik的中文分词插件！
            MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
            boolQuery.must(matchQuery);
        }
        if (!StringUtils.isEmpty(describe)) {
            //模糊查询  一定要用ik的中文分词插件！
            MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("describe", describe);
            boolQuery.must(matchQuery);
        }
        
        Iterable<CloudDiskEntity> search = cloudDiskDao.search(boolQuery); //查询所有的数据
        return Lists.newArrayList(search);  //通过这个api可以进行转换  //这样查询除了所有的 然后添加match 
    }
}

yml:

spring:
  data:
    elasticsearch:
    ####集群名称
     cluster-name: myes
    ####地址 
     cluster-nodes: 192.168.91.7:9300
  freemarker:
    # 设置模板后缀名
    suffix: .ftl
    # 设置文档类型
    content-type: text/html
    # 设置页面编码格式
    charset: UTF-8
    # 设置页面缓存
    cache: false
    # 设置ftl文件路径
    template-loader-path:
      - classpath:/templates
  # 设置静态文件路径，js,css等
  mvc:
    static-path-pattern: /static/**

启动类：

package com.toov5;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;

@SpringBootApplication
@EnableElasticsearchRepositories(basePackages="com.toov5.dao")   //dao 代码没有@Component的情况下可以这么玩儿 而且这样比较节省代码哈哈
public class AppEs {
   public static void main(String[] args) {
    SpringApplication.run(AppEs.class, args);
}
}

 关键字查询时候，可以进行关键字查询，比like强悍！

分页查询： 很简单 传入参数 Pageable 就OK了  封装好了

GET /clouddisk/disk/_search
{
  "from": 0,
  "size":2,
   "query": {
     "match": {
       "name": "2018"
     }
   }
  
}

page= 0 ,请求页数从0开始   2代表size 每一页代表多少条数据

前端传来参数是第几页

package com.toov5.controller;

import java.util.List;
import java.util.Optional;

import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import com.toov5.dao.CloudDiskDao;
import com.toov5.entity.CloudDiskEntity;

//SpringBoot 整合 ES

@RestController
public class CloudDiskController {
    @Autowired
    private CloudDiskDao cloudDiskDao;
    
    @RequestMapping("/findById/{id}")
    public Optional<CloudDiskEntity> findById(@PathVariable String id) {
        Optional<CloudDiskEntity> findById = cloudDiskDao.findById(id);
        return  findById;
    }
    @RequestMapping("/search")                  //写死了 默认值
    public Page<CloudDiskEntity> search(String keyWord,@PageableDefault(page=0,value=1) Pageable pageable) {
        //创建查询   查询所有的
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
        if (!StringUtils.isEmpty(keyWord)) {
            //模糊查询  一定要用ik的中文分词插件！
            MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
            boolQuery.must(matchQuery);
        }
        
        Page<CloudDiskEntity> search = cloudDiskDao.search(boolQuery,pageable);
        return search;
    }
}

访问：

也可以传入 页数 size数    

 整合到前端展示：

 Controller：

package com.toov5.controller;

import javax.servlet.http.HttpServletRequest;

import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;

import com.toov5.dao.CloudDiskDao;
import com.toov5.entity.CloudDiskEntity;

@Controller
public class PageController {
    @Autowired
    private CloudDiskDao cloudDiskDao;
    
    @RequestMapping("/search")                  //写死了 默认值
    public String search(String keyWord,@PageableDefault(page=0,value=2) Pageable pageable, HttpServletRequest req) {
        Long starTime = System.currentTimeMillis();
        //创建查询   查询所有的
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
        if (!StringUtils.isEmpty(keyWord)) {
            //模糊查询  一定要用ik的中文分词插件！
            MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
            boolQuery.must(matchQuery);
        }
        
        Page<CloudDiskEntity> page = cloudDiskDao.search(boolQuery,pageable);
        req.setAttribute("page", page);
        req.setAttribute("total", page.getTotalElements());
        req.setAttribute("keyword", keyWord);
        Long endTime = System.currentTimeMillis();
        req.setAttribute("time", endTime-starTime );
        return "search";
    }
}

页面展示：

页面：

<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
<head>
    <meta charset="UTF-8">
    <title>Toov5|ES网盘搜索引擎</title>
   <!-- 新 Bootstrap 核心 CSS 文件 -->

<link href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet">

<!-- 可选的Bootstrap主题文件（一般不使用） -->

<script src="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap-theme.min.css"></script>

<!-- jQuery文件。务必在bootstrap.min.js 之前引入 -->

<script src="https://cdn.bootcss.com/jquery/2.1.1/jquery.min.js"></script>

<!-- 最新的 Bootstrap 核心 JavaScript 文件 -->

<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>

</head>
<body  style="display: block; margin: 0 auto;  50%; " >
      <div style="100%;height:60px;" align="center">
           <h2 style="color:#985f0d;">Toov5|ES网盘搜索引擎</h2>
      </div>
      <br/>
      <div align="center">
          <span style="font-size: 18px;" >检索出${total}条数据,耗时:${time}毫秒</span>
      </div>
      <br/>
      <br/>
      <div class="bs-example" data-example-id="striped-table">
      <table class="table table-bordered table-hover">
             <thead>
                 <tr>
                     <th style="text-align:center;" scope="row">链接名称</th>
                     <th style="text-align:center;">文件大小GB</th>
                     <th style="text-align:center;">分享人</th>
                      <th style="text-align:center;">云盘地址</th>
                 </tr>
             </thead>
             <tbody>
                 <#list page.content as p>
                 <tr >
                     <th style="text-align: left;" >
                      <#if keyword??>
                     ${p.name?replace(keyword, '<span style="color: red">${keyword}</span>')}
                        <#else>
                       ${p.name}
                    </#if>

                     
                     </th>
                     <th style="text-align: center;">${p.filesize}</th>
                     <th style="text-align: center;">${p.sharpeople}</th>
                     <th style="text-align: center;"><a href="${p.baiduaddres}">云盘地址</a> </th>
                </tr>
               </#list>    
             </tbody>
      </table>
  <!--    <div style="font-size: 21px;">
              <#list 1..totalPage as i>    
           <#if keyword??>
                 <a href="/search?keyword=${keyword}&page=${i-1}" >${i}</a>
                <#else>
                   <a href="/search?page=${i-1}" >${i}</a>
              </#if>     
  
         </#list>
     
              页
      </div>   -->
         
      </div>
</body>
</html>

分页展示数据：

package com.toov5.controller;

import javax.servlet.http.HttpServletRequest;

import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;

import com.toov5.dao.CloudDiskDao;
import com.toov5.entity.CloudDiskEntity;

@Controller
public class PageController {
    @Autowired
    private CloudDiskDao cloudDiskDao;
    
    @RequestMapping("/search")                  //写死了 默认值
    public String search(String keyWord,@PageableDefault(page=0,value=2) Pageable pageable, HttpServletRequest req) {
        Long starTime = System.currentTimeMillis();
        //创建查询   查询所有的
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();   //创建查询
        if (!StringUtils.isEmpty(keyWord)) {
            //模糊查询  一定要用ik的中文分词插件！
            MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("name", keyWord);
            boolQuery.must(matchQuery);
        }
                
        Page<CloudDiskEntity> page = cloudDiskDao.search(boolQuery,pageable);
        // 计算查询总数
        long total = page.getTotalElements();
        req.setAttribute("total", page.getTotalElements());
        // 计算分页数
        int totalPage = (int) ((total - 1) / pageable.getPageSize() + 1);
        req.setAttribute("totalPage", totalPage);
        req.setAttribute("page", page);
        req.setAttribute("total", page.getTotalElements());
        req.setAttribute("keyword", keyWord);
        Long endTime = System.currentTimeMillis();
        req.setAttribute("time", endTime-starTime );
        return "search";
    }
}

效果：