zoukankan      html  css  js  c++  java
  • RestHighLevelClient 操作es

    1,根据id 列表查询

      @Autowired
        private RestHighLevelClient restHighLevelClient;
    
        public <T>List<T> getByIds(String index, List<String> ids,Class<T> tClass) {
    
            SearchRequest searchRequest = new SearchRequest();
            //索引
            searchRequest.indices(index);
    //        searchRequest.types(PsdAppListConstant._DOC);
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query( QueryBuilders.idsQuery().addIds(ids.toArray(new String[0])) );
            searchSourceBuilder.size(ids.size());
            log.info(searchSourceBuilder.toString());
    
            searchRequest.source( searchSourceBuilder );
            SearchResponse searchResponse = null;
            try {
                searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            SearchHits hits = searchResponse.getHits();
            log.info("total:{},hitsize:{}",hits.totalHits,hits.getHits().length);
            List<T> result = getResult(hits, tClass);
            return result;
        }
        private <T> List<T> getResult(SearchHits hits, Class<T> tClass) {
            List<T> result = new ArrayList<>();
            for (SearchHit hit : hits) {
                String sourceAsString = hit.getSourceAsString();
                T t = JsonUtils.json2Object(sourceAsString, tClass);
                result.add(t);
            }
            return result;
        }

    2,根据bulk 批量更新

      private void batchUpdateTag(List<V1PsdAppExcelReqDTO> list) throws IOException {
            if (CollectionUtils.isEmpty(list)){
                return;
            }
            BulkRequest request = new BulkRequest();
            for (V1PsdAppExcelReqDTO reqDTO : list) {
                String pkg = reqDTO.getPackageName();
                String appClass = reqDTO.getAppClass();
                String field = getEsFieldFromChinese(appClass);
                if (field == null){
                    String msg = "系统没有找到 "+ appClass +" 对应的 es 三级分类" ;
                    throw new BaseException("120030", msg, 400,  msg);
                }
                Map<String,Object> esMap = new HashMap<>();
                esMap.put("source",getSource(reqDTO.getSource()));
                esMap.put(field,1);
                request.add(new UpdateRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC,pkg).doc(esMap));
            }
            BulkResponse bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
            log.info("update to es has fail:{},msg:{}",bulk.hasFailures(),bulk.buildFailureMessage());
        }

    3,根据bulk 插入数据

      private void batchAddTag(List<V1PsdAppExcelReqDTO> list) throws IOException {
            if (CollectionUtils.isEmpty(list)){
                return;
            }
            BulkRequest request = new BulkRequest();
            UserRequestInfo user = UserRequestInfoHolder.getInstance();
            Integer userNo = user.getUserNo();
            for (V1PsdAppExcelReqDTO reqDTO : list) {
                String pkg = reqDTO.getPackageName();
                String appClass = reqDTO.getAppClass();
                Map<String,Object> esMap = new HashMap<>();
                esMap.put("name",reqDTO.getName());
                esMap.put("package",pkg);
                esMap.put("uploader",userNo.toString());
                esMap.put("source",getSource(reqDTO.getSource()));
                esMap.put("load_time",System.currentTimeMillis());
                String esClassThree = getEsFieldFromChinese(appClass);
                if (esClassThree != null){
                    esMap.put(esClassThree,1);
                }
                request.add(new IndexRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC)
                        .source(JsonUtils.object2Json(esMap), XContentType.JSON)
                        .id(pkg)
                        .create(true));
            }
            BulkResponse bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
            log.info("write to es has fail:{},msg:{}",bulk.hasFailures(),bulk.buildFailureMessage());
        }

    4,利用script 更新指定条件数据

     private void invalidAppTag(V1PsdAppTagTriggerReqDTO reqDTO) throws IOException {
            String esClassThree= getEsClassThreeFromAppClass(reqDTO.getClassThree());
            String reqEsClassThree= getEsClassThreeFromAppClass(reqDTO.getClassThree());
    
            List<String> pkgs = reqDTO.getPkgs();
    
            if (CollectionUtils.isEmpty(pkgs)){// 全选
                V1PsdTagDetailReqDTO esReqDTO = BeanCopierUtils.createAndCopy(reqDTO, V1PsdTagDetailReqDTO.class);
                SearchSourceBuilder builder = buildQueryBuilder(esReqDTO,reqEsClassThree);
    
                UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest();
                updateByQueryRequest.indices(PsdAppListConstant.APP_LIST_DIM);
                updateByQueryRequest.setDocTypes(PsdAppListConstant._DOC);
    
                updateByQueryRequest.setQuery(builder.query());
    
                Map<String, Object> paramsMap = new HashMap<>();
                paramsMap.put("enable", 0);
    
                updateByQueryRequest.setScript(new Script(ScriptType.INLINE,
                        "painless",
                        "ctx._source." + esClassThree + " = params.enable;" +
                        "ctx._source['platform.kepler."+esClassThree+"']= params.enable;" +
                        "ctx._source['platform.anti_defraud."+esClassThree+"']= params.enable;" +
                        "ctx._source['platform.thunderbird."+esClassThree+"']= params.enable;" ,
                        paramsMap));
                log.info("update query:{} ,script:{},source:{}", updateByQueryRequest, updateByQueryRequest.getScript(), updateByQueryRequest.getSearchRequest().source());
                BulkByScrollResponse bulkByScrollResponse = restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
                long updated = bulkByScrollResponse.getUpdated();
                log.info("es  updated:{}", updated);
    
            }else {
                UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest();
                updateByQueryRequest.indices(PsdAppListConstant.APP_LIST_DIM);
                updateByQueryRequest.setDocTypes(PsdAppListConstant._DOC);
    
    
                updateByQueryRequest.setQuery( QueryBuilders.idsQuery().addIds(reqDTO.getPkgs().toArray(new String[0])) );
                Map<String, Object> paramsMap = new HashMap<>();
                paramsMap.put("enable", 0);
                // 外层tag
                updateByQueryRequest.setScript(new Script(ScriptType.INLINE,
                        "painless",
                        "ctx._source." + esClassThree + " = params.enable;" +
                        "ctx._source['platform.kepler."+esClassThree+"']= params.enable;" +
                        "ctx._source['platform.anti_defraud."+esClassThree+"']= params.enable;" +
                        "ctx._source['platform.thunderbird."+esClassThree+"']= params.enable;" ,
                        paramsMap));
                BulkByScrollResponse bulkByScrollResponse = restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
                long updated = bulkByScrollResponse.getUpdated();
                log.info("es updated:{}",updated);
            }
    
        }

     5,条件查询es

    private SearchSourceBuilder buildQueryBuilder(V1PsdTagDetailReqDTO reqDTO,String esField){
            SearchSourceBuilder builder = new SearchSourceBuilder();
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
            if (StringUtils.isNotBlank(reqDTO.getName())) {
                boolQueryBuilder.must(QueryBuilders.matchQuery("name", reqDTO.getName() ).operator(Operator.AND));
            }
            if (StringUtils.isNotBlank(reqDTO.getPkg())) {
                boolQueryBuilder.must(QueryBuilders.matchQuery("package", reqDTO.getPkg() ).operator(Operator.AND));
            }
            if (StringUtils.isNotBlank(reqDTO.getPkgInclude())) {
                String pkgInclude = reqDTO.getPkgInclude();
                String[] pkgs = pkgInclude.split(",");
                boolQueryBuilder.should(QueryBuilders.termsQuery("package", pkgs ));
            }
            if (StringUtils.isNotBlank(reqDTO.getPkgExclude())) {
                String pkgExclude = reqDTO.getPkgExclude();
                String[] pkgs = pkgExclude.split(",");
                boolQueryBuilder.mustNot(QueryBuilders.termsQuery("package", pkgs ));
            }
            if (StringUtils.isNotBlank(reqDTO.getPkgNameInclude())) {
                String pkgNameInclude = reqDTO.getPkgNameInclude();
                String[] names = pkgNameInclude.split(",");
                boolQueryBuilder.should(QueryBuilders.termsQuery("name", names ));
            }
            if (StringUtils.isNotBlank(reqDTO.getPkgNameExclude())) {
                String pkgNameExclude = reqDTO.getPkgNameExclude();
                String[] names = pkgNameExclude.split(",");
                boolQueryBuilder.mustNot(QueryBuilders.termsQuery("name", names ));
            }
    
            boolQueryBuilder.should(QueryBuilders.termQuery(esField, 1 ));
            boolQueryBuilder.minimumShouldMatch(1);
            boolQueryBuilder.mustNot(QueryBuilders.termQuery(esField, 0 ));// 弃用和删除的不查
            List<String> systems = reqDTO.getSystems();
            if (CollectionUtils.isNotEmpty(systems)){
                for (String system : systems) {
                    boolQueryBuilder.must(QueryBuilders.termQuery("platform."+system+"." + esField, 1));
                }
            }
    
            if (StringUtils.isNotBlank(reqDTO.getLoadTimeSort())) {
                if (PsdTagConstant.ASC.equals(reqDTO.getLoadTimeSort())){
                    builder.sort("load_time",SortOrder.ASC);
                }else {
                    builder.sort("load_time",SortOrder.DESC);
                }
            }
    
    
    
            builder.query(boolQueryBuilder);
            Integer current = reqDTO.getCurrent();
            if (current != null){
                int size = reqDTO.getSize();
                int from = (current - 1) * size;
                if (from + size > PsdAppListConstant.ES_SEARCH_LIMIT) {
                    from = PsdAppListConstant.ES_SEARCH_LIMIT - size;
                }
                builder.from(from);
                builder.size(size);
    
            }
            return builder;
        }
    
     SearchSourceBuilder builder = buildQueryBuilder(reqDTO,esField);
    
            SearchRequest searchRequest = new SearchRequest();
            //索引
            searchRequest.indices(PsdAppListConstant.APP_LIST_DIM);
            searchRequest.types(PsdAppListConstant._DOC);
            searchRequest.source(builder);
            log.info(searchRequest.source().toString());
            SearchResponse searchResponse = null;
            try {
                searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            SearchHits hits = searchResponse.getHits();
            long totalHits = hits.getTotalHits();
            log.info("total:{}",totalHits);

     6.添加/修改单个es数据

     public void addToEs(V1PsdUserAppTagReqDTO reqDTO) throws IOException {
            String pkg = reqDTO.getPackageName();
      if (CollectionUtils.isEmpty(appEsMap)){
                Map<String,Object> esMap = new HashMap<>();
                esMap.put("name",reqDTO.getName());
                esMap.put("package",reqDTO.getPackageName());
                esMap.put("uploader",userNo.toString());
                esMap.put("source",PsdTagConstant.SOURCE_PERSON);
                esMap.put("load_time",System.currentTimeMillis());
                for (String esClassThree : esClassThrees) {
                    esMap.put(esClassThree,1);
                }
    
                IndexRequest indexRequest = new IndexRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC);
                indexRequest.source(JsonUtils.object2Json(esMap), XContentType.JSON);
                indexRequest.id(pkg);
                indexRequest.create(true);
                IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
                String id = indexResponse.getId();
                log.info("es create doc id :{}",id);
            }else {
                Map esApp = appEsMap.get(pkg);
                UpdateRequest request = new UpdateRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC,pkg);
                Map<String, Object> jsonMap = new HashMap<>();
                jsonMap.put("load_time",System.currentTimeMillis());
                jsonMap.put("source",PsdTagConstant.SOURCE_PERSON);
                jsonMap.put("uploader",userNo.toString());
    
                Map<String, Object> activeTag = getActiveTag(esApp);
                jsonMap.putAll(activeTag);
    
                for (String esClassThree : esClassThrees) {
                    jsonMap.put(esClassThree,1);
    
                }
                request.doc(jsonMap);
                UpdateResponse updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT);
                String id = updateResponse.getId();
                log.info("es update doc id :{}",id);
            }
    
        }
  • 相关阅读:
    jQuery的遍历方法
    xampp配置host和httpd可以随意访问任何本机的地址
    JavaScript的this简单实用
    移动端rem布局和百分比栅格化布局
    你知道用AngularJs怎么定义指令吗?
    谈谈Angular关于$watch,$apply 以及 $digest的工作原理
    深入了解Angularjs指令中的ngModel
    如何将angularJs项目与requireJs集成
    requireJS(二)
    requireJS(一)
  • 原文地址:https://www.cnblogs.com/dongma/p/15073973.html
Copyright © 2011-2022 走看看