1 import com.alibaba.fastjson.JSON; 2 import lombok.extern.slf4j.Slf4j; 3 import org.apache.commons.collections4.CollectionUtils; 4 import org.apache.commons.collections4.MapUtils; 5 import org.apache.commons.lang.ArrayUtils; 6 import org.apache.commons.lang.StringUtils; 7 import org.apache.poi.ss.formula.functions.T; 8 import org.elasticsearch.action.bulk.BulkRequest; 9 import org.elasticsearch.action.bulk.BulkResponse; 10 import org.elasticsearch.action.get.GetRequest; 11 import org.elasticsearch.action.get.GetResponse; 12 import org.elasticsearch.action.index.IndexRequest; 13 import org.elasticsearch.action.search.SearchRequest; 14 import org.elasticsearch.action.search.SearchResponse; 15 import org.elasticsearch.action.search.SearchScrollRequest; 16 import org.elasticsearch.action.update.UpdateRequest; 17 import org.elasticsearch.action.update.UpdateResponse; 18 import org.elasticsearch.client.RequestOptions; 19 import org.elasticsearch.client.RestHighLevelClient; 20 import org.elasticsearch.client.indices.GetIndexRequest; 21 import org.elasticsearch.common.unit.TimeValue; 22 import org.elasticsearch.common.xcontent.XContentType; 23 import org.elasticsearch.index.query.BoolQueryBuilder; 24 import org.elasticsearch.index.query.QueryBuilders; 25 import org.elasticsearch.search.Scroll; 26 import org.elasticsearch.search.SearchHit; 27 import org.elasticsearch.search.aggregations.Aggregation; 28 import org.elasticsearch.search.aggregations.AggregationBuilder; 29 import org.elasticsearch.search.aggregations.AggregationBuilders; 30 import org.elasticsearch.search.aggregations.BucketOrder; 31 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; 32 import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; 33 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; 34 import org.elasticsearch.search.aggregations.metrics.Cardinality; 35 import org.elasticsearch.search.builder.SearchSourceBuilder; 36 import org.springframework.beans.factory.annotation.Autowired; 37 import org.springframework.beans.factory.annotation.Qualifier; 38 import org.springframework.stereotype.Component; 39 import org.trimps.ticp.fuzhou.telecom.vo.pagination.BasePageCondition; 40 41 import java.io.IOException; 42 import java.util.ArrayList; 43 import java.util.HashMap; 44 import java.util.List; 45 import java.util.Map; 46 47 @Component 48 @Slf4j 49 public class ESUtils { 50 51 @Autowired 52 @Qualifier("highLevelClient") 53 RestHighLevelClient client; 54 55 public long countLogs(String esIndex, SearchSourceBuilder builder) { 56 SearchRequest searchRequest = new SearchRequest(); 57 searchRequest.indices(esIndex); 58 searchRequest.source(builder); 59 try { 60 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 61 return result.getHits().getTotalHits().value; 62 } catch (Exception e) { 63 log.error("error query data", e); 64 return 0; 65 } 66 } 67 68 /** 69 * 根据id精确查询 70 * 71 * @param esIndex 72 * @param id 73 * @return 74 */ 75 public Map<String, Object> getOne(String esIndex, String id) { 76 GetRequest getRequest = new GetRequest(esIndex, id); 77 try { 78 GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT); 79 if (getResponse.getSource() == null) { 80 return null; 81 } 82 return getResponse.getSourceAsMap(); 83 } catch (Exception e) { 84 log.warn("getOne from ES Exception : ", e); 85 return null; 86 } 87 } 88 89 public SearchHit[] queryDocsByIds(String esIndex, List<String> ids) { 90 SearchSourceBuilder builder = new SearchSourceBuilder(); 91 SearchRequest searchRequest = new SearchRequest(); 92 builder.query(QueryBuilders.termsQuery("_id", ids)); 93 searchRequest.indices(esIndex); 94 searchRequest.source(builder); 95 try { 96 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 97 return result.getHits().getHits(); 98 } catch (Exception e) { 99 log.error("Batch query docs by ids failed!", e); 100 } 101 return null; 102 } 103 104 public void insertOrUpdateOne(String index, String id, String data) { 105 IndexRequest request = new IndexRequest(index); 106 request.id(id); 107 request.source(data, XContentType.JSON); 108 try { 109 client.index(request, RequestOptions.DEFAULT); 110 } catch (Exception e) { 111 log.error("error save info", e); 112 } 113 } 114 115 /** 116 * 修改文档-只需要给要修改的字段 117 */ 118 public void updateOne(String index, String id, Map<String, Object> map) { 119 UpdateRequest updateRequest = new UpdateRequest(index, id); 120 updateRequest.doc(map); 121 try { 122 UpdateResponse updateResponse = client.update(updateRequest, RequestOptions.DEFAULT); 123 log.info("updateOne result:{}", JSON.toJSONString(updateResponse)); 124 } catch (Exception e) { 125 log.warn("updateOne to ES Exception : ", e); 126 throw new RuntimeException(e); 127 } 128 } 129 130 /** 131 * 批量插入文档 132 * 133 * @param index ES索引 134 * @param documents 待提交的批量文档 135 * @param uuidKey 文档中ID字段对应的key值 136 */ 137 public BulkResponse insertDocumentsAsBatch(String index, List<Map<String, Object>> documents, String uuidKey) { 138 BulkResponse response = null; 139 if (StringUtils.isBlank(index) || CollectionUtils.isEmpty(documents)) { 140 log.warn("Es index is blank or documents is empty."); 141 return response; 142 } 143 144 try { 145 int size = documents.size(); 146 BulkRequest bulkRequest = new BulkRequest(); 147 for (int i = 0; i < size; i++) { 148 Map<String, Object> document = documents.get(i); 149 if (MapUtils.isEmpty(document) || !document.containsKey(uuidKey)) { 150 continue; 151 } 152 bulkRequest.add(new IndexRequest(index).opType("create").id(document.get(uuidKey).toString()).source(document)); 153 } 154 response = client.bulk(bulkRequest, RequestOptions.DEFAULT); 155 } catch (Exception e) { 156 log.error("Insert documents to es as batch failed!", e); 157 } 158 return response; 159 } 160 161 162 /** 163 * 批量更新文档 164 * 165 * @param index ES索引 166 * @param documents 待提交的批量文档 167 * @param uuidKey 文档中ID字段对应的key值 168 */ 169 public BulkResponse updateDocumentsAsBatch(String index, List<Map<String, Object>> documents, String uuidKey) { 170 BulkResponse response = null; 171 if (StringUtils.isBlank(index) || CollectionUtils.isEmpty(documents)) { 172 log.warn("Es index is blank or documents is empty."); 173 return response; 174 } 175 176 try { 177 int size = documents.size(); 178 BulkRequest bulkRequest = new BulkRequest(); 179 for (int i = 0; i < size; i++) { 180 Map<String, Object> document = documents.get(i); 181 if (MapUtils.isEmpty(document) || !document.containsKey(uuidKey)) { 182 continue; 183 } 184 bulkRequest.add(new UpdateRequest(index, document.get(uuidKey).toString()).doc(document)); 185 } 186 response = client.bulk(bulkRequest, RequestOptions.DEFAULT); 187 } catch (Exception e) { 188 log.error("Update documents to es as batch failed!", e); 189 } 190 return response; 191 } 192 193 public SearchResponse queryData2(SearchSourceBuilder sourceBuilder, String... esIndex) { 194 SearchRequest searchRequest = new SearchRequest(); 195 searchRequest.indices(esIndex); 196 searchRequest.source(sourceBuilder); 197 try { 198 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 199 return result; 200 } catch (Exception e) { 201 log.error("error query info", e); 202 } 203 return null; 204 } 205 206 /** 207 * 判断索引名是否存在 208 * 209 * @param indexName 210 * @param 211 * @return 212 */ 213 public boolean isExistsIndex(String indexName) { 214 GetIndexRequest request = new GetIndexRequest(indexName); 215 try { 216 boolean response = client.indices().exists(request, RequestOptions.DEFAULT); 217 return response; 218 } catch (IOException e) { 219 log.error("error", e); 220 return false; 221 } 222 223 } 224 225 public long counts4Index(String esIndex) { 226 SearchRequest searchRequest = new SearchRequest(); 227 searchRequest.indices(esIndex); 228 try { 229 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 230 return result.getHits().getTotalHits().value; 231 } catch (Exception e) { 232 log.error("error query data", e); 233 return 0; 234 } 235 } 236 237 public long countDistinctField(String esIndex, String countField, SearchSourceBuilder sourceBuilder) { 238 long count = 0; 239 if (StringUtils.isBlank(esIndex) || StringUtils.isBlank(countField)) { 240 return count; 241 } 242 243 SearchRequest searchRequest = new SearchRequest(); 244 searchRequest.indices(esIndex);
String identifier = UUID.randomUUID().toString(); 245 AggregationBuilder aggregationBuilder = AggregationBuilders.cardinality(identifier).field(countField); 246 sourceBuilder.aggregation(aggregationBuilder); 247 sourceBuilder.size(0); 248 searchRequest.source(sourceBuilder); 249 try { 250 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 251 Histogram histogram = (Histogram) result.getAggregations().asMap().get(countField); 252 long total_value = 0; 253 for (Histogram.Bucket t : histogram.getBuckets()) { 254 Cardinality cardinality = t.getAggregations().get(identifier); 255 long value = cardinality.getValue(); 256 total_value = total_value + value; 257 } 258 return total_value; 259 } catch (Exception e) { 260 log.error("Count field failed!", e); 261 } 262 return 0; 263 } 264 265 public long countDistinctField(String esIndex, String countField) { 266 long count = 0; 267 if (StringUtils.isBlank(esIndex) || StringUtils.isBlank(countField)) { 268 return count; 269 } 270 271 SearchRequest searchRequest = new SearchRequest(); 272 searchRequest.indices(esIndex); 273 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); 274 AggregationBuilder aggregationBuilder = AggregationBuilders.cardinality("field_count").field(countField); 275 sourceBuilder.aggregation(aggregationBuilder); 276 sourceBuilder.size(0); 277 searchRequest.source(sourceBuilder); 278 try { 279 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 280 Histogram histogram = (Histogram) result.getAggregations().asMap().get(countField); 281 long total_value = 0; 282 for (Histogram.Bucket t : histogram.getBuckets()) { 283 Cardinality cardinality = t.getAggregations().get("field_count"); 284 long value = cardinality.getValue(); 285 total_value = total_value + value; 286 } 287 return total_value; 288 } catch (Exception e) { 289 log.error("Count field failed!", e); 290 } 291 return 0; 292 } 293 294 public boolean deleteData(String esIndexName, String esId) { 295 org.elasticsearch.action.delete.DeleteRequest deleteRequest = new org.elasticsearch.action.delete.DeleteRequest(esIndexName, esId); 296 org.elasticsearch.action.delete.DeleteResponse deleteResponse = null; 297 try { 298 deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); 299 } catch (Exception e) { 300 log.error("error is " + e); 301 return false; 302 303 } 304 return true; 305 } 306 307 /** 308 * 分组统计 309 * 310 * @param esIndex 索引 311 * @param groupFiled 被统计字段 312 */ 313 public Map<String, Long> groupCount(String esIndex, String groupFiled) { 314 Map<String, Long> statistics = new HashMap<>(); 315 SearchRequest searchRequest = new SearchRequest(); 316 searchRequest.indices(esIndex); 317 318 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); 319 try { 320 sourceBuilder.size(0); 321 searchRequest.source(sourceBuilder); 322 323 TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("fieldCount").field(groupFiled); 324 sourceBuilder.aggregation(aggregationBuilder); 325 326 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 327 Map<String, Aggregation> aggregationMap = result.getAggregations().asMap(); 328 ParsedStringTerms grageTerms = (ParsedStringTerms) aggregationMap.get("fieldCount"); 329 List buckets = grageTerms.getBuckets(); 330 331 for (Object object : buckets) { 332 ParsedStringTerms.ParsedBucket obj = (ParsedStringTerms.ParsedBucket) object; 333 String name = obj.getKeyAsString(); 334 long count = obj.getDocCount(); 335 statistics.putIfAbsent(name, count); 336 } 337 } catch (Exception e) { 338 log.error("Group count failed!", e); 339 } 340 return statistics; 341 } 342 343 /** 344 * 分组统计 345 * 346 * @param esIndex 索引 347 * @param groupFiled 被统计字段 348 * @param boolQueryBuilder 查询条件 349 */ 350 public Map<String, Long> groupCount(String esIndex, String groupFiled, BoolQueryBuilder boolQueryBuilder) { 351 Map<String, Long> statistics = new HashMap<>(); 352 SearchRequest searchRequest = new SearchRequest(); 353 searchRequest.indices(esIndex); 354 355 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); 356 try { 357 sourceBuilder.size(0); 358 searchRequest.source(sourceBuilder); 359 sourceBuilder.query(boolQueryBuilder); 360 361 TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("fieldCount").field(groupFiled); 362 aggregationBuilder.order(BucketOrder.count(false)); 363 aggregationBuilder.size(10000); 364 sourceBuilder.aggregation(aggregationBuilder); 365 log.info("sourceBuilder:" + sourceBuilder.toString()); 366 367 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 368 Map<String, Aggregation> aggregationMap = result.getAggregations().asMap(); 369 ParsedStringTerms grageTerms = (ParsedStringTerms) aggregationMap.get("fieldCount"); 370 List buckets = grageTerms.getBuckets(); 371 372 for (Object object : buckets) { 373 ParsedStringTerms.ParsedBucket obj = (ParsedStringTerms.ParsedBucket) object; 374 String name = obj.getKeyAsString(); 375 long count = obj.getDocCount(); 376 statistics.putIfAbsent(name, count); 377 } 378 } catch (Exception e) { 379 log.error("Group count failed!", e); 380 } 381 return statistics; 382 } 383 384 public SearchHit[] queryDataBig(String esIndex, SearchSourceBuilder sourceBuilder) { 385 final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1)); 386 SearchRequest searchRequest = new SearchRequest(); 387 searchRequest.indices(esIndex); 388 searchRequest.source(sourceBuilder); 389 searchRequest.scroll(scroll); 390 try { 391 SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT); 392 String scrollId = result.getScrollId(); 393 SearchHit[] searchHits2 = result.getHits().getHits(); 394 List<SearchHit> searchHitList = new ArrayList<>(); 395 for (int i = 0; i < searchHits2.length; i++) { 396 searchHitList.add(searchHits2[i]); 397 } 398 if (result.getHits().getTotalHits().value > 100000) { 399 long count = searchHits2.length; 400 while (count < 100000) { 401 SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); 402 scrollRequest.scroll(scroll); 403 try { 404 result = client.scroll(scrollRequest, RequestOptions.DEFAULT); 405 } catch (Exception e) { 406 log.error("error is " + e); 407 } 408 scrollId = result.getScrollId(); 409 searchHits2 = result.getHits().getHits(); 410 for (int i = 0; i < searchHits2.length; i++) { 411 searchHitList.add(searchHits2[i]); 412 } 413 long size = searchHits2.length; 414 count = count + size; 415 416 } 417 SearchHit[] searchHits = new SearchHit[searchHitList.size()]; 418 searchHitList.toArray(searchHits); 419 return searchHits; 420 } else { 421 while (searchHits2 != null && searchHits2.length > 0) { 422 SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); 423 scrollRequest.scroll(scroll); 424 try { 425 result = client.scroll(scrollRequest, RequestOptions.DEFAULT); 426 } catch (Exception e) { 427 log.error("error is " + e); 428 } 429 scrollId = result.getScrollId(); 430 searchHits2 = result.getHits().getHits(); 431 for (int i = 0; i < searchHits2.length; i++) { 432 searchHitList.add(searchHits2[i]); 433 } 434 } 435 SearchHit[] searchHits = new SearchHit[searchHitList.size()]; 436 searchHitList.toArray(searchHits); 437 return searchHits; 438 } 439 440 } catch (Exception e) { 441 log.error("error query info", e); 442 } 443 return null; 444 } 445 }