diff --git a/core/pom.xml b/core/pom.xml index 22dee3b902a2..3c22fea211c0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -577,9 +577,10 @@ spring-test compile + - org.elasticsearch.client - elasticsearch-rest-high-level-client + co.elastic.clients + elasticsearch-java diff --git a/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java b/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java index cd4003319640..a42a9e982e99 100644 --- a/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java @@ -23,13 +23,14 @@ package org.fao.geonet.kernel; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import jeeves.server.UserSession; import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Edit; import org.fao.geonet.constants.Geonet; @@ -248,8 +249,9 @@ public void selectAll(String type, ServiceContext context, UserSession session) EsSearchManager searchManager = context.getBean(EsSearchManager.class); searchResponse = searchManager.query(request.get("query"), FIELDLIST_UUID, 0, maxhits); List uuidList = new ArrayList(); - for (SearchHit h : Arrays.asList(searchResponse.getHits().getHits())) { - uuidList.add((String) h.getSourceAsMap().get(Geonet.IndexFieldNames.UUID)); + ObjectMapper objectMapper = new ObjectMapper(); + for (Hit h : (List) searchResponse.hits().hits()) { + uuidList.add((String) objectMapper.convertValue(h.source(), Map.class).get(Geonet.IndexFieldNames.UUID)); } if (selection != null) { diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 61ba2251061e..24ca8c65a4e1 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -1342,7 +1342,7 @@ public boolean isValid(Integer id) { boolean hasReferencingMetadata(ServiceContext context, AbstractMetadata metadata) throws Exception { StringBuilder query = new StringBuilder(String.format("xlink:*%s*", metadata.getUuid())); - return this.searchManager.query(query.toString(), null, 0, 0).getHits().getTotalHits().value > 0; + return this.searchManager.query(query.toString(), null, 0, 0).hits().hits().size() > 0; } } diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java b/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java index 04d8e9cff3cb..b202ab051a76 100644 --- a/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java +++ b/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java @@ -23,10 +23,11 @@ package org.fao.geonet.kernel.mef; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import com.fasterxml.jackson.databind.ObjectMapper; import jeeves.server.context.ServiceContext; import org.apache.commons.io.FileUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; import org.fao.geonet.Constants; import org.fao.geonet.GeonetContext; import org.fao.geonet.ZipUtil; @@ -137,8 +138,9 @@ public static Path doExport(ServiceContext context, Set uuids, String mdSchema = null, mdTitle = null, mdAbstract = null, isHarvested = null; MetadataType mdType = null; - SearchHit[] hits = result.getHits().getHits(); - final Map source = hits[0].getSourceAsMap(); + List hits = result.hits().hits(); + ObjectMapper objectMapper = new ObjectMapper(); + final Map source = objectMapper.convertValue(hits.get(0).source(), Map.class); mdSchema = (String) source.get(Geonet.IndexFieldNames.SCHEMA); mdTitle = (String) source.get(Geonet.IndexFieldNames.RESOURCETITLE); mdAbstract = (String) source.get(Geonet.IndexFieldNames.RESOURCEABSTRACT); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 97e76787f97a..f55201b787ba 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -23,6 +23,14 @@ package org.fao.geonet.kernel.search; +import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch.core.*; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.indices.*; +import co.elastic.clients.elasticsearch.indices.ExistsRequest; +import co.elastic.clients.transport.endpoints.BooleanResponse; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -35,25 +43,6 @@ import jeeves.server.context.ServiceContext; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.client.indices.CreateIndexResponse; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.*; @@ -82,8 +71,6 @@ import java.nio.file.StandardOpenOption; import java.util.*; -import static org.elasticsearch.rest.RestStatus.CREATED; -import static org.elasticsearch.rest.RestStatus.OK; import static org.fao.geonet.constants.Geonet.IndexFieldNames.IS_TEMPLATE; import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_FIELD; import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_MSG; @@ -268,9 +255,12 @@ public void init(boolean dropIndexFirst, Optional> indices) throws private void createIndex(String indexId, String indexName, boolean dropIndexFirst) throws IOException { if (dropIndexFirst) { try { - DeleteIndexRequest request = new DeleteIndexRequest(indexName); - AcknowledgedResponse deleteIndexResponse = client.getClient().indices().delete(request, RequestOptions.DEFAULT); - if (deleteIndexResponse.isAcknowledged()) { + DeleteIndexRequest deleteIndexRequest = DeleteIndexRequest.of( + b -> b.index(indexName) + ); + + DeleteIndexResponse deleteIndexResponse = client.getClient().indices().delete(deleteIndexRequest); + if (deleteIndexResponse.acknowledged()) { LOGGER.debug("Index '{}' removed.", new Object[]{indexName}); } } catch (Exception e) { @@ -280,35 +270,37 @@ private void createIndex(String indexId, String indexName, boolean dropIndexFirs } // Check index exist first - GetIndexRequest request = new GetIndexRequest(indexName); + ExistsRequest existsRequest = ExistsRequest.of( + b -> b.index(indexName) + ); + try { - boolean exists = client.getClient().indices().exists(request, RequestOptions.DEFAULT); - if (exists && !dropIndexFirst) { + BooleanResponse exists = client.getClient().indices().exists(existsRequest); + if (exists.value() && !dropIndexFirst) { return; } - - if (!exists || dropIndexFirst) { + if (!exists.value() || dropIndexFirst) { // Check version of the index - how ? // Create it if not Path indexConfiguration = dataDirectory.getIndexConfigDir().resolve(indexId + ".json"); if (Files.exists(indexConfiguration)) { - String configuration; - try (InputStream is = Files.newInputStream(indexConfiguration, StandardOpenOption.READ)) { - configuration = IOUtils.toString(is); - } - - CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); - createIndexRequest.source(configuration, XContentType.JSON); - CreateIndexResponse createIndexResponse = client.getClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); - if (createIndexResponse.isAcknowledged()) { - LOGGER.debug("Index '{}' created", new Object[]{indexName}); - } else { - final String message = String.format("Index '%s' was not created. Error is: %s", indexName, createIndexResponse.toString()); - LOGGER.error(message); - throw new IllegalStateException(message); + try (InputStream is = Files.newInputStream(indexConfiguration, StandardOpenOption.READ)) { + CreateIndexRequest createIndexRequest = CreateIndexRequest.of( + b -> b.index(indexName).withJson(is) + ); + + CreateIndexResponse createIndexResponse = client.getClient().indices().create(createIndexRequest); + + if (createIndexResponse.acknowledged()) { + LOGGER.debug("Index '{}' created", new Object[]{indexName}); + } else { + final String message = String.format("Index '%s' was not created. Error is: %s", indexName, createIndexResponse.toString()); + LOGGER.error(message); + throw new IllegalStateException(message); + } } } else { throw new FileNotFoundException(String.format( @@ -331,8 +323,14 @@ public void end() { public UpdateResponse updateFields(String id, Map fields) throws IOException { fields.put(Geonet.IndexFieldNames.INDEXING_DATE, new Date()); - UpdateRequest updateRequest = new UpdateRequest(defaultIndex, id).doc(fields); - return client.getClient().update(updateRequest, RequestOptions.DEFAULT); + + UpdateRequest updateRequest = UpdateRequest.of( + b -> b.index(defaultIndex) + .id(id) + .doc(fields) + ); + + return client.getClient().update(updateRequest, Void.class); } public BulkResponse updateFields(String id, Multimap fields, Set fieldsToRemove) throws IOException { @@ -342,36 +340,62 @@ public BulkResponse updateFields(String id, Multimap fields, Set } public BulkResponse updateFields(String id, Map fieldMap, Set fieldsToRemove) throws IOException { fieldMap.put(Geonet.IndexFieldNames.INDEXING_DATE, new Date()); - BulkRequest bulkrequest = new BulkRequest(); - StringBuilder script = new StringBuilder(); - fieldsToRemove.forEach(f -> - script.append(String.format("ctx._source.remove('%s');", f))); - UpdateRequest deleteFieldRequest = - new UpdateRequest(defaultIndex, id).script(new Script(ScriptType.INLINE, - "painless", - script.toString(), - Collections.emptyMap())); - bulkrequest.add(deleteFieldRequest); - UpdateRequest addFieldRequest = new UpdateRequest(defaultIndex, id) - .doc(fieldMap); - bulkrequest.add(addFieldRequest); - return client.getClient().bulk(bulkrequest, RequestOptions.DEFAULT); + StringBuilder scriptSource = new StringBuilder(); + fieldsToRemove.forEach(f -> + scriptSource.append(String.format("ctx._source.remove('%s');", f))); + + UpdateOperation deleteFieldsOperation = UpdateOperation.of( + b -> b.id(id) + .index(defaultIndex) + .action(action -> action + .scriptedUpsert(true) + .upsert(ImmutableMap.of()) + .script(script -> script + .inline(inlineScript -> inlineScript + .lang("painless") + .source(scriptSource.toString()) + ) + ) + ) + ); + + UpdateOperation addFieldRequestOperation = UpdateOperation.of( + b -> b.id(id) + .index(defaultIndex) + .action(action -> action.upsert(fieldMap)) + ); + + List bulkOperationList = new ArrayList<>(); + bulkOperationList.add(BulkOperation.of(b -> b.update(deleteFieldsOperation))); + bulkOperationList.add(BulkOperation.of(b -> b.update(addFieldRequestOperation))); + + BulkRequest bulkRequest = BulkRequest.of( + b -> b.index(defaultIndex) + .operations(bulkOperationList) + ); + + return client.getClient().bulk(bulkRequest); } public void updateFieldsAsynch(String id, Map fields) { fields.put(Geonet.IndexFieldNames.INDEXING_DATE, new Date()); - UpdateRequest request = new UpdateRequest(defaultIndex, id).doc(fields); - ActionListener listener = new ActionListener() { - @Override - public void onResponse(UpdateResponse updateResponse) { - } - @Override - public void onFailure(Exception e) { - } - }; - client.getClient().updateAsync(request, RequestOptions.DEFAULT, listener); + UpdateRequest updateRequest = UpdateRequest.of( + b -> b.index(defaultIndex) + .id(id) + .doc(fields) + ); + + client.getAsynchClient() + .update(updateRequest, ObjectNode.class) + .whenComplete((response, exception) -> { + if (exception != null) { + LOGGER.error("Failed to index", exception); + } else { + LOGGER.info("Updated fields for document " + id); + } + }); } public UpdateResponse updateField(String id, String field, Object value) throws Exception { @@ -454,22 +478,24 @@ private void sendDocumentsToIndex() { private void checkIndexResponse(BulkResponse bulkItemResponses, Map documents) throws IOException { - if (bulkItemResponses.hasFailures()) { - Map listErrorOfDocumentsToIndex = new HashMap<>(bulkItemResponses.getItems().length); + if (bulkItemResponses.errors()) { + Map listErrorOfDocumentsToIndex = new HashMap<>(bulkItemResponses.items().size()); List errorDocumentIds = new ArrayList<>(); // Add information in index that some items were not properly indexed - Arrays.stream(bulkItemResponses.getItems()).forEach(e -> { - if (e.status() != OK - && e.status() != CREATED) { - errorDocumentIds.add(e.getId()); + bulkItemResponses.items().forEach(e -> { + // TODO: ES 8, check e.status values + //if (e.status() != OK + // && e.status() != CREATED) { + if (e.error() != null) { + errorDocumentIds.add(e.id()); ObjectMapper mapper = new ObjectMapper(); ObjectNode docWithErrorInfo = mapper.createObjectNode(); - String resourceTitle = String.format("Document #%s", e.getId()); + String resourceTitle = String.format("Document #%s", e.id()); String id = ""; String uuid = ""; String isTemplate = ""; - String failureDoc = documents.get(e.getId()); + String failureDoc = documents.get(e.id()); try { JsonNode node = mapper.readTree(failureDoc); resourceTitle = node.get("resourceTitleObject").get("default").asText(); @@ -485,25 +511,25 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, docWithErrorInfo.put(IndexFields.DRAFT, "n"); docWithErrorInfo.put(INDEXING_ERROR_FIELD, true); ArrayNode errors = docWithErrorInfo.putArray(INDEXING_ERROR_MSG); - errors.add(e.getFailureMessage()); + errors.add(e.error().reason()); // TODO: Report the JSON which was causing the error ? LOGGER.error("Document with error #{}: {}.", - new Object[]{e.getId(), e.getFailureMessage()}); + new Object[]{e.id(), e.error().reason()}); LOGGER.error(failureDoc); try { - listErrorOfDocumentsToIndex.put(e.getId(), mapper.writeValueAsString(docWithErrorInfo)); + listErrorOfDocumentsToIndex.put(e.id(), mapper.writeValueAsString(docWithErrorInfo)); } catch (JsonProcessingException e1) { LOGGER.error("Generated document for the index is not properly formatted. Check document #{}: {}.", - new Object[]{e.getId(), e1.getMessage()}); + new Object[]{e.id(), e1.getMessage()}); } } }); if (listErrorOfDocumentsToIndex.size() > 0) { BulkResponse response = client.bulkRequest(defaultIndex, listErrorOfDocumentsToIndex); - if (response.status().getStatus() != 201) { + if (response.errors()) { LOGGER.error("Failed to save error documents {}.", new Object[]{Arrays.toString(errorDocumentIds.toArray())}); } @@ -761,7 +787,7 @@ public SearchResponse query(String luceneQuery, String filterQuery, int startPos return client.query(defaultIndex, luceneQuery, filterQuery, new HashSet<>(), new HashMap<>(), startPosition, maxRecords); } - public SearchResponse query(String luceneQuery, String filterQuery, int startPosition, int maxRecords, List> sort) throws Exception { + public SearchResponse query(String luceneQuery, String filterQuery, int startPosition, int maxRecords, List sort) throws Exception { return client.query(defaultIndex, luceneQuery, filterQuery, new HashSet<>(), new HashMap<>(), startPosition, maxRecords, sort); } @@ -777,7 +803,7 @@ public SearchResponse query(String luceneQuery, String filterQuery, Set } public SearchResponse query(JsonNode jsonRequest, Set includedFields, - int from, int size, List> sort) throws Exception { + int from, int size, List sort) throws Exception { // TODO: Review postFilterBuilder return client.query(defaultIndex, jsonRequest, null, includedFields, new HashMap<>(), from, size, sort); } @@ -837,7 +863,13 @@ public Map getDocsChangeDate() throws Exception { final SearchResponse response = client.query(defaultIndex, "*", null, docsChangeIncludedFields, from, size); - response.getHits().forEach(r -> docs.put(r.getId(), (String) r.getSourceAsMap().get(Geonet.IndexFieldNames.DATABASE_CHANGE_DATE))); + ObjectMapper objectMapper = new ObjectMapper(); + + response.hits().hits().forEach(r -> { + Hit h = (Hit) r; + + docs.put(h.id(), (String) objectMapper.convertValue(h.source(), Map.class).get(Geonet.IndexFieldNames.DATABASE_CHANGE_DATE)); + }); } catch (Exception e) { LOGGER.error("Error while collecting all documents: {}", e.getMessage()); e.printStackTrace(); @@ -854,9 +886,12 @@ public ISODate getDocChangeDate(String mdId) throws Exception { final SearchResponse response = client.query(defaultIndex, "_id:" + mdId, null, docsChangeIncludedFields, from, size); - if (response.getHits().getTotalHits().value == 1) { + if (response.hits().hits().size() == 1) { + Hit hit = (Hit) response.hits().hits().get(0); + ObjectMapper objectMapper = new ObjectMapper(); + String date = - (String) response.getHits().getAt(0).getSourceAsMap().get(Geonet.IndexFieldNames.DATABASE_CHANGE_DATE); + (String) objectMapper.convertValue(hit.source(), Map.class).get(Geonet.IndexFieldNames.DATABASE_CHANGE_DATE); return date != null ? new ISODate(date) : null; } else { return null; @@ -911,11 +946,7 @@ public Set getDocsWithXLinks() throws Exception { @Override public void delete(String txt) throws Exception { - DeleteByQueryRequest request = new DeleteByQueryRequest(); - request.indices(defaultIndex); - request.setQuery(new QueryStringQueryBuilder(txt)); - request.setRefresh(true); - client.getClient().deleteByQuery(request, RequestOptions.DEFAULT); + client.deleteByQuery(defaultIndex, txt); } @Override @@ -944,7 +975,7 @@ public long getNumDocs(String query) throws Exception { int size = Integer.parseInt(si.getSelectionMaxRecords()); final SearchResponse response = client.query(defaultIndex, query, null, docsChangeIncludedFields, from, size); - return response.getHits().getTotalHits().value; + return response.hits().hits().size(); } public EsRestClient getClient() { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/OverviewIndexFieldUpdater.java b/core/src/main/java/org/fao/geonet/kernel/search/index/OverviewIndexFieldUpdater.java index 19944c779e4f..fd97543ef154 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/OverviewIndexFieldUpdater.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/OverviewIndexFieldUpdater.java @@ -23,8 +23,10 @@ package org.fao.geonet.kernel.search.index; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.util.XslUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -63,9 +65,11 @@ private void processOverview(String id) { response = searchManager.query(String.format( "+id:\"%s\" _exists_:overview.url -_exists_:overview.data", id), null, source, 0, 1); - response.getHits().forEach(hit -> { + ObjectMapper objectMapper = new ObjectMapper(); + response.hits().hits().forEach(h -> { + Hit hit = (Hit) h; AtomicBoolean updates = new AtomicBoolean(false); - Map fields = hit.getSourceAsMap(); + Map fields = objectMapper.convertValue(hit.source(), Map.class); getHitOverviews(fields) .stream() .forEach(overview -> { diff --git a/core/src/main/java/org/fao/geonet/util/XslUtil.java b/core/src/main/java/org/fao/geonet/util/XslUtil.java index 4ac4474571c7..f249d513a9eb 100644 --- a/core/src/main/java/org/fao/geonet/util/XslUtil.java +++ b/core/src/main/java/org/fao/geonet/util/XslUtil.java @@ -23,6 +23,7 @@ package org.fao.geonet.util; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -43,16 +44,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.SystemInfo; import org.fao.geonet.api.records.attachments.FilesystemStore; @@ -469,7 +461,7 @@ public static String getNodeId() { return ApplicationContextHolder.get().getBean(org.fao.geonet.NodeInfo.class).getId(); } - + public static String getNodeLogo(String key) { Optional source = getSource(key); return source.isPresent() ? source.get().getLogo() : ""; @@ -1587,7 +1579,8 @@ public static Element getTargetAssociatedResources(String uuid, String parentUui Element recordLinks = new Element("recordLinks"); try { - MultiSearchRequest request = new MultiSearchRequest(); + // TODO: ES 8 + /*MultiSearchRequest request = new MultiSearchRequest(); SearchRequest serviceRequest = new SearchRequest(searchManager.getDefaultIndex()); @@ -1656,7 +1649,7 @@ public static Element getTargetAssociatedResources(String uuid, String parentUui if (hasParent) { recordLinks.addContent(buildRecordLink(response.getResponses()[3].getResponse().getHits(), "brothersAndSisters")); - } + }*/ } catch (Exception e) { Log.error(Geonet.GEONETWORK, "Get related document '" + uuid + "' error: " + e.getMessage(), e); @@ -1677,24 +1670,27 @@ public static Node getTargetAssociatedResourcesAsNode(String uuid, String parent return null; } - private static List buildRecordLink(SearchHits hits, String type) { + private static List buildRecordLink(List hits, String type) { ObjectMapper mapper = new ObjectMapper(); SettingManager settingManager = ApplicationContextHolder.get().getBean(SettingManager.class); String recordUrlPrefix = settingManager.getNodeURL() + "api/records/"; ArrayList listOfLinks = new ArrayList<>(); + + ObjectMapper objectMapper = new ObjectMapper(); + hits.forEach(record -> { Element recordLink = new Element("recordLink"); recordLink.setAttribute("type", "object"); ObjectNode recordLinkProperties = mapper.createObjectNode(); - recordLinkProperties.put("to", record.getId()); + recordLinkProperties.put("to", record.id()); recordLinkProperties.put("origin", "catalog"); recordLinkProperties.put("created", "bySearch"); - Map titleObject = (Map) record.getSourceAsMap().get("resourceTitleObject"); + Map titleObject = (Map) objectMapper.convertValue(record.source(), Map.class).get("resourceTitleObject"); if (titleObject != null) { recordLinkProperties.put("title", titleObject.get("default")); } - recordLinkProperties.put("url", recordUrlPrefix + record.getId()); + recordLinkProperties.put("url", recordUrlPrefix + record.id()); recordLinkProperties.put("type", type); try { diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index 29748c46e3a3..baf0778d1a46 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -194,12 +194,12 @@ public void testDeleteBatchMetadata() throws Exception { importMetadata(serviceContext, uuid1); importMetadata(serviceContext, uuid2); assertEquals(2, metadataRepository.findAll(spec).size()); - assertEquals(2, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).getHits().getTotalHits().value); + assertEquals(2, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).hits().hits().size()); dataManager.batchDeleteMetadataAndUpdateIndex(spec); assertEquals(0, metadataRepository.findAll(spec).size()); - assertEquals(0, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).getHits().getTotalHits().value); + assertEquals(0, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).hits().hits().size()); } @Test diff --git a/core/src/test/java/org/fao/geonet/kernel/ElasticsearchIndexingTest.java b/core/src/test/java/org/fao/geonet/kernel/ElasticsearchIndexingTest.java index 6170271f79ad..e13577def460 100644 --- a/core/src/test/java/org/fao/geonet/kernel/ElasticsearchIndexingTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/ElasticsearchIndexingTest.java @@ -1,7 +1,7 @@ package org.fao.geonet.kernel; +import co.elastic.clients.elasticsearch.core.SearchResponse; import jeeves.server.context.ServiceContext; -import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.AbstractCoreIntegrationTest; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.search.EsSearchManager; @@ -47,7 +47,7 @@ public void complexDatesAreIndexedCheck() throws Exception { //THEN SearchResponse response = this.searchManager.query("_id:" + dbInsertedMetadata.getUuid() + " AND resourceTitleObject.default:holocene", null, 0, 10); - long actualHitNbr = response.getHits().getTotalHits().value; + long actualHitNbr = response.hits().hits().size(); assertEquals(String.format("Incorrect indexation of Holocene data with complex date due to: %s and %s", response, dbInsertedMetadata), 1, actualHitNbr); } @@ -59,7 +59,7 @@ private AbstractMetadata loadMetadataWithTemporalExtentUsingSimpleDates() throws private void validateIndexedExpectedData(AbstractMetadata dbInsertedSimpleDataMetadata, String resourceTitle, long expectedHitNbr) throws Exception { SearchResponse searchResponse = this.searchManager.query("resourceTitleObject.default:" + resourceTitle, null, 0, 10); - long actualHitNbr = searchResponse.getHits().getTotalHits().value; + long actualHitNbr = searchResponse.hits().hits().size(); String assertionErrorMessage = "The %s data was not indexed the expected number of times due to: %s and %s"; assertEquals(String.format(assertionErrorMessage, resourceTitle, searchResponse, dbInsertedSimpleDataMetadata), expectedHitNbr, actualHitNbr); } diff --git a/csw-server/pom.xml b/csw-server/pom.xml index 521aa8259509..dd0789e33601 100644 --- a/csw-server/pom.xml +++ b/csw-server/pom.xml @@ -105,6 +105,11 @@ elasticsearch + + co.elastic.clients + elasticsearch-java + + org.springframework spring-beans diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java index feffc0696ab7..5b0fb4e224dc 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java @@ -23,11 +23,9 @@ package org.fao.geonet.component.csw; +import co.elastic.clients.elasticsearch._types.SortOptions; import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortOrder; import org.fao.geonet.GeonetContext; import org.fao.geonet.constants.Geonet; import org.fao.geonet.csw.common.*; @@ -244,7 +242,7 @@ public Element execute(Element request, ServiceContext context) throws CatalogEx response.addContent(echoedRequest); } else { - List> sort = _sortByParser.parseSortBy(request); + List sort = _sortByParser.parseSortBy(request); response = new Element(getName() + "Response", Csw.NAMESPACE_CSW); diff --git a/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SearchController.java b/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SearchController.java index 3002391bbf6a..667d9e469cd2 100644 --- a/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SearchController.java +++ b/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SearchController.java @@ -23,14 +23,13 @@ package org.fao.geonet.kernel.csw.services.getrecords; +import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.GeonetContext; import org.fao.geonet.NodeInfo; @@ -53,20 +52,12 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.utils.Log; import org.fao.geonet.utils.Xml; -import org.geotools.xsd.Configuration; -import org.geotools.xsd.Parser; import org.jdom.Attribute; import org.jdom.Content; import org.jdom.Element; import org.jdom.Namespace; -import org.opengis.filter.Filter; -import org.opengis.filter.capability.FilterCapabilities; import org.springframework.beans.factory.annotation.Autowired; -import org.xml.sax.SAXException; -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.io.StringReader; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -439,7 +430,7 @@ private static Element applyElementNames(ServiceContext context, Set ele */ public Element search(ServiceContext context, int startPos, int maxRecords, ResultType resultType, String outSchema, ElementSetName setName, - Element filterExpr, String filterVersion, List> sort, + Element filterExpr, String filterVersion, List sort, Set elemNames, String typeName, int maxHitsFromSummary, String strategy) throws CatalogException { @@ -462,11 +453,11 @@ public Element search(ServiceContext context, int startPos, int maxRecords, // TODO: Check to get summary or remove custom summary output try { - SearchResponse result = searchManager.query(esJsonQuery, new HashSet(), startPos-1, maxRecords, sort); + SearchResponse result = searchManager.query(esJsonQuery, new HashSet<>(), startPos-1, maxRecords, sort); - SearchHit[] hits = result.getHits().getHits(); + List hits = result.hits().hits(); - long numMatches = result.getHits().getTotalHits().value; + long numMatches = result.hits().hits().size(); if (numMatches != 0 && startPos > numMatches) { throw new InvalidParameterValueEx("startPosition", String.format( @@ -477,8 +468,10 @@ public Element search(ServiceContext context, int startPos, int maxRecords, int counter = 0; - for(SearchHit hit : hits) { - int mdId = Integer.parseInt((String) hit.getSourceAsMap().get("id")); + ObjectMapper objectMapper = new ObjectMapper(); + + for(Hit hit : hits) { + int mdId = Integer.parseInt((String) objectMapper.convertValue(hit.source(), Map.class).get("id")); AbstractMetadata metadata = metadataUtils.findOne(mdId); @@ -543,7 +536,7 @@ public Element applyElementSetName(ServiceContext context, SchemaManager schemaM outputSchema, id, schema, styleSheet.toString())); return null; } else { - Map params = new HashMap(); + Map params = new HashMap<>(); params.put("lang", displayLanguage); try { diff --git a/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SortByParser.java b/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SortByParser.java index 87a00f763669..0efcae035ff7 100644 --- a/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SortByParser.java +++ b/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SortByParser.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2021 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -23,19 +23,20 @@ package org.fao.geonet.kernel.csw.services.getrecords; +import co.elastic.clients.elasticsearch._types.FieldSort; +import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.SortOrder; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortOrder; import org.fao.geonet.csw.common.Csw; import org.fao.geonet.kernel.csw.CatalogConfiguration; import org.jdom.Element; import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import static co.elastic.clients.elasticsearch._types.SortOptions.*; + public class SortByParser { @Autowired @@ -44,7 +45,7 @@ public class SortByParser { @Autowired private CatalogConfiguration _catalogConfig; - public List> parseSortBy(Element request) { + public List parseSortBy(Element request) { Element query = request.getChild("Query", Csw.NAMESPACE_CSW); if (query == null) { return getDefaultSort(); @@ -55,14 +56,22 @@ public List> parseSortBy(Element request) { return getDefaultSort(); } - List> sortFields = new ArrayList<>(); + List sortFields = new ArrayList<>(); @SuppressWarnings("unchecked") List list = sortBy.getChildren(); for (Element el : list) { String esSortFieldName = getEsSortFieldName(el); if (!StringUtils.isEmpty(esSortFieldName)) { SortOrder esSortOrder = getEsSortOrder(el); - sortFields.add(new FieldSortBuilder(esSortFieldName).order(esSortOrder)); + + SortOptions sortFieldOptions = + new Builder() + .field(new FieldSort.Builder() + .field(esSortFieldName) + .order(esSortOrder).build()) + .build(); + + sortFields.add(sortFieldOptions); } } @@ -72,11 +81,13 @@ public List> parseSortBy(Element request) { return sortFields; } - private List> getDefaultSort() { - List> sortFields = new ArrayList<>(); - FieldSortBuilder defaultSortField = - new FieldSortBuilder(_catalogConfig.getDefaultSortField()) - .order(SortOrder.fromString(_catalogConfig.getDefaultSortOrder())); + private List getDefaultSort() { + List sortFields = new ArrayList<>(); + + SortOptions defaultSortField = SortOptions.of( + b -> b.field(fb -> fb.field(_catalogConfig.getDefaultSortField()) + .order(_catalogConfig.getDefaultSortOrder().equals("DESC")?SortOrder.Desc:SortOrder.Asc)) + ); sortFields.add(defaultSortField); return sortFields; @@ -97,6 +108,6 @@ private String getEsSortFieldName(Element el) { private SortOrder getEsSortOrder(Element el) { String order = el.getChildText("SortOrder", Csw.NAMESPACE_OGC); boolean isDescOrder = "DESC".equals(order); - return isDescOrder ? SortOrder.DESC : SortOrder.ASC; + return isDescOrder ? SortOrder.Desc : SortOrder.Asc; } } diff --git a/csw-server/src/test/java/org/fao/geonet/kernel/csw/services/getrecords/es/CswSortBy2EsTest.java b/csw-server/src/test/java/org/fao/geonet/kernel/csw/services/getrecords/es/CswSortBy2EsTest.java index e233ed945a80..0d1775995466 100644 --- a/csw-server/src/test/java/org/fao/geonet/kernel/csw/services/getrecords/es/CswSortBy2EsTest.java +++ b/csw-server/src/test/java/org/fao/geonet/kernel/csw/services/getrecords/es/CswSortBy2EsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2021 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -25,9 +25,7 @@ import static junit.framework.TestCase.assertEquals; -import org.checkerframework.checker.units.qual.A; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; +import co.elastic.clients.elasticsearch._types.SortOptions; import org.fao.geonet.constants.Geonet; import org.fao.geonet.csw.common.Csw; import org.fao.geonet.kernel.csw.CatalogConfiguration; @@ -70,12 +68,12 @@ void configureDefaultSort() { Element request = createSortByBaseRequest( new Element("Empty", Geonet.Namespaces.OGC)); - List> sortFields = toTest.parseSortBy(request); + List sortFields = toTest.parseSortBy(request); assertEquals(1, sortFields.size()); - FieldSortBuilder sortField = (FieldSortBuilder)sortFields.get(0); - assertEquals(sortField.getFieldName(), "resourceTitleObject.default.keyword"); - assertEquals(sortField.order().toString(), "desc"); + SortOptions sortField = sortFields.get(0); + assertEquals(sortField.field().field(), "resourceTitleObject.default.keyword"); + assertEquals(sortField.field().order().jsonValue(), "desc"); } @Test @@ -86,12 +84,12 @@ void sortByRelevanceDESC() { .addContent(new Element("PropertyName", Geonet.Namespaces.OGC).setText("Relevance")) .addContent(new Element("SortOrder", Geonet.Namespaces.OGC).setText("DESC")))); - List> sortFields = toTest.parseSortBy(request); + List sortFields = toTest.parseSortBy(request); assertEquals(1, sortFields.size()); - FieldSortBuilder sortField = (FieldSortBuilder)sortFields.get(0); - assertEquals(sortField.getFieldName(), "_score"); - assertEquals(sortField.order().toString(), "desc"); + SortOptions sortField = sortFields.get(0); + assertEquals(sortField.field().field(), "_score"); + assertEquals(sortField.field().order().jsonValue(), "desc"); } @Test @@ -102,12 +100,12 @@ void sortByRelevanceASC() { .addContent(new Element("PropertyName", Geonet.Namespaces.OGC).setText("Relevance")) .addContent(new Element("SortOrder", Geonet.Namespaces.OGC).setText("ASC")))); - List> sortFields = toTest.parseSortBy(request); + List sortFields = toTest.parseSortBy(request); assertEquals(1, sortFields.size()); - FieldSortBuilder sortField = (FieldSortBuilder)sortFields.get(0); - assertEquals(sortField.getFieldName(), "_score"); - assertEquals(sortField.order().toString(), "asc"); + SortOptions sortField = sortFields.get(0); + assertEquals(sortField.field().field(), "_score"); + assertEquals(sortField.field().order().jsonValue(), "asc"); } @Test @@ -118,12 +116,12 @@ void sortByIndexField() { .addContent(new Element("PropertyName", Geonet.Namespaces.OGC).setText("title")) .addContent(new Element("SortOrder", Geonet.Namespaces.OGC).setText("DESC")))); - List> sortFields = toTest.parseSortBy(request); + List sortFields = toTest.parseSortBy(request); assertEquals(1, sortFields.size()); - FieldSortBuilder sortField = (FieldSortBuilder)sortFields.get(0); - assertEquals(sortField.getFieldName(), "title"); - assertEquals(sortField.order().toString(), "desc"); + SortOptions sortField = sortFields.get(0); + assertEquals(sortField.field().field(), "title"); + assertEquals(sortField.field().order().jsonValue(), "desc"); } @Test @@ -137,15 +135,15 @@ void sortByMultipleProperties() { .addContent(new Element("PropertyName", Geonet.Namespaces.OGC).setText("Relevance")) .addContent(new Element("SortOrder", Geonet.Namespaces.OGC).setText("DESC")))); - List> sortFields = toTest.parseSortBy(request); + List sortFields = toTest.parseSortBy(request); assertEquals(2, sortFields.size()); - FieldSortBuilder sortField = (FieldSortBuilder)sortFields.get(0); - assertEquals(sortField.getFieldName(), "title"); - assertEquals(sortField.order().toString(), "desc"); - FieldSortBuilder sortField2 = (FieldSortBuilder)sortFields.get(1); - assertEquals(sortField2.getFieldName(), "_score"); - assertEquals(sortField2.order().toString(), "desc"); + SortOptions sortField = sortFields.get(0); + assertEquals(sortField.field().field(), "title"); + assertEquals(sortField.field().order().jsonValue(), "desc"); + SortOptions sortField2 = sortFields.get(1); + assertEquals(sortField2.field().field(), "_score"); + assertEquals(sortField2.field().order().jsonValue(), "desc"); } private Element createSortByBaseRequest(Element SortBy) { diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 3efac1ba79c5..7781a1f37ccb 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -23,12 +23,12 @@ package org.fao.geonet.kernel.harvest.harvester.csw; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; import org.fao.geonet.GeonetContext; import org.fao.geonet.Logger; import org.fao.geonet.constants.Geonet; @@ -672,8 +672,8 @@ private Set retrieveMetadataUuidsFromIdentifier(EsSearchManager searchMa FIELDLIST_UUID, 0, 1000); - for (SearchHit hit : queryResult.getHits()) { - String uuid = hit.getSourceAsMap().get(Geonet.IndexFieldNames.UUID).toString(); + for (Hit hit : (List) queryResult.hits().hits()) { + String uuid = objectMapper.convertValue(hit.source(), Map.class).get(Geonet.IndexFieldNames.UUID).toString(); metadataUuids.add(uuid); } diff --git a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexHealthCheck.java b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexHealthCheck.java index 14b3f44db41c..91d9823e9980 100644 --- a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexHealthCheck.java +++ b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexHealthCheck.java @@ -23,14 +23,12 @@ package org.fao.geonet.monitor.health; +import co.elastic.clients.elasticsearch.core.SearchResponse; import com.yammer.metrics.core.HealthCheck; import jeeves.monitor.HealthCheckFactory; import jeeves.server.context.ServiceContext; -import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.setting.SettingInfo; -import org.openrdf.sesame.sail.query.In; import org.springframework.context.ApplicationContext; /** @@ -45,17 +43,12 @@ protected Result check() { ApplicationContext applicationContext = ApplicationContextHolder.get(); EsSearchManager searchMan = applicationContext.getBean(EsSearchManager.class); final SearchResponse result = searchMan.query("*", null, 0, 0); - if (result.status().getStatus() == 200) { - return Result.healthy(String.format( - "%s records indexed in remote index currently.", - result.getHits().getTotalHits().value - )); - } else { - return Result.unhealthy( - "Index storing records is not available currently. " + - "This component is required. Check your installation."); - } - } catch (Throwable e) { + + return Result.healthy(String.format( + "%s records indexed in remote index currently.", + result.hits().hits().size() + )); + } catch (Exception e) { return Result.unhealthy(e); } } diff --git a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java index cc0d104f4857..467f05e4f727 100644 --- a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java +++ b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -23,13 +23,11 @@ package org.fao.geonet.monitor.health; +import co.elastic.clients.elasticsearch.indices.GetIndicesSettingsRequest; +import co.elastic.clients.elasticsearch.indices.GetIndicesSettingsResponse; import com.yammer.metrics.core.HealthCheck; import jeeves.monitor.HealthCheckFactory; import jeeves.server.context.ServiceContext; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.kernel.search.EsSearchManager; import org.springframework.context.ApplicationContext; @@ -54,12 +52,16 @@ protected Result check() { ApplicationContext applicationContext = ApplicationContextHolder.get(); EsSearchManager searchMan = applicationContext.getBean(EsSearchManager.class); String indexBlockRead = "index.blocks.read_only_allow_delete"; - GetSettingsRequest request = new GetSettingsRequest(); - request.names(indexBlockRead); - GetSettingsResponse settings = searchMan.getClient().getClient() - .indices().getSettings(request, RequestOptions.DEFAULT); - Boolean isReadOnly = "true".equals(settings.getSetting(searchMan.getDefaultIndex(), indexBlockRead)); + GetIndicesSettingsRequest request = GetIndicesSettingsRequest.of( + b -> b.index(searchMan.getDefaultIndex()) + .name(indexBlockRead) + ); + + GetIndicesSettingsResponse settings = searchMan.getClient().getClient() + .indices().getSettings(request); + + Boolean isReadOnly = "true".equals(settings.get(indexBlockRead).toString()); if (!isReadOnly) { return Result.healthy(String.format( diff --git a/index/pom.xml b/index/pom.xml index 317a0e2ccc77..342df610e78a 100644 --- a/index/pom.xml +++ b/index/pom.xml @@ -33,8 +33,8 @@ transport --> - org.elasticsearch.client - elasticsearch-rest-high-level-client + co.elastic.clients + elasticsearch-java org.springframework diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index 9837d551f128..f0dc77005548 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -23,7 +23,27 @@ package org.fao.geonet.index.es; +import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.*; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.WrapperQuery; +import co.elastic.clients.elasticsearch.cluster.HealthResponse; +import co.elastic.clients.elasticsearch.core.*; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.indices.AnalyzeRequest; +import co.elastic.clients.elasticsearch.indices.AnalyzeResponse; +import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.json.spi.JsonProvider; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -37,33 +57,8 @@ import org.apache.http.nio.conn.SchemeIOSessionStrategy; import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContextBuilder; -import org.apache.lucene.search.TotalHits; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.*; -import org.elasticsearch.client.core.MainResponse; -import org.elasticsearch.client.indices.AnalyzeRequest; -import org.elasticsearch.client.indices.AnalyzeResponse; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; -import org.elasticsearch.script.Script; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; @@ -71,6 +66,7 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import java.io.IOException; +import java.io.StringReader; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.*; @@ -82,7 +78,9 @@ public class EsRestClient implements InitializingBean { private static EsRestClient instance; - private RestHighLevelClient client; + private ElasticsearchClient client; + + private ElasticsearchAsyncClient asyncClient; @Value("${es.url}") private String serverUrl; @@ -108,10 +106,14 @@ public static EsRestClient get() { return instance; } - public RestHighLevelClient getClient() { + public ElasticsearchClient getClient() { return client; } + public ElasticsearchAsyncClient getAsynchClient() { + return asyncClient; + } + public String getDashboardAppUrl() { return dashboardAppUrl; } @@ -159,7 +161,14 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } } - client = new RestHighLevelClient(builder); + + RestClient restClient = builder.build(); + + ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); + + client = new ElasticsearchClient(transport); + + asyncClient = new ElasticsearchAsyncClient(transport); synchronized (EsRestClient.class) { instance = this; @@ -206,20 +215,29 @@ public BulkResponse bulkRequest(String index, Map docs) throws I throw new IOException("Index not yet activated."); } - BulkRequest request = new BulkRequest(); - request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() + .index(index) + .refresh(Refresh.True); + + JsonpMapper jsonpMapper = client._transport().jsonpMapper(); + JsonProvider jsonProvider = jsonpMapper.jsonProvider(); + Iterator> iterator = docs.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); - request.add(new IndexRequest(index).id(entry.getKey()) - .source(entry.getValue(), XContentType.JSON)); - // https://www.elastic.co/fr/blog/customizing-your-document-routing - // For features & record search we need to route the record - // document to the same place as the features in order to make join -// .routing(ROUTING_KEY)); + + JsonData jd = JsonData.from(jsonProvider.createParser(new StringReader(entry.getValue())), jsonpMapper); + + requestBuilder + .operations(op -> op.index(idx -> idx.index(index) + .id(entry.getKey()) + .document(jd))); } + + BulkRequest request = requestBuilder.build(); + try { - return client.bulk(request, RequestOptions.DEFAULT); + return client.bulk(request); } catch (IOException e) { e.printStackTrace(); throw e; @@ -254,86 +272,92 @@ public SearchResponse query(String index, String luceneQuery, String filterQuery public SearchResponse query(String index, String luceneQuery, String filterQuery, Set includedFields, Map scriptedFields, - int from, int size, List> sort) throws Exception { - final QueryBuilder query = QueryBuilders.queryStringQuery(luceneQuery); - QueryBuilder filter = null; + int from, int size, List sort) throws Exception { + + Query.Builder queryBuilder = new Query.Builder(); + queryBuilder.queryString(new QueryStringQuery.Builder().query(luceneQuery).build()); + + Query.Builder filterBuilder = null; if (StringUtils.isNotEmpty(filterQuery)) { - filter = QueryBuilders.queryStringQuery(filterQuery); + filterBuilder = new Query.Builder(); + filterBuilder.queryString(new QueryStringQuery.Builder().query(filterQuery).build()); } - return query(index, query, filter, includedFields, scriptedFields, from, size, sort); + return query(index, queryBuilder, filterBuilder, includedFields, scriptedFields, from, size, sort); } /** * Query using JSON elastic query */ - public SearchResponse query(String index, JsonNode jsonQuery, QueryBuilder postFilterBuilder, + public SearchResponse query(String index, JsonNode jsonQuery, Query.Builder postFilterBuilder, Set includedFields, Map scriptedFields, int from, int size) throws Exception { return query(index, jsonQuery, postFilterBuilder, includedFields, scriptedFields, from, size, null); } - public SearchResponse query(String index, JsonNode jsonQuery, QueryBuilder postFilterBuilder, Set includedFields, + public SearchResponse query(String index, JsonNode jsonQuery, Query.Builder postFilterBuilder, Set includedFields, int from, int size) throws Exception { return query(index, jsonQuery, postFilterBuilder, includedFields, new HashMap<>(), from, size, null); } - public SearchResponse query(String index, JsonNode jsonQuery, QueryBuilder postFilterBuilder, + public SearchResponse query(String index, JsonNode jsonQuery, Query.Builder postFilterBuilder, Set includedFields, Map scriptedFields, - int from, int size, List> sort) throws Exception { - final QueryBuilder query = QueryBuilders.wrapperQuery(String.valueOf(jsonQuery)); + int from, int size, List sort) throws Exception { + final Query.Builder query = new Query.Builder(); + + WrapperQuery.Builder wrapperQueryBuilder = new WrapperQuery.Builder(); + wrapperQueryBuilder.query(String.valueOf(jsonQuery)); + query.wrapper(wrapperQueryBuilder.build()); + return query(index, query, postFilterBuilder, includedFields, scriptedFields, from, size, sort); } - public SearchResponse query(String index, QueryBuilder queryBuilder, QueryBuilder postFilterBuilder, + public SearchResponse query(String index, Query.Builder queryBuilder, Query.Builder postFilterBuilder, Set includedFields, Map scriptedFields, - int from, int size, List> sort) throws Exception { + int from, int size, List sort) throws Exception { if (!activated) { return null; } // TODOES: Add permission if index is gn-records // See EsHTTPProxy#addUserInfo - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(index); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(queryBuilder); + SearchRequest.Builder searchRequestBuilder = new SearchRequest.Builder() + .index(index) + .from(from) + .size(size) + .query(queryBuilder.build()) + .trackTotalHits(th -> th.enabled(true)) + .source(sc -> sc.filter(f -> f.includes(new ArrayList<>(includedFields)))); + + if (postFilterBuilder != null) { + searchRequestBuilder.postFilter(postFilterBuilder.build()); + } if (MapUtils.isNotEmpty(scriptedFields)) { for (Map.Entry scriptedField: scriptedFields.entrySet()) { - searchSourceBuilder.scriptField(scriptedField.getKey(), new Script(scriptedField.getValue())); - } - } + ScriptField scriptField = ScriptField.of( + b -> b.script(sb -> sb.inline(is -> is.source(scriptedField.getValue()))) + ); - searchSourceBuilder.fetchSource(includedFields.toArray(new String[includedFields.size()]), null); - searchSourceBuilder.from(from); - searchSourceBuilder.size(size); - searchSourceBuilder.trackTotalHits(true); - if (postFilterBuilder != null) { - searchSourceBuilder.postFilter(postFilterBuilder); + searchRequestBuilder.scriptFields(scriptedField.getKey(), scriptField); + } } - if ((sort != null) && (!sort.isEmpty())) { - sort.forEach(searchSourceBuilder::sort); + if (sort != null) { + searchRequestBuilder.sort(sort); } - searchRequest.source(searchSourceBuilder); + SearchRequest searchRequest = searchRequestBuilder.build(); try { - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - if (searchResponse.status().getStatus() == 200) { - return searchResponse; - } else { - throw new IOException(String.format( - "Error during querying index. Errors is '%s'.", searchResponse.status().toString() - )); - } - } catch (ElasticsearchStatusException esException) { - Throwable[] suppressed = esException.getSuppressed(); + return client.search(searchRequest, ObjectNode.class); + + } catch (ElasticsearchException esException) { + /*Throwable[] suppressed = esException.getSuppressed(); if (suppressed.length > 0 && suppressed[0] instanceof ResponseException) { ResponseException re = (ResponseException) suppressed[0]; Log.error("geonetwork.index", String.format( "Error during querying index. %s", re.getMessage())); - } + }*/ throw esException; } } @@ -344,19 +368,24 @@ public String deleteByQuery(String index, String query) throws Exception { return ""; } - DeleteByQueryRequest request = new DeleteByQueryRequest(); - request.setRefresh(true); - request.indices(index); - request.setQuery(new QueryStringQueryBuilder(query)); + DeleteByQueryRequest request = DeleteByQueryRequest.of( + b -> b.index(new ArrayList<>(Arrays.asList(index))) + .q(query) + .refresh(true)); - final BulkByScrollResponse deleteByQueryResponse = - client.deleteByQuery(request, RequestOptions.DEFAULT); + final DeleteByQueryResponse deleteByQueryResponse = + client.deleteByQuery(request); - if (deleteByQueryResponse.getStatus().getDeleted() >= 0) { - return String.format("Record removed. %s.", deleteByQueryResponse.getStatus().getDeleted()); + + if (deleteByQueryResponse.deleted() >= 0) { + return String.format("Record removed. %s.", deleteByQueryResponse.deleted()); } else { + StringBuilder stringBuilder = new StringBuilder(); + + deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); + throw new IOException(String.format( - "Error during removal. Errors is '%s'.", deleteByQueryResponse.getStatus().getReasonCancelled() + "Error during removal. Errors are '%s'.", stringBuilder.toString() )); } } @@ -370,9 +399,19 @@ public Map getDocument(String index, String id) throws Exception if (!activated) { return Collections.emptyMap(); } - GetRequest request = new GetRequest().index(index).id(id); - return client.get(request, RequestOptions.DEFAULT).getSourceAsMap(); + GetRequest request = GetRequest.of( + b -> b.index(index).id(id) + ); + + GetResponse response = client.get(request, ObjectNode.class); + + if (response.found()) { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.convertValue(response.source(), Map.class); + } else { + throw new Exception(String.format("Document with id %s not found", id)); + } } /** @@ -388,35 +427,33 @@ public Map getFieldsValues(String index, String id, Set String query = String.format("_id:\"%s\"", id); // TODO: Check maxRecords // TODO: Use _doc API? + + final SearchResponse searchResponse = this.query(index, query, null, fields, new HashMap<>(), 0, 1, null); - if (searchResponse.status().getStatus() == 200) { - TotalHits totalHits = searchResponse.getHits().getTotalHits(); - long matches = totalHits == null ? -1 : totalHits.value; - if (matches == 0) { - return fieldValues; - } else if (matches == 1) { - final SearchHit[] hits = searchResponse.getHits().getHits(); - - fields.forEach(f -> { - final Object o = hits[0].getSourceAsMap().get(f); - if (o instanceof String) { - fieldValues.put(f, (String) o); - } else if (o instanceof HashMap && f.endsWith("Object")) { - fieldValues.put(f, (String) ((HashMap) o).get("default")); - } - }); - } else { - throw new IOException(String.format( - "Your query '%s' returned more than one record, %d in fact. Can't retrieve field values for more than one record.", - query, - matches - )); - } + + List totalHits = searchResponse.hits().hits(); + long matches = totalHits.size(); + if (matches == 0) { + return fieldValues; + } else if (matches == 1) { + final Hit hit = totalHits.get(0); + + fields.forEach(f -> { + final Object o = hit.fields().get(f); + if (o instanceof String) { + fieldValues.put(f, (String) o); + } else if (o instanceof HashMap && f.endsWith("Object")) { + fieldValues.put(f, (String) ((HashMap) o).get("default")); + } + }); } else { throw new IOException(String.format( - "Error during fields value retrieval. Status is '%s'.", searchResponse.status().getStatus() + "Your query '%s' returned more than one record, %d in fact. Can't retrieve field values for more than one record.", + query, + matches )); } + } catch (Exception e) { throw new IOException(String.format( "Error during fields value retrieval. Errors is '%s'.", e.getMessage() @@ -464,19 +501,21 @@ public static String analyzeField(String collection, String analyzer, String fieldValue) { - AnalyzeRequest request = AnalyzeRequest.withIndexAnalyzer(collection, - analyzer, - // Replace , as it is meaningful in synonym map format - fieldValue.replace(",", "")); + AnalyzeRequest analyzeRequest = AnalyzeRequest.of( + b -> b.index(collection) + .analyzer(analyzer) + // Replace , as it is meaningful in synonym map format + .text(fieldValue.replace(",", "")) + ); try { - AnalyzeResponse response = EsRestClient.get().client.indices().analyze(request, RequestOptions.DEFAULT); + AnalyzeResponse response = EsRestClient.get().client.indices().analyze(analyzeRequest); - final List tokens = response.getTokens(); + final List tokens = response.tokens(); if (tokens.size() == 1) { - final String type = tokens.get(0).getType(); + final String type = tokens.get(0).type(); if ("SYNONYM".equals(type) || "word".equals(type)) { - return tokens.get(0).getTerm(); + return tokens.get(0).token(); } return ""; } else { @@ -487,27 +526,16 @@ public static String analyzeField(String collection, } } - - protected void finalize() { - try { - client.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - // TODO: check index exist too public String getServerStatus() throws IOException { - ClusterHealthRequest request = new ClusterHealthRequest(); - ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); - return response.getStatus().toString(); -// return getClient().ping(RequestOptions.DEFAULT); + HealthResponse response = client.cluster().health(); + return response.status().toString(); } public String getServerVersion() throws IOException, ElasticsearchException { - MainResponse.Version version = client.info(RequestOptions.DEFAULT).getVersion(); + ElasticsearchVersionInfo version = client.info().version(); - return version.getNumber(); + return version.number(); } } diff --git a/inspire-atom/pom.xml b/inspire-atom/pom.xml index 0b9c800c5d66..e70d1dda4134 100644 --- a/inspire-atom/pom.xml +++ b/inspire-atom/pom.xml @@ -93,6 +93,11 @@ h2 test + + + co.elastic.clients + elasticsearch-java + diff --git a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/util/InspireAtomUtil.java b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/util/InspireAtomUtil.java index 6c6980b6f8c1..a452d0733d0e 100644 --- a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/util/InspireAtomUtil.java +++ b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/util/InspireAtomUtil.java @@ -22,13 +22,13 @@ //============================================================================== package org.fao.geonet.inspireatom.util; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.TotalHits; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.search.TotalHits; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; import org.fao.geonet.GeonetContext; import org.fao.geonet.api.exception.ResourceNotFoundException; import org.fao.geonet.constants.Geonet; @@ -372,8 +372,8 @@ public static List searchMetadataByTypeAndProtocol(ServiceCont 0, 10000); IMetadataUtils dataManager = context.getBean(IMetadataUtils.class); - for (SearchHit hit : result.getHits()) { - String id = hit.getSourceAsMap().get(Geonet.IndexFieldNames.ID).toString(); + for (Hit hit : (List) result.hits().hits()) { + String id = objectMapper.convertValue(hit.source(), Map.class).get(Geonet.IndexFieldNames.ID).toString(); allMdInfo.add(dataManager.findOne(id)); } } catch (Exception ex) { @@ -409,10 +409,10 @@ public static String retrieveDatasetUuidFromIdentifier(EsSearchManager searchMan FIELDLIST_CORE, 0, 1); - TotalHits totalHits = result.getHits().getTotalHits(); + TotalHits totalHits = result.hits().total(); - if ((totalHits != null) && (totalHits.value > 0)) { - id = result.getHits().getAt(0).getId(); + if ((totalHits != null) && (totalHits.value() > 0)) { + id = ((Hit) result.hits().hits().get(0)).id(); } } catch (Exception ex) { Log.error(Geonet.ATOM, ex.getMessage(), ex); @@ -552,8 +552,8 @@ public static Element getMetadataFeedByResourceIdentifier(final ServiceContext c esJsonQuery, FIELDLIST_CORE, 0, 1); - for (SearchHit hit : result.getHits()) { - datasetMd = repo.findOneByUuid(hit.getId()); + for (Hit hit : (List) result.hits().hits()) { + datasetMd = repo.findOneByUuid(hit.id()); } } catch (Exception e) { @@ -600,8 +600,8 @@ public static Element getMetadataFeedByResourceIdentifier(final ServiceContext c esJsonQuery, FIELDLIST_CORE, 0, 1); - for (SearchHit hit : result.getHits()) { - serviceMetadata = repo.findOneByUuid(hit.getId()); + for (Hit hit : (List) result.hits().hits()) { + serviceMetadata = repo.findOneByUuid(hit.id()); } } catch (Exception e) { diff --git a/inspire-atom/src/main/java/org/fao/geonet/services/inspireatom/AtomSearch.java b/inspire-atom/src/main/java/org/fao/geonet/services/inspireatom/AtomSearch.java index b88a509687c9..c5a19f656718 100644 --- a/inspire-atom/src/main/java/org/fao/geonet/services/inspireatom/AtomSearch.java +++ b/inspire-atom/src/main/java/org/fao/geonet/services/inspireatom/AtomSearch.java @@ -22,6 +22,8 @@ //============================================================================== package org.fao.geonet.services.inspireatom; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.Parameter; @@ -30,8 +32,6 @@ import io.swagger.v3.oas.annotations.tags.Tag; import jeeves.server.context.ServiceContext; import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; import org.fao.geonet.api.ApiUtils; import org.fao.geonet.api.tools.i18n.LanguageUtils; import org.fao.geonet.constants.Geonet; @@ -61,6 +61,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Map; import static org.fao.geonet.kernel.search.EsFilterBuilder.buildPermissionsFilter; import static org.fao.geonet.kernel.search.EsSearchManager.FIELDLIST_CORE; @@ -176,14 +177,14 @@ public Element feeds( // Loop over the results and retrieve feeds to add in results // First element in results (pos=0) is the summary, ignore it - for (SearchHit hit : result.getHits().getHits()) { - String id = hit.getSourceAsMap().get(Geonet.IndexFieldNames.ID).toString(); + for (Hit hit : (List) result.hits().hits()) { + String id = objectMapper.convertValue(hit.source(), Map.class).get(Geonet.IndexFieldNames.ID).toString(); InspireAtomFeed feed = service.findByMetadataId(Integer.parseInt(id)); if (feed != null) { Element feedEl = Xml.loadString(feed.getAtom(), false); feeds.addContent((Content) feedEl.clone()); } else { - Log.debug(Geonet.ATOM, String.format("No feed available for %s", hit.getId())); + Log.debug(Geonet.ATOM, String.format("No feed available for %s", hit.id())); } } return feeds; diff --git a/pom.xml b/pom.xml index 0e763691e030..07a49161f065 100644 --- a/pom.xml +++ b/pom.xml @@ -1192,9 +1192,10 @@ jaxb-impl 2.3.1 + - org.elasticsearch.client - elasticsearch-rest-high-level-client + co.elastic.clients + elasticsearch-java ${es.version} @@ -1521,7 +1522,7 @@ 8080 8090 - 7.15.1 + 8.4.2 linux-x86_64 tar.gz http diff --git a/services/pom.xml b/services/pom.xml index d5f15b37c60e..b278bab12745 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -222,6 +222,11 @@ org.apache.commons commons-csv + + + co.elastic.clients + elasticsearch-java + diff --git a/services/src/main/java/org/fao/geonet/api/records/CatalogApi.java b/services/src/main/java/org/fao/geonet/api/records/CatalogApi.java index 77f6b2833ea8..56ad6f25b3f9 100644 --- a/services/src/main/java/org/fao/geonet/api/records/CatalogApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/CatalogApi.java @@ -23,6 +23,9 @@ package org.fao.geonet.api.records; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameters; @@ -42,7 +45,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.text.StrSubstitutor; -import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.api.API; import org.fao.geonet.api.ApiParams; @@ -401,9 +403,11 @@ public void exportAsPdf( }); Element response = new Element("response"); - Arrays.asList(searchResponse.getHits().getHits()).forEach(h -> { + ObjectMapper objectMapper = new ObjectMapper(); + searchResponse.hits().hits().forEach(h1 -> { + Hit h = (Hit) h1; Element r = new Element("metadata"); - final Map source = h.getSourceAsMap(); + final Map source = objectMapper.convertValue(h.source(), Map.class); source.entrySet().forEach(e -> { Object v = e.getValue(); if (v instanceof String) { @@ -548,8 +552,9 @@ public void exportAsCsv( EsFilterBuilder.buildPermissionsFilter(ApiUtils.createServiceContext(httpRequest)), FIELDLIST_CORE, 0, maxhits); - List idsToExport = Arrays.stream(searchResponse.getHits().getHits()) - .map(h -> (String) h.getSourceAsMap().get("id")) + ObjectMapper objectMapper = new ObjectMapper(); + List idsToExport = (List) searchResponse.hits().hits().stream() + .map(h -> (String) objectMapper.convertValue(((Hit) h).source(), Map.class).get("id")) .collect(Collectors.toList()); // Determine filename to use diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java index 89ec9b7ab0f0..17fcff7bb331 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java @@ -23,6 +23,9 @@ package org.fao.geonet.api.records; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.json.JsonData; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -31,9 +34,6 @@ import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; import org.apache.commons.text.StringEscapeUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.search.SearchHit; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.GeonetContext; import org.fao.geonet.NodeInfo; @@ -294,29 +294,32 @@ public static Map> getAssociated( Set remoteRecords = relatedTypeDetails.getRemoteRecords(); List records = new ArrayList<>(); - if (result.getHits().getTotalHits().value > 0) { - for (SearchHit e : Arrays.asList(result.getHits().getHits())) { - allCatalogueUuids.add(e.getId()); + if (result.hits().hits().size() > 0) { + for (Hit e : (List) result.hits().hits()) { + allCatalogueUuids.add(e.id()); AssociatedRecord associatedRecord = new AssociatedRecord(); - associatedRecord.setUuid(e.getId()); + associatedRecord.setUuid(e.id()); // Set properties eg. remote, associationType, ... - associatedRecord.setProperties(relatedTypeDetails.recordsProperties.get(e.getId())); + associatedRecord.setProperties(relatedTypeDetails.recordsProperties.get(e.id())); // Add scripted field values to the properties of the record - if (!e.getFields().isEmpty()) { + if (!e.fields().isEmpty()) { FIELDLIST_RELATED_SCRIPTED.keySet().forEach(f -> { - DocumentField dc = e.getFields().get(f); + JsonData dc = (JsonData) e.fields().get(f); - if (dc != null) { + // TODO: ES 8 + /*if (dc != null) { if (associatedRecord.getProperties() == null) { associatedRecord.setProperties(new HashMap<>()); } associatedRecord.getProperties().put(dc.getName(), dc.getValue()); - } + }*/ }); } - JsonNode source = mapper.readTree(e.getSourceAsString()); + // TODO: ES 8 Check conversion class + JsonNode source = mapper.convertValue(e.source(), JsonNode.class); + //JsonNode source = mapper.read(e.getSourceAsString()); ObjectNode doc = mapper.createObjectNode(); doc.set("_source", source); EsHTTPProxy.addUserInfo(doc, context); @@ -330,12 +333,12 @@ public static Map> getAssociated( associatedRecord.setRecord(source); associatedRecord.setOrigin(RelatedItemOrigin.catalog.name()); records.add(associatedRecord); - if (expectedUuids.contains(e.getId())) { - expectedUuids.remove(e.getId()); + if (expectedUuids.contains(e.id())) { + expectedUuids.remove(e.id()); } // Remote records may be found in current catalogue (eg. if harvested) - if (remoteRecords.contains(e.getId())) { - remoteRecords.remove(e.getId()); + if (remoteRecords.contains(e.id())) { + remoteRecords.remove(e.id()); } } } @@ -382,9 +385,9 @@ private static void assignPortalOrigin(int start, int size, EsSearchManager sear start, size); Set allPortalUuids = new HashSet<>(); - if (recordsInPortal.getHits().getTotalHits().value > 0) { - for (SearchHit e : Arrays.asList(recordsInPortal.getHits().getHits())) { - allPortalUuids.add(e.getId()); + if (recordsInPortal.hits().hits().size() > 0) { + for (Hit e : (List) recordsInPortal.hits().hits()) { + allPortalUuids.add(e.id()); } } @@ -662,12 +665,17 @@ private static Element search(String uuidQueryValue, String type, String from, S fromValue, (toValue - fromValue)); Element typeResponse = new Element(type.equals("brothersAndSisters") ? "siblings" : type); - if (result.getHits().getTotalHits().value > 0) { + if (result.hits().hits().size() > 0) { // Build the old search service response format Element response = new Element("response"); - Arrays.asList(result.getHits().getHits()).forEach(e -> { + + ObjectMapper objectMapper = new ObjectMapper(); + + result.hits().hits().forEach(e1 -> { + Hit e = (Hit) e1; + Element recordMetadata = new Element("metadata"); - final Map source = e.getSourceAsMap(); + final Map source = objectMapper.convertValue(e.source(), Map.class); recordMetadata.addContent(new Element("id").setText((String) source.get(Geonet.IndexFieldNames.ID))); recordMetadata.addContent(new Element("uuid").setText((String) source.get(Geonet.IndexFieldNames.UUID))); if (type.equals("brothersAndSisters")) { @@ -718,9 +726,10 @@ public static Set getUuidsToExport(String query) throws Exception { int size = Integer.parseInt(si.getSelectionMaxRecords()); final SearchResponse result = searchMan.query(query, null, from, size); - if (result.getHits().getTotalHits().value > 0) { - final SearchHit[] elements = result.getHits().getHits(); - Arrays.asList(elements).forEach(e -> uuids.add((String) e.getSourceAsMap().get(Geonet.IndexFieldNames.UUID))); + if (result.hits().hits().size() > 0) { + final List elements = result.hits().hits(); + ObjectMapper objectMapper = new ObjectMapper(); + elements.forEach(e -> uuids.add((String) objectMapper.convertValue(e.source(), Map.class).get(Geonet.IndexFieldNames.UUID))); } Log.info(Geonet.MEF, " Found " + uuids.size() + " record(s)."); return uuids; diff --git a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java index c4983b337829..4004000ef2bf 100644 --- a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java +++ b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java @@ -1,11 +1,12 @@ package org.fao.geonet.api.registries; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.google.common.collect.Table; import jeeves.server.ServiceConfig; import jeeves.server.context.ServiceContext; import jeeves.xlink.XLink; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.domain.*; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.UpdateDatestamp; @@ -408,8 +409,8 @@ private static String search(ServiceContext context, Map searchP try { SearchResponse results = searchMan.query(query.toString(), null, 0, 1); - if (results.getHits().getTotalHits().value > 0) { - return results.getHits().getHits()[0].getId(); + if (results.hits().hits().size() > 0) { + return ((Hit) results.hits().hits().get(0)).id(); } } catch (Exception e) { e.printStackTrace(); diff --git a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java index 00f478966be1..005b8de07644 100644 --- a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java +++ b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java @@ -23,6 +23,8 @@ package org.fao.geonet.api.site; +import co.elastic.clients.elasticsearch.core.CountRequest; +import co.elastic.clients.elasticsearch.core.CountResponse; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -34,8 +36,6 @@ import jeeves.server.context.ServiceContext; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.client.core.CountResponse; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.GeonetContext; import org.fao.geonet.NodeInfo; @@ -676,9 +676,9 @@ public Map indexAndDbSynchronizationStatus( EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); CountResponse countResponse = esRestClient.getClient().count( - new CountRequest(searchMan.getDefaultIndex()), - RequestOptions.DEFAULT); - infoIndexDbSynch.put("index.count", countResponse.getCount()); + CountRequest.of(b -> b.index(searchMan.getDefaultIndex())) + ); + infoIndexDbSynch.put("index.count", countResponse.count()); return infoIndexDbSynch; } diff --git a/services/src/main/java/org/fao/geonet/api/site/SiteInformation.java b/services/src/main/java/org/fao/geonet/api/site/SiteInformation.java index 89a930aec059..2361c39d1086 100644 --- a/services/src/main/java/org/fao/geonet/api/site/SiteInformation.java +++ b/services/src/main/java/org/fao/geonet/api/site/SiteInformation.java @@ -23,12 +23,12 @@ package org.fao.geonet.api.site; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; import com.fasterxml.jackson.annotation.JsonProperty; import jeeves.server.ServiceConfig; import jeeves.server.context.ServiceContext; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.ElasticsearchException; import org.fao.geonet.GeonetContext; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.search.EsSearchManager; diff --git a/web/src/main/webResources/WEB-INF/data/config/index/records.json b/web/src/main/webResources/WEB-INF/data/config/index/records.json index 743392a0cf93..413753e2effd 100644 --- a/web/src/main/webResources/WEB-INF/data/config/index/records.json +++ b/web/src/main/webResources/WEB-INF/data/config/index/records.json @@ -8,6 +8,7 @@ "analysis": { "normalizer": { "sorting": { + "type": "custom", "filter": ["lowercase", "asciifolding"] } }, @@ -23,6 +24,7 @@ "filter": ["reverse"] }, "french_rebuilt": { + "type": "custom", "tokenizer": "standard", "filter": [ "lowercase", @@ -35,6 +37,7 @@ ] }, "english_rebuilt": { + "type": "custom", "tokenizer": "standard", "filter": [ "english_possessive_stemmer", @@ -47,6 +50,7 @@ ] }, "synInspireAnnexes": { + "type": "custom", "tokenizer": "keyword", "filter": [ "lowercase", @@ -56,22 +60,27 @@ ] }, "synInspireThemeUris": { + "type": "custom", "tokenizer": "keyword", "filter": ["lowercase", "synInspireThemeUris"] }, "synInspireThemes": { + "type": "custom", "tokenizer": "keyword", "filter": ["lowercase", "synInspireThemes"] }, "keepInspireServiceTypes": { + "type": "custom", "tokenizer": "keyword", "filter": ["lowercase", "keepInspireServiceTypes"] }, "keepInspireAnnexes": { + "type": "custom", "tokenizer": "keyword", "filter": ["lowercase", "keepInspireAnnexes"] }, "keepInspireThemes": { + "type": "custom", "tokenizer": "keyword", "filter": ["lowercase", "keepInspireThemes"] } @@ -1252,9 +1261,8 @@ { "tag": { "match": "th_*", + "match_mapping_type": "object", "mapping": { - "type": "object", - "copy_to": ["tag"], "properties": { "default": { "type": "keyword", @@ -1405,7 +1413,7 @@ "synInspireAnnexesType": { "match": "inspireAnnex*", "mapping": { - "type": "keyword", + "type": "text", "fielddata": true, "analyzer": "synInspireAnnexes", "search_analyzer": "keyword" @@ -1416,7 +1424,7 @@ "synInspireThemeType": { "match": "inspireTheme*", "mapping": { - "type": "keyword", + "type": "text", "fielddata": true, "analyzer": "synInspireThemes", "search_analyzer": "keyword" @@ -1427,7 +1435,7 @@ "inspireServiceType": { "match": "inspireServiceType", "mapping": { - "type": "keyword", + "type": "text", "fielddata": true, "analyzer": "keepInspireServiceTypes", "search_analyzer": "keyword" @@ -1526,8 +1534,7 @@ }, "document": { "type": "text", - "index": false, - "doc_values": false + "index": false }, "documentStandard": { "type": "keyword" @@ -1901,8 +1908,7 @@ }, "validReport": { "type": "text", - "index": false, - "doc_values": false + "index": false }, "useLimitation": { "type": "text", diff --git a/web/src/main/webapp/WEB-INF/data/config/index/features.json b/web/src/main/webapp/WEB-INF/data/config/index/features.json index 7033c9147a70..1f109914abd8 100644 --- a/web/src/main/webapp/WEB-INF/data/config/index/features.json +++ b/web/src/main/webapp/WEB-INF/data/config/index/features.json @@ -3,6 +3,7 @@ "analysis": { "analyzer": { "pathAnalyzer": { + "type": "custom", "tokenizer": "pathTokenizer" } }, @@ -12,6 +13,7 @@ "delimiter": "/", "replacement": "/", "skip": 0, + "buffer_size": 1024, "reverse": false } } @@ -43,15 +45,6 @@ } } }, - { - "dateType": { - "match": "ft_*_d", - "mapping": { - "type": "double", - "format": "" - } - } - }, { "dateTimeType": { "match": "ft_*_dt", @@ -66,7 +59,6 @@ "match": "ft_*_s", "mapping": { "type": "keyword", - "fielddata": true, "copy_to": "any" } } @@ -75,7 +67,7 @@ "stringPathType": { "match": "ft_*_s_tree", "mapping": { - "type": "keyword", + "type": "text", "fielddata": true, "analyzer": "pathAnalyzer", "search_analyzer": "keyword" @@ -86,8 +78,7 @@ "operationType": { "match": "_op*", "mapping": { - "type": "keyword", - "fielddata": true + "type": "keyword" } } } diff --git a/workers/wfsfeature-harvester/pom.xml b/workers/wfsfeature-harvester/pom.xml index 6fe55ad2a6ee..74c0202ec2c3 100644 --- a/workers/wfsfeature-harvester/pom.xml +++ b/workers/wfsfeature-harvester/pom.xml @@ -118,5 +118,10 @@ mockito-all test + + + co.elastic.clients + elasticsearch-java + diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java index 9c7cf4329093..d870278a445a 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2015 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -23,6 +23,11 @@ package org.fao.geonet.harvester.wfsfeatures.worker; +import co.elastic.clients.elasticsearch._types.Result; +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.IndexRequest; +import co.elastic.clients.elasticsearch.core.IndexResponse; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -33,14 +38,6 @@ import org.apache.camel.Exchange; import org.apache.camel.spring.SpringCamelContext; import org.apache.jcs.access.exception.InvalidArgumentException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.RestStatus; import org.fao.geonet.harvester.wfsfeatures.model.WFSHarvesterParameter; import org.fao.geonet.index.es.EsRestClient; import org.fao.geonet.kernel.search.EsSearchManager; @@ -82,8 +79,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.rest.RestStatus.CREATED; -import static org.elasticsearch.rest.RestStatus.OK; // TODO: GeoServer WFS 1.0.0 in some case return @@ -552,18 +547,21 @@ public void success(int nbOfFeatures) { } public boolean saveHarvesterReport() { - IndexRequest request = new IndexRequest(index); - request.id(report.get("id").toString()); - request.source(report); + IndexRequest request = IndexRequest.of( + b -> b.index(index) + .id(report.get("id").toString()) + .document(report) + ); + try { - IndexResponse response = client.getClient().index(request, RequestOptions.DEFAULT); - if (response.status() == RestStatus.CREATED || response.status() == RestStatus.OK) { + IndexResponse response = client.getClient().index(request); + if (response.result() == Result.Created) { LOGGER.info("Report saved for service {} and typename {}. Report id is {}", url, typeName, report.get("id")); } else { LOGGER.info("Failed to save report for {}. Error was '{}'.", typeName, - response.getResult()); + response.result()); } } catch (Exception e) { e.printStackTrace(); @@ -595,7 +593,8 @@ abstract class BulkResutHandler { protected BulkRequest bulk; protected int bulkSize; protected int failuresCount; - ActionListener listener; + // TODO: ES 8 + //ActionListener listener; public BulkResutHandler(Phaser phaser, String typeName, String url, int firstFeatureIndex, Report report, String metadataUuid) { this.phaser = phaser; @@ -605,22 +604,23 @@ public BulkResutHandler(Phaser phaser, String typeName, String url, int firstFea this.report = report; this.metadataUuid = metadataUuid; - this.bulk = new BulkRequest(index); + this.bulk = new BulkRequest.Builder().index(index).build(); this.bulkSize = 0; this.failuresCount = 0; LOGGER.debug(" {} - Indexing bulk (size {}) starting at {} ...", typeName, featureCommitInterval, firstFeatureIndex); - listener = new ActionListener() { + // TODO: ES 8 + /*listener = new ActionListener() { @Override public void onResponse(BulkResponse bulkResponse) { AtomicInteger bulkFailures = new AtomicInteger(); - if (bulkResponse.hasFailures()) { - Arrays.stream(bulkResponse.getItems()).forEach(e -> { - if (e.status() != OK - && e.status() != CREATED) { + if (bulkResponse.errors()) { + bulkResponse.items().forEach(e -> { + if (e.status() != 200 + && e.status() != 201) { String msg = String.format( - "Feature %s: Indexing error. Error is: %s", e.getId(), e.getFailure().toString()); + "Feature %s: Indexing error. Error is: %s", e.id(), e.error().toString()); report.put("error_ss", msg); LOGGER.warn(msg); bulkFailures.getAndIncrement(); @@ -630,7 +630,7 @@ public void onResponse(BulkResponse bulkResponse) { LOGGER.debug(" {} - Features [{}-{}] indexed in {} ms{}.", new Object[]{ typeName, firstFeatureIndex, firstFeatureIndex + bulkSize, System.currentTimeMillis() - begin, - bulkResponse.hasFailures() ? + bulkResponse.errors() ? " but with " + bulkFailures + " errors" : "" }); failuresCount = bulkFailures.get(); @@ -649,7 +649,7 @@ public void onFailure(Exception e) { LOGGER.error(msg); phaser.arriveAndDeregister(); } - }; + };*/ } public int getBulkSize() { @@ -668,8 +668,10 @@ public void addAction(ObjectNode rootNode, SimpleFeature feature) throws JsonPro } String id = String.format("%s#%s#%s", url, typeName, featureId); - bulk.add(new IndexRequest(index).id(id) - .source(jacksonMapper.writeValueAsString(rootNode), XContentType.JSON)); + // TODO: ES 8 + //bulk.operations().add(new IndexRequest.Builder().index(index).id(id) + // .document(jacksonMapper.writeValueAsString(rootNode)).build()); + // .routing(ROUTING_KEY)); bulkSize++; } @@ -690,7 +692,9 @@ public AsyncBulkResutHandler(Phaser phaser, String typeName, String url, int fir public void launchBulk(EsRestClient client) throws Exception { prepareLaunch(); - client.getClient().bulkAsync(this.bulk, RequestOptions.DEFAULT, this.listener); + + // TODO: ES 8 + //client.getAsynchClient().bulk(this.bulk, this.listener); } }