Update to Elasticsearch 8. Use of Elasticsearch Java API Client inste…

…ad of Java High Level REST Client
GeoCat · Oct 22, 2023 · 69f611c · 69f611c
1 parent 1943d69
commit 69f611c
Show file tree

Hide file tree

Showing 33 changed files with 567 additions and 470 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -577,9 +577,10 @@
       <artifactId>spring-test</artifactId>
       <scope>compile</scope>
     </dependency>
+
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
+      <groupId>co.elastic.clients</groupId>
+      <artifactId>elasticsearch-java</artifactId>
     </dependency>
 
     <dependency>

diff --git a/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java b/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java
@@ -23,13 +23,14 @@
 
 package org.fao.geonet.kernel;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import jeeves.server.UserSession;
 import jeeves.server.context.ServiceContext;
 
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.SearchHit;
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.constants.Edit;
 import org.fao.geonet.constants.Geonet;
@@ -248,8 +249,9 @@ public void selectAll(String type, ServiceContext context, UserSession session)
                     EsSearchManager searchManager = context.getBean(EsSearchManager.class);
                     searchResponse = searchManager.query(request.get("query"), FIELDLIST_UUID, 0, maxhits);
                     List<String> uuidList = new ArrayList();
-                    for (SearchHit h : Arrays.asList(searchResponse.getHits().getHits())) {
-                        uuidList.add((String) h.getSourceAsMap().get(Geonet.IndexFieldNames.UUID));
+                    ObjectMapper objectMapper = new ObjectMapper();
+                    for (Hit h : (List<Hit>) searchResponse.hits().hits()) {
+                        uuidList.add((String) objectMapper.convertValue(h.source(), Map.class).get(Geonet.IndexFieldNames.UUID));
                     }
 
                     if (selection != null) {

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java
@@ -1342,7 +1342,7 @@ public boolean isValid(Integer id) {
 
     boolean hasReferencingMetadata(ServiceContext context, AbstractMetadata metadata) throws Exception {
         StringBuilder query = new StringBuilder(String.format("xlink:*%s*", metadata.getUuid()));
-        return this.searchManager.query(query.toString(), null, 0, 0).getHits().getTotalHits().value > 0;
+        return this.searchManager.query(query.toString(), null, 0, 0).hits().hits().size() > 0;
     }
 
 }
diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java b/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java
@@ -23,10 +23,11 @@
 
 package org.fao.geonet.kernel.mef;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import jeeves.server.context.ServiceContext;
 import org.apache.commons.io.FileUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.SearchHit;
 import org.fao.geonet.Constants;
 import org.fao.geonet.GeonetContext;
 import org.fao.geonet.ZipUtil;
@@ -137,8 +138,9 @@ public static Path doExport(ServiceContext context, Set<String> uuids,
                 String mdSchema = null, mdTitle = null, mdAbstract = null, isHarvested = null;
                 MetadataType mdType = null;
 
-                SearchHit[] hits = result.getHits().getHits();
-                final Map<String, Object> source = hits[0].getSourceAsMap();
+                List<Hit> hits = result.hits().hits();
+                ObjectMapper objectMapper = new ObjectMapper();
+                final Map<String, Object> source = objectMapper.convertValue(hits.get(0).source(), Map.class);
                 mdSchema = (String) source.get(Geonet.IndexFieldNames.SCHEMA);
                 mdTitle = (String) source.get(Geonet.IndexFieldNames.RESOURCETITLE);
                 mdAbstract = (String) source.get(Geonet.IndexFieldNames.RESOURCEABSTRACT);

diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java
diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/OverviewIndexFieldUpdater.java b/core/src/main/java/org/fao/geonet/kernel/search/index/OverviewIndexFieldUpdater.java
@@ -23,8 +23,10 @@
 
 package org.fao.geonet.kernel.search.index;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
 import org.fao.geonet.kernel.search.EsSearchManager;
 import org.fao.geonet.util.XslUtil;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -63,9 +65,11 @@ private void processOverview(String id) {
             response = searchManager.query(String.format(
                 "+id:\"%s\" _exists_:overview.url -_exists_:overview.data",
                 id), null, source, 0, 1);
-            response.getHits().forEach(hit -> {
+            ObjectMapper objectMapper = new ObjectMapper();
+            response.hits().hits().forEach(h -> {
+                Hit hit = (Hit) h;
                 AtomicBoolean updates = new AtomicBoolean(false);
-                Map<String, Object> fields = hit.getSourceAsMap();
+                Map<String, Object> fields = objectMapper.convertValue(hit.source(), Map.class);
                 getHitOverviews(fields)
                     .stream()
                     .forEach(overview -> {

diff --git a/core/src/main/java/org/fao/geonet/util/XslUtil.java b/core/src/main/java/org/fao/geonet/util/XslUtil.java
@@ -23,6 +23,7 @@
 
 package org.fao.geonet.util;
 
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -43,16 +44,7 @@
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.action.search.MultiSearchRequest;
-import org.elasticsearch.action.search.MultiSearchResponse;
-import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.SystemInfo;
 import org.fao.geonet.api.records.attachments.FilesystemStore;
@@ -469,7 +461,7 @@ public static String getNodeId() {
         return ApplicationContextHolder.get().getBean(org.fao.geonet.NodeInfo.class).getId();
     }
 
-    
+
     public static String getNodeLogo(String key) {
         Optional<Source> source = getSource(key);
         return source.isPresent() ? source.get().getLogo() : "";
@@ -1587,7 +1579,8 @@ public static Element getTargetAssociatedResources(String uuid, String parentUui
         Element recordLinks = new Element("recordLinks");
 
         try {
-            MultiSearchRequest request = new MultiSearchRequest();
+            // TODO: ES 8
+            /*MultiSearchRequest request = new MultiSearchRequest();
 
 
             SearchRequest serviceRequest = new SearchRequest(searchManager.getDefaultIndex());
@@ -1656,7 +1649,7 @@ public static Element getTargetAssociatedResources(String uuid, String parentUui
 
             if (hasParent) {
                 recordLinks.addContent(buildRecordLink(response.getResponses()[3].getResponse().getHits(), "brothersAndSisters"));
-            }
+            }*/
         } catch (Exception e) {
             Log.error(Geonet.GEONETWORK,
                     "Get related document '" + uuid + "' error: " + e.getMessage(), e);
@@ -1677,24 +1670,27 @@ public static Node getTargetAssociatedResourcesAsNode(String uuid, String parent
         return null;
     }
 
-    private static List<Element> buildRecordLink(SearchHits hits, String type) {
+    private static List<Element> buildRecordLink(List<Hit> hits, String type) {
         ObjectMapper mapper = new ObjectMapper();
         SettingManager settingManager = ApplicationContextHolder.get().getBean(SettingManager.class);
         String recordUrlPrefix = settingManager.getNodeURL() + "api/records/";
         ArrayList<Element> listOfLinks = new ArrayList<>();
+
+        ObjectMapper objectMapper = new ObjectMapper();
+
         hits.forEach(record -> {
             Element recordLink = new Element("recordLink");
             recordLink.setAttribute("type", "object");
             ObjectNode recordLinkProperties = mapper.createObjectNode();
 
-            recordLinkProperties.put("to", record.getId());
+            recordLinkProperties.put("to", record.id());
             recordLinkProperties.put("origin", "catalog");
             recordLinkProperties.put("created", "bySearch");
-            Map<String, String> titleObject = (Map<String, String>) record.getSourceAsMap().get("resourceTitleObject");
+            Map<String, String> titleObject = (Map<String, String>) objectMapper.convertValue(record.source(), Map.class).get("resourceTitleObject");
             if (titleObject != null) {
                 recordLinkProperties.put("title", titleObject.get("default"));
             }
-            recordLinkProperties.put("url", recordUrlPrefix + record.getId());
+            recordLinkProperties.put("url", recordUrlPrefix + record.id());
             recordLinkProperties.put("type", type);
 
             try {

diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java
@@ -194,12 +194,12 @@ public void testDeleteBatchMetadata() throws Exception {
         importMetadata(serviceContext, uuid1);
         importMetadata(serviceContext, uuid2);
         assertEquals(2, metadataRepository.findAll(spec).size());
-        assertEquals(2, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).getHits().getTotalHits().value);
+        assertEquals(2, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).hits().hits().size());
 
         dataManager.batchDeleteMetadataAndUpdateIndex(spec);
 
         assertEquals(0, metadataRepository.findAll(spec).size());
-        assertEquals(0, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).getHits().getTotalHits().value);
+        assertEquals(0, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).hits().hits().size());
     }
 
     @Test

diff --git a/core/src/test/java/org/fao/geonet/kernel/ElasticsearchIndexingTest.java b/core/src/test/java/org/fao/geonet/kernel/ElasticsearchIndexingTest.java
@@ -1,7 +1,7 @@
 package org.fao.geonet.kernel;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
 import jeeves.server.context.ServiceContext;
-import org.elasticsearch.action.search.SearchResponse;
 import org.fao.geonet.AbstractCoreIntegrationTest;
 import org.fao.geonet.domain.AbstractMetadata;
 import org.fao.geonet.kernel.search.EsSearchManager;
@@ -47,7 +47,7 @@ public void complexDatesAreIndexedCheck() throws Exception {
 
         //THEN
         SearchResponse response = this.searchManager.query("_id:" + dbInsertedMetadata.getUuid() + " AND resourceTitleObject.default:holocene", null, 0, 10);
-        long actualHitNbr = response.getHits().getTotalHits().value;
+        long actualHitNbr = response.hits().hits().size();
         assertEquals(String.format("Incorrect indexation of Holocene data with complex date due to: %s and %s", response, dbInsertedMetadata), 1, actualHitNbr);
     }
 
@@ -59,7 +59,7 @@ private AbstractMetadata loadMetadataWithTemporalExtentUsingSimpleDates() throws
 
     private void validateIndexedExpectedData(AbstractMetadata dbInsertedSimpleDataMetadata, String resourceTitle, long expectedHitNbr) throws Exception {
         SearchResponse searchResponse = this.searchManager.query("resourceTitleObject.default:" + resourceTitle, null, 0, 10);
-        long actualHitNbr = searchResponse.getHits().getTotalHits().value;
+        long actualHitNbr = searchResponse.hits().hits().size();
         String assertionErrorMessage = "The %s data was not indexed the expected number of times due to: %s and %s";
         assertEquals(String.format(assertionErrorMessage, resourceTitle, searchResponse, dbInsertedSimpleDataMetadata), expectedHitNbr, actualHitNbr);
     }

diff --git a/csw-server/pom.xml b/csw-server/pom.xml
@@ -105,6 +105,11 @@
       <artifactId>elasticsearch</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>co.elastic.clients</groupId>
+      <artifactId>elasticsearch-java</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.springframework</groupId>
       <artifactId>spring-beans</artifactId>

diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java
@@ -23,11 +23,9 @@
 
 package org.fao.geonet.component.csw;
 
+import co.elastic.clients.elasticsearch._types.SortOptions;
 import jeeves.server.context.ServiceContext;
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilder;
-import org.elasticsearch.search.sort.SortOrder;
 import org.fao.geonet.GeonetContext;
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.csw.common.*;
@@ -244,7 +242,7 @@ public Element execute(Element request, ServiceContext context) throws CatalogEx
 
             response.addContent(echoedRequest);
         } else {
-            List<SortBuilder<FieldSortBuilder>> sort = _sortByParser.parseSortBy(request);
+            List<SortOptions> sort = _sortByParser.parseSortBy(request);
 
             response = new Element(getName() + "Response", Csw.NAMESPACE_CSW);
 

diff --git a/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SearchController.java b/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/SearchController.java
@@ -23,14 +23,13 @@
 
 package org.fao.geonet.kernel.csw.services.getrecords;
 
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jeeves.server.context.ServiceContext;
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilder;
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.GeonetContext;
 import org.fao.geonet.NodeInfo;
@@ -53,20 +52,12 @@
 import org.fao.geonet.kernel.search.EsSearchManager;
 import org.fao.geonet.utils.Log;
 import org.fao.geonet.utils.Xml;
-import org.geotools.xsd.Configuration;
-import org.geotools.xsd.Parser;
 import org.jdom.Attribute;
 import org.jdom.Content;
 import org.jdom.Element;
 import org.jdom.Namespace;
-import org.opengis.filter.Filter;
-import org.opengis.filter.capability.FilterCapabilities;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.xml.sax.SAXException;
 
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.IOException;
-import java.io.StringReader;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -439,7 +430,7 @@ private static Element applyElementNames(ServiceContext context, Set<String> ele
      */
         public Element search(ServiceContext context, int startPos, int maxRecords,
                                          ResultType resultType, String outSchema, ElementSetName setName,
-                                         Element filterExpr, String filterVersion, List<SortBuilder<FieldSortBuilder>> sort,
+                                         Element filterExpr, String filterVersion, List<SortOptions> sort,
                                          Set<String> elemNames, String typeName, int maxHitsFromSummary,
                                          String strategy) throws CatalogException {
 
@@ -462,11 +453,11 @@ public Element search(ServiceContext context, int startPos, int maxRecords,
         // TODO: Check to get summary or remove custom summary output
 
         try {
-            SearchResponse result = searchManager.query(esJsonQuery, new HashSet<String>(), startPos-1, maxRecords, sort);
+            SearchResponse result = searchManager.query(esJsonQuery, new HashSet<>(), startPos-1, maxRecords, sort);
 
-            SearchHit[] hits = result.getHits().getHits();
+            List<Hit> hits = result.hits().hits();
 
-            long numMatches = result.getHits().getTotalHits().value;
+            long numMatches = result.hits().hits().size();
 
             if (numMatches != 0 && startPos > numMatches) {
                 throw new InvalidParameterValueEx("startPosition", String.format(
@@ -477,8 +468,10 @@ public Element search(ServiceContext context, int startPos, int maxRecords,
 
             int counter = 0;
 
-            for(SearchHit hit : hits) {
-                int mdId =  Integer.parseInt((String) hit.getSourceAsMap().get("id"));
+            ObjectMapper objectMapper = new ObjectMapper();
+
+            for(Hit hit : hits) {
+                int mdId =  Integer.parseInt((String) objectMapper.convertValue(hit.source(), Map.class).get("id"));
 
                 AbstractMetadata metadata = metadataUtils.findOne(mdId);
 
@@ -543,7 +536,7 @@ public Element applyElementSetName(ServiceContext context, SchemaManager schemaM
                     outputSchema, id, schema, styleSheet.toString()));
             return null;
         } else {
-            Map<String, Object> params = new HashMap<String, Object>();
+            Map<String, Object> params = new HashMap<>();
             params.put("lang", displayLanguage);
 
             try {