From 81efcfc8a4456fe7596e2b1328a74fe3a699dad8 Mon Sep 17 00:00:00 2001 From: Francois Prunayre Date: Fri, 27 Oct 2023 10:57:42 +0200 Subject: [PATCH] Harvesting / WFS feature / Add WFS2 support for MapServer Mapserver may return by default GetCapabilities in version 2.0.0. Depending on the URL set in metadata records, WFS harvesting may fails with: ``` 2023-10-27T10:54:42,350 ERROR [geonetwork.harvest.wfs.features] - Failed to connect to server 'http://geoservices.brgm.fr/geologie?language=fre&'. Error is class net.opengis.wfs20.impl.WFSCapabilitiesTypeImpl cannot be cast to class net.opengis.wfs.WFSCapabilitiesType (net.opengis.wfs20.impl.WFSCapabilitiesTypeImpl and net.opengis.wfs.WFSCapabilitiesType are in unnamed module of loader org.eclipse.jetty.webapp.WebAppClassLoader @cda6100) ``` because of the `MapServerWFSStrategy` extending WFS1 strategy. Use `StrictWFS_2_0_Strategy` if harvesting is done using 2.0.0 on MapServer. Can be tested with ``` curl 'http://localhost:8080/geonetwork/srv/api/workers/data/wfs/actions/start' \ -X 'PUT' \ -H 'Accept: application/json, text/plain, */*' \ -H 'Accept-Language: eng' \ -H 'Content-Type: application/json;charset=UTF-8' \ -H 'Cookie: JSESSIONID=node0hjdye0js612o1qex2n36lekt81.node0; serverTime=1698396822574; sessionExpiry=1698398922574; XSRF-TOKEN=c353b2ca-a77c-4552-839c-37405771c4e2;' \ -H 'X-XSRF-TOKEN: c353b2ca-a77c-4552-839c-37405771c4e2' \ --data-raw '{"url":"http://geoservices.brgm.fr/geologie?language=fre&","strategy":"investigator","typeName":"GITES_EMP","version":"2.0.0"}' \ --compressed ``` --- .../wfsfeatures/model/WFSHarvesterParameter.java | 3 ++- .../worker/WFSClientWithStrategyInvestigator.java | 10 ++++++++-- .../worker/WFSDataStoreWithStrategyInvestigator.java | 7 +++---- .../wfsfeatures/worker/WFSHarvesterExchangeState.java | 8 +------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/model/WFSHarvesterParameter.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/model/WFSHarvesterParameter.java index d356526f399..b5c6bf80826 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/model/WFSHarvesterParameter.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/model/WFSHarvesterParameter.java @@ -33,6 +33,7 @@ */ @XmlRootElement(name = "wfs") public class WFSHarvesterParameter implements Serializable { + public static final String DEFAULT_VERSION = "1.1.0"; private String metadataUuid; private String url; @@ -41,7 +42,7 @@ public class WFSHarvesterParameter implements Serializable { private String typeName; - private String version = "1.1.0"; + private String version = DEFAULT_VERSION; private int timeOut = 300000; diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSClientWithStrategyInvestigator.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSClientWithStrategyInvestigator.java index 140238d92e5..cc3b979d250 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSClientWithStrategyInvestigator.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSClientWithStrategyInvestigator.java @@ -30,6 +30,7 @@ import org.geotools.data.wfs.internal.WFSGetCapabilities; import org.geotools.data.wfs.internal.WFSStrategy; import org.geotools.data.wfs.internal.v1_x.MapServerWFSStrategy; +import org.geotools.data.wfs.internal.v2_0.StrictWFS_2_0_Strategy; import org.geotools.http.HTTPClient; import org.geotools.http.HTTPResponse; import org.geotools.ows.ServiceException; @@ -38,6 +39,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.nio.charset.StandardCharsets; import static org.fao.geonet.harvester.wfsfeatures.worker.WFSHarvesterExchangeState.MAPSERVER_STRATEGY; import static org.fao.geonet.harvester.wfsfeatures.worker.WFSHarvesterExchangeState.QGIS_STRATEGY; @@ -84,12 +86,16 @@ private WFSStrategy determineCorrectStrategy(HTTPClient httpClient) { while ((readCount = inputStream.read(buff)) != -1) { out.write(buff, 0, readCount); } - String responsePayload = out.toString("UTF-8"); + String responsePayload = out.toString(StandardCharsets.UTF_8); if (responsePayload.contains("targetNamespace=\"http://www.qgis.org/gml\"")) { strategy = new QgisStrategy(); strategyId = QGIS_STRATEGY; } else if (responsePayload.contains("targetNamespace=\"http://mapserver.gis.umn.edu/mapserver\"")) { - strategy = new MapServerWFSStrategy(capabilities.getRawDocument()); + if (capabilities.getVersion().equals("2.0.0")) { + strategy = new StrictWFS_2_0_Strategy(); + } else { + strategy = new MapServerWFSStrategy(capabilities.getRawDocument()); + } strategyId = MAPSERVER_STRATEGY; } } catch (Exception e) { diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSDataStoreWithStrategyInvestigator.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSDataStoreWithStrategyInvestigator.java index 4849f8a7fa2..eac7b317ea1 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSDataStoreWithStrategyInvestigator.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSDataStoreWithStrategyInvestigator.java @@ -45,11 +45,10 @@ */ public class WFSDataStoreWithStrategyInvestigator extends WFSDataStoreFactory { - private static Logger LOGGER = LoggerFactory.getLogger(WFSHarvesterRouteBuilder.LOGGER_NAME); private String describeFeatureTypeUrl; - public void init (String url, String typeName) throws Exception { - this.describeFeatureTypeUrl = new OwsUtils().getDescribeFeatureTypeUrl(url, typeName, "1.1.0"); + public void init (String url, String typeName, String version) throws Exception { + this.describeFeatureTypeUrl = new OwsUtils().getDescribeFeatureTypeUrl(url, typeName, version); } @Override @@ -66,7 +65,7 @@ public WFSDataStore createDataStore(Map params) throws IOException { } } - final URL capabilitiesURL = (URL) URL.lookUp(params); + final URL capabilitiesURL = URL.lookUp(params); final HTTPClient http = getHttpClient(params); http.setTryGzip(config.isTryGZIP()); diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSHarvesterExchangeState.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSHarvesterExchangeState.java index d9b563dadad..e26f463e995 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSHarvesterExchangeState.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/WFSHarvesterExchangeState.java @@ -116,7 +116,7 @@ public void initDataStore() throws Exception { if (INVESTIGATOR_STRATEGY.equals(parameters.getStrategy())) { factory = new WFSDataStoreWithStrategyInvestigator(); ((WFSDataStoreWithStrategyInvestigator) factory).init( - parameters.getUrl(), parameters.getTypeName()); + parameters.getUrl(), parameters.getTypeName(), parameters.getVersion()); } else { factory = new WFSDataStoreFactory(); } @@ -145,15 +145,9 @@ public void initDataStore() throws Exception { } wfsDatastore = factory.createDataStore(m); - // Default to GeoTools auto mode for MapServer. if(factory instanceof WFSDataStoreWithStrategyInvestigator) { WFSClientWithStrategyInvestigator wfsClientWithStrategyInvestigator = (WFSClientWithStrategyInvestigator) wfsDatastore.getWfsClient(); this.strategyId = wfsClientWithStrategyInvestigator.getStrategyId(); - if (MAPSERVER_STRATEGY.equals(wfsClientWithStrategyInvestigator.getStrategyId())) { - Map connectionParameters = new HashMap<>(); - connectionParameters.put("WFSDataStoreFactory:GET_CAPABILITIES_URL", parameters.getUrl()); - wfsDatastore = (WFSDataStore) DataStoreFinder.getDataStore(connectionParameters); - } } } catch (IOException e) { String errorMsg = String.format(