Merge branch 'development' into 'fix/test-cases'

# Conflicts: # buildNumber.properties
RMLio · Jul 1, 2020 · ab9d322 · ab9d322
2 parents 1a95f78 + e176714
commit ab9d322
Show file tree

Hide file tree

Showing 21 changed files with 285 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,8 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Changed
+- updated grel-functions-java to 0.5.2
+
 ### Fixed
 - ObjectMap with type Blank Node is ignored (see [issue 164](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/164))
+- Support double quotes in references of RDBs (see [issue 163](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/163))
 
 ## [4.8.0] - 2020-05-25
 

diff --git a/buildNumber.properties b/buildNumber.properties
@@ -1,3 +1,3 @@
 #maven.buildNumber.plugin properties file
-#Wed Jun 17 09:39:07 CEST 2020
-buildNumber0=252
+#Fri Jun 19 14:06:02 CEST 2020
+buildNumber0=254
diff --git a/pom.xml b/pom.xml
@@ -137,7 +137,7 @@
         <dependency>
             <groupId>com.github.fnoio</groupId>
             <artifactId>grel-functions-java</artifactId>
-            <version>v0.4.0</version>
+            <version>v0.5.2</version>
         </dependency>
     </dependencies>
 

diff --git a/src/main/java/be/ugent/rml/MappingFactory.java b/src/main/java/be/ugent/rml/MappingFactory.java
@@ -14,7 +14,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.naming.Name;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -31,6 +30,9 @@ public class MappingFactory {
     private Term triplesMap;
     private QuadStore store;
     private List<PredicateObjectGraphMapping> predicateObjectGraphMappings;
+    // This boolean is true when the double in a reference need to be ignored.
+    // For example, when accessing data in a RDB.
+    private boolean ignoreDoubleQuotes;
     protected Logger logger = LoggerFactory.getLogger(this.getClass());
 
     public MappingFactory(FunctionLoader functionLoader) {
@@ -43,6 +45,7 @@ public Mapping createMapping(Term triplesMap, QuadStore store) throws Exception
         this.subjectMappingInfo = null;
         this.predicateObjectGraphMappings = new ArrayList<>();
         this.graphMappingInfos = null;
+        this.ignoreDoubleQuotes = this.areDoubleQuotesIgnored(store, triplesMap);
 
         parseSubjectMap();
         parsePredicateObjectMaps();
@@ -71,7 +74,7 @@ private void parseSubjectMap() throws Exception {
 
                     //checking if we are dealing with a Blank Node as subject
                     if (!termTypes.isEmpty() && termTypes.get(0).equals(new NamedNode(NAMESPACES.RR  + "BlankNode"))) {
-                        SingleRecordFunctionExecutor executor = RecordFunctionExecutorFactory.generate(store, subjectmap, true);
+                        SingleRecordFunctionExecutor executor = RecordFunctionExecutorFactory.generate(store, subjectmap, true, ignoreDoubleQuotes);
 
                         if (executor != null) {
                             generator = new BlankNodeGenerator(executor);
@@ -80,7 +83,7 @@ private void parseSubjectMap() throws Exception {
                         }
                     } else {
                         //we are not dealing with a Blank Node, so we create the template
-                        generator = new NamedNodeGenerator(RecordFunctionExecutorFactory.generate(store, subjectmap, true));
+                        generator = new NamedNodeGenerator(RecordFunctionExecutorFactory.generate(store, subjectmap, true, ignoreDoubleQuotes));
                     }
                 } else {
                     SingleRecordFunctionExecutor functionExecutor = parseFunctionTermMap(functionValues.get(0));
@@ -186,7 +189,7 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer<MappingInfo,
 
         if (functionValues.isEmpty()) {
             boolean encodeIRI = termType != null && termType.getValue().equals(NAMESPACES.RR + "IRI");
-            SingleRecordFunctionExecutor executor = RecordFunctionExecutorFactory.generate(store, objectmap, encodeIRI);
+            SingleRecordFunctionExecutor executor = RecordFunctionExecutorFactory.generate(store, objectmap, encodeIRI, ignoreDoubleQuotes);
 
             if (parentTriplesMaps.isEmpty() && parentTermMaps.isEmpty()) {
                 TermGenerator oGen;
@@ -237,11 +240,12 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer<MappingInfo,
                         FunctionModel equal = functionLoader.getFunction(new NamedNode("http://example.com/idlab/function/equal"));
                         Map<String, Object[]> parameters = new HashMap<>();
 
-                        SingleRecordFunctionExecutor parent = new ReferenceExtractor(parents.get(0));
+                        boolean ignoreDoubleQuotesInParent = this.areDoubleQuotesIgnored(store, parentTriplesMap);
+                        SingleRecordFunctionExecutor parent = new ReferenceExtractor(parents.get(0), ignoreDoubleQuotesInParent);
                         Object[] detailsParent = {"parent", parent};
                         parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter", detailsParent);
 
-                        SingleRecordFunctionExecutor child = new ReferenceExtractor(childs.get(0));
+                        SingleRecordFunctionExecutor child = new ReferenceExtractor(childs.get(0), ignoreDoubleQuotes);
                         Object[] detailsChild = {"child", child};
                         parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter2", detailsChild);
 
@@ -306,7 +310,7 @@ private List<MappingInfo> parseGraphMapsAndShortcuts(Term termMap) throws Except
             TermGenerator generator;
 
             if (functionValues.isEmpty()) {
-                SingleRecordFunctionExecutor executor = RecordFunctionExecutorFactory.generate(store, graphMap, true);
+                SingleRecordFunctionExecutor executor = RecordFunctionExecutorFactory.generate(store, graphMap, true, ignoreDoubleQuotes);
 
                 if (termType == null || termType.equals(new NamedNode(NAMESPACES.RR + "IRI"))) {
                     generator = new NamedNodeGenerator(executor);
@@ -351,7 +355,7 @@ private List<MappingInfo> parsePredicateMapsAndShortcuts(Term termMap) throws IO
 
             if (functionValues.isEmpty()) {
                 predicateMappingInfos.add(new MappingInfo(predicateMap,
-                        new NamedNodeGenerator(RecordFunctionExecutorFactory.generate(store, predicateMap, false))));
+                        new NamedNodeGenerator(RecordFunctionExecutorFactory.generate(store, predicateMap, false, ignoreDoubleQuotes))));
             } else {
                 SingleRecordFunctionExecutor functionExecutor = parseFunctionTermMap(functionValues.get(0));
 
@@ -454,7 +458,7 @@ private List<SingleRecordFunctionExecutor> getLanguageExecutorsForObjectMap(Term
             List<Term> functionValues = Utils.getObjectsFromQuads(store.getQuads(languageMap, new NamedNode(NAMESPACES.FNML + "functionValue"), null));
 
             if (functionValues.isEmpty()) {
-                executors.add(RecordFunctionExecutorFactory.generate(store, languageMap, false));
+                executors.add(RecordFunctionExecutorFactory.generate(store, languageMap, false, ignoreDoubleQuotes));
             } else {
                 executors.add(parseFunctionTermMap(functionValues.get(0)));
             }
@@ -521,4 +525,31 @@ private List<PredicateObjectGraphMapping> getPredicateObjectGraphMappingFromMult
         return list;
     }
 
+    /**
+     * This function returns true if double quotes should be ignored in references.
+     * @param store The store with the RML rules.
+     * @param triplesMap The Triples Map that should be checked.
+     * @return true if double quotes should be ignored in references, else false.
+     */
+    private boolean areDoubleQuotesIgnored(QuadStore store, Term triplesMap) {
+        List<Term> logicalSources = Utils.getObjectsFromQuads(store.getQuads(triplesMap, new NamedNode(NAMESPACES.RML + "logicalSource"), null));
+
+        if (!logicalSources.isEmpty()) {
+            Term logicalSource = logicalSources.get(0);
+
+            List<Term> sources = Utils.getObjectsFromQuads(store.getQuads(logicalSource, new NamedNode(NAMESPACES.RML + "source"), null));
+
+            if (!sources.isEmpty()) {
+                Term source = sources.get(0);
+
+                if (! (sources.get(0) instanceof Literal)) {
+                    List<Term> sourceType = Utils.getObjectsFromQuads(store.getQuads(source, new NamedNode(NAMESPACES.RDF + "type"), null));
+
+                    return sourceType.get(0).getValue().equals(NAMESPACES.D2RQ + "Database");
+                }
+            }
+        }
+
+        return false;
+    }
 }
diff --git a/src/main/java/be/ugent/rml/MyFileUtils.java b/src/main/java/be/ugent/rml/MyFileUtils.java
@@ -119,4 +119,23 @@ public File getFile() throws IOException {
         }
 
     }
+
+    /**
+     * This method returns the path of the parent of a file.
+     * @param c The class to which the file path is relative.
+     * @param path The path of the file.
+     * @return The path of the parent.
+     */
+    public static String getParentPath(Class c, String path) {
+        ClassLoader classLoader = c.getClassLoader();
+        URL url = classLoader.getResource(path);
+
+        if (url != null) {
+            path = url.getFile();
+        }
+
+        File outputFile = new File(path);
+
+        return outputFile.getParent();
+    }
 }
diff --git a/src/main/java/be/ugent/rml/RecordFunctionExecutorFactory.java b/src/main/java/be/ugent/rml/RecordFunctionExecutorFactory.java
@@ -12,15 +12,15 @@
 
 public class RecordFunctionExecutorFactory {
 
-    public static SingleRecordFunctionExecutor generate(QuadStore store, Term termMap, boolean encodeURI) {
+    public static SingleRecordFunctionExecutor generate(QuadStore store, Term termMap, boolean encodeURI, boolean ignoreDoubleQuotes) {
         List<Term> references = Utils.getObjectsFromQuads(store.getQuads(termMap, new NamedNode(NAMESPACES.RML + "reference"), null));
         List<Term> templates = Utils.getObjectsFromQuads(store.getQuads(termMap, new NamedNode(NAMESPACES.RR + "template"), null));
         List<Term> constants = Utils.getObjectsFromQuads(store.getQuads(termMap, new NamedNode(NAMESPACES.RR + "constant"), null));
 
         if (!references.isEmpty()) {
-            return new ReferenceExtractor(references.get(0).getValue());
+            return new ReferenceExtractor(references.get(0).getValue(), ignoreDoubleQuotes);
         } else if (!templates.isEmpty()) {
-            return new ConcatFunction(Utils.parseTemplate(templates.get(0).getValue()), encodeURI);
+            return new ConcatFunction(Utils.parseTemplate(templates.get(0).getValue(), ignoreDoubleQuotes), encodeURI);
         } else if (!constants.isEmpty()) {
             return new ConstantExtractor(constants.get(0).getValue());
         } else {

diff --git a/src/main/java/be/ugent/rml/Utils.java b/src/main/java/be/ugent/rml/Utils.java
@@ -360,7 +360,7 @@ public static int getFreePortNumber() throws IOException {
      * @param template template string
      * @return list of extractors
      **/
-    public static List<Extractor> parseTemplate(String template) {
+    public static List<Extractor> parseTemplate(String template, boolean ignoreDoubleQuotes) {
         ArrayList<Extractor> extractors = new ArrayList<>();
         String current = "";
         boolean previousWasBackslash = false;
@@ -389,7 +389,7 @@ public static List<Extractor> parseTemplate(String template) {
                         current += c;
                         previousWasBackslash = false;
                     } else if (variableBusy) {
-                        extractors.add(new ReferenceExtractor(current));
+                        extractors.add(new ReferenceExtractor(current, ignoreDoubleQuotes));
                         current = "";
                         variableBusy = false;
                     } else {

diff --git a/src/main/java/be/ugent/rml/extractor/ReferenceExtractor.java b/src/main/java/be/ugent/rml/extractor/ReferenceExtractor.java
@@ -9,14 +9,26 @@
 public class ReferenceExtractor implements Extractor, SingleRecordFunctionExecutor {
 
     public String reference;
+    private boolean ignoreDoubleQuotes;
 
-    public ReferenceExtractor(String reference) {
+    public ReferenceExtractor(String reference, boolean ignoreDoubleQuotes) {
         this.reference = reference;
+        this.ignoreDoubleQuotes = ignoreDoubleQuotes;
+    }
+
+    public ReferenceExtractor(String reference) {
+        this(reference, false);
     }
 
     @Override
     public List<Object> extract(Record record) {
-        return record.get(reference);
+        String temp = this.reference;
+
+        if (ignoreDoubleQuotes && temp.startsWith("\"") && temp.endsWith("\"")) {
+            temp = temp.substring(1, temp.length() - 1);
+        }
+
+        return record.get(temp);
     }
 
     @Override

diff --git a/src/test/java/be/ugent/rml/Mapper_CSV_Test.java b/src/test/java/be/ugent/rml/Mapper_CSV_Test.java
@@ -258,4 +258,9 @@ public void evaluate_1017_CSV() {
     public void evaluate_1018_CSV() {
         doMapping("test-cases/RMLTC1018-CSV/mapping.ttl", "test-cases/RMLTC1018-CSV/output.nq");
     }
+
+    @Test
+    public void evaluate_1019_CSV() {
+        doMapping("test-cases/RMLTC1019-CSV/mapping.ttl", "test-cases/RMLTC1019-CSV/output.nq");
+    }
 }
diff --git a/src/test/java/be/ugent/rml/Mapper_MySQL_Test.java b/src/test/java/be/ugent/rml/Mapper_MySQL_Test.java
@@ -12,6 +12,8 @@
 
 import java.util.Arrays;
 
+import static be.ugent.rml.MyFileUtils.getParentPath;
+
 @RunWith(Parameterized.class)
 public class Mapper_MySQL_Test extends MySQLTestCore {
 
@@ -105,6 +107,8 @@ public static Iterable<Object[]> data() {
                 {"RMLTC0019b", null},
                 {"RMLTC0020a", null},
                 {"RMLTC0020b", null},
+                {"RMLTC1019", null},
+                {"RMLTC1020", null}
         });
     }
 
@@ -124,9 +128,10 @@ private void mappingTest(String testCaseName, Class expectedException) throws Ex
         mysqlDB.source(resourcePath);
 
         // mapping
+        String parentPath = getParentPath(getClass(), outputPath);
 
         if (expectedException == null) {
-            doMapping(tempMappingPath, outputPath);
+            doMapping(tempMappingPath, outputPath, parentPath);
         } else {
             doMappingExpectError(tempMappingPath);
         }

diff --git a/src/test/java/be/ugent/rml/TestCore.java b/src/test/java/be/ugent/rml/TestCore.java
@@ -25,7 +25,11 @@ public abstract class TestCore {
     final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     Executor createExecutor(String mapPath) throws Exception {
-        return createExecutor(mapPath, new ArrayList<>());
+        return createExecutor(mapPath, new ArrayList<>(), null);
+    }
+
+    Executor createExecutor(String mapPath, List<Quad> extraQuads) throws Exception {
+        return createExecutor(mapPath, extraQuads, null);
     }
 
     /**
@@ -36,7 +40,7 @@ Executor createExecutor(String mapPath) throws Exception {
      * @return An executor.
      * @throws Exception
      */
-    Executor createExecutor(String mapPath, List<Quad> extraQuads) throws Exception {
+    Executor createExecutor(String mapPath, List<Quad> extraQuads, String parentPath) throws Exception {
         ClassLoader classLoader = getClass().getClassLoader();
         // execute mapping file
         URL url = classLoader.getResource(mapPath);
@@ -47,10 +51,19 @@ Executor createExecutor(String mapPath, List<Quad> extraQuads) throws Exception
 
         File mappingFile = new File(mapPath);
         QuadStore rmlStore = QuadStoreFactory.read(mappingFile);
+
+        if (parentPath == null) {
+            parentPath = mappingFile.getParent();
+        }
+
         rmlStore.addQuads(extraQuads);
 
         return new Executor(rmlStore,
-                new RecordsFactory(mappingFile.getParent()), Utils.getBaseDirectiveTurtle(mappingFile));
+                new RecordsFactory(parentPath), Utils.getBaseDirectiveTurtle(mappingFile));
+    }
+
+    Executor createExecutor(String mapPath, String parentPath) throws Exception {
+        return createExecutor(mapPath, new ArrayList<>(), parentPath);
     }
 
     Executor createExecutor(String mapPath, FunctionLoader functionLoader) throws Exception {
@@ -90,6 +103,12 @@ public void doMapping(Path cwd, String mappingFiles, String output, String expec
         }
     }
 
+    /**
+     * This method executes a mapping, compares it to the expected out, and returns the used Executor.
+     * @param mapPath The path of the mapping file.
+     * @param outPath The path of the file with the expected output.
+     * @return The Executor used to execute the mapping.
+     */
     public Executor doMapping(String mapPath, String outPath) {
         try {
             Executor executor = this.createExecutor(mapPath);
@@ -103,6 +122,31 @@ public Executor doMapping(String mapPath, String outPath) {
         return null;
     }
 
+    /**
+     * This method executes a mapping, compares it to the expected out, and returns the used Executor.
+     * @param mapPath The path of the mapping file.
+     * @param outPath The path of the file with the expected output.
+     * @param parentPath The path of the folder where the Executor looks for files, such as CSV files.
+     * @return The Executor used to execute the mapping.
+     */
+    public Executor doMapping(String mapPath, String outPath, String parentPath) {
+        try {
+            Executor executor = this.createExecutor(mapPath, parentPath);
+            doMapping(executor, outPath);
+            return executor;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            fail();
+        }
+
+        return null;
+    }
+
+    /**
+     * This method executes a mapping and compares with the expected output.
+     * @param executor The Executor that is used to execute the mapping.
+     * @param outPath The path of the file with the expected output.
+     */
     void doMapping(Executor executor, String outPath) throws Exception {
         QuadStore result = executor.execute(null);
         result.removeDuplicates();

diff --git a/src/test/resources/test-cases/RMLTC1019-CSV/country_info.csv b/src/test/resources/test-cases/RMLTC1019-CSV/country_info.csv
@@ -0,0 +1,4 @@
+Country Code,Name,"""ISO 3166"""
+1,"Bolivia, Plurinational State of",BO
+2,Ireland,IE
+3,Saint Martin (French part),MF