Merge join (#428)

* move to RDF4J 5.0.0-SNAPSHOT * implement support for merge join * code cleanup * code cleanup * code cleanup * code cleanup * fixed compliance tests
the-qa-company · Jan 16, 2024 · a697f7b · a697f7b
1 parent 6d79aa0
commit a697f7b
Show file tree

Hide file tree

Showing 56 changed files with 1,669 additions and 383 deletions.
diff --git a/.gitignore b/.gitignore
@@ -55,3 +55,6 @@ release/RELEASE_OUTPUT.md
 .idea
 data
 .nvmrc
+wikidata
+qendpoint-store/wdbench-indexes
+wdbench-results
diff --git a/pom.xml b/pom.xml
@@ -69,6 +69,19 @@
         <maven.compiler.target>17</maven.compiler.target>
     </properties>
 
+    <repositories>
+        <repository>
+            <id>oss.sonatype.org-snapshot</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
     <distributionManagement>
         <snapshotRepository>
             <id>sonatype-nexus-snapshots</id>
@@ -205,4 +218,4 @@
         <system>Github</system>
         <url>https://github.com/the-qa-company/qEndpoint/issues</url>
     </issueManagement>
-</project>
+</project>
diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml
@@ -25,12 +25,25 @@
         </license>
     </licenses>
 
+    <repositories>
+        <repository>
+            <id>oss.sonatype.org-snapshot</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
     <properties>
         <common_codec.version>1.15</common_codec.version>
         <json_simple.version>1.1.1</json_simple.version>
         <junit.version>4.13.2</junit.version>
         <lwjgl.version>3.3.1</lwjgl.version>
-        <rdf4j.version>4.2.3</rdf4j.version>
+        <rdf4j.version>5.0.0-SNAPSHOT</rdf4j.version>
         <spring.version>3.0.2</spring.version>
         <logback.version>1.4.5</logback.version>
 
@@ -132,4 +145,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml
@@ -26,9 +26,7 @@
 
     <parent>
         <groupId>com.the-qa-company</groupId>
-        <artifactId>
-            qendpoint-parent
-        </artifactId>
+        <artifactId>qendpoint-parent</artifactId>
         <version>1.14.1</version>
     </parent>
 
@@ -46,17 +44,14 @@
         <json_simple.version>1.1.1</json_simple.version>
         <junit.version>4.13.2</junit.version>
         <lwjgl.version>3.3.1</lwjgl.version>
-        <rdf4j.version>4.2.3</rdf4j.version>
         <spring.version>2.7.5</spring.version>
         <logback.version>1.4.5</logback.version>
         <roaringbitmap.version>0.9.44</roaringbitmap.version>
 
         <jena.version>4.3.2</jena.version>
         <slf4j.version>1.7.30</slf4j.version>
 
-        <project.build.sourceEncoding>
-            UTF-8
-        </project.build.sourceEncoding>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     </properties>
 
@@ -99,18 +94,14 @@
         </dependency>
         <dependency>
             <groupId>pl.pragmatists</groupId>
-            <artifactId>
-                JUnitParams
-            </artifactId>
+            <artifactId>JUnitParams</artifactId>
             <version>1.1.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.tukaani</groupId>
             <artifactId>xz</artifactId>
-            <version>
-                1.6
-            </version>
+            <version>1.6</version>
         </dependency>
         <dependency>
             <groupId>com.beust</groupId>
@@ -123,4 +114,4 @@
             <version>${roaringbitmap.version}</version>
         </dependency>
     </dependencies>
-</project>
+</project>
diff --git a/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java b/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java
@@ -19,6 +19,8 @@
 
 package com.the_qa_company.qendpoint.core.enums;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -78,6 +80,24 @@ public enum TripleComponentOrder {
 		this.mask = mask;
 	}
 
+	/**
+	 * Search for an acceptable value in a map of orders
+	 *
+	 * @param flags flags to search the value
+	 * @param map   map
+	 * @param <T>   value type
+	 * @return find value, null for no matching value
+	 */
+	public static <T, Z extends TripleComponentOrder> List<Z> fetchAllBestForCfg(int flags, Map<Z, T> map) {
+		ArrayList<Z> ret = new ArrayList<>();
+		for (Map.Entry<Z, T> e : map.entrySet()) {
+			if ((e.getKey().mask & flags) != 0) {
+				ret.add(e.getKey());
+			}
+		}
+		return ret;
+	}
+
 	/**
 	 * Search for an acceptable value in a map of orders
 	 *
@@ -123,4 +143,5 @@ public TripleComponentRole getPredicateMapping() {
 	public TripleComponentRole getObjectMapping() {
 		return objectMapping;
 	}
+
 }
diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java
@@ -206,7 +206,8 @@ public void mapFromHDT(File f, long offset, ProgressListener listener) throws IO
 			String hdtFormat = ci.getFormat();
 			if (!hdtFormat.equals(HDTVocabulary.HDT_CONTAINER) && !hdtFormat.equals(HDTVocabulary.HDT_CONTAINER_2)) {
 				throw new IllegalFormatException("This software (v" + HDTVersion.HDT_VERSION + ".x.x | v"
-						+ HDTVersion.HDT_VERSION_2 + ".x.x) cannot open this version of HDT File (" + hdtFormat + ")");
+						+ HDTVersion.HDT_VERSION_2 + ".x.x) cannot open this version of HDT File hdtFileName:"
+						+ hdtFileName + " format:" + hdtFormat + "");
 			}
 
 			input.printIndex("HDT Header");

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPComponent.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPComponent.java
@@ -10,6 +10,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.Serial;
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -21,10 +23,14 @@
  *
  * @author Antoine Willerval
  */
-public class QEPComponent implements Cloneable {
-	record SharedElement(long id, DictionarySectionRole role, QEPDataset dataset, String debugMapped) {}
+public class QEPComponent implements Cloneable, Serializable {
+	@Serial
+	private static final long serialVersionUID = 6621230579376315429L;
 
-	record PredicateElement(long id, QEPDataset dataset) {}
+	record SharedElement(long id, DictionarySectionRole role, QEPDataset dataset, String debugMapped)
+			implements Serializable {}
+
+	record PredicateElement(long id, QEPDataset dataset) implements Serializable {}
 
 	private static final Logger logger = LoggerFactory.getLogger(QEPComponent.class);
 
@@ -34,7 +40,7 @@ record PredicateElement(long id, QEPDataset dataset) {}
 	RDFNodeType rdfNodeType;
 	Optional<CharSequence> language;
 	CharSequence datatype;
-	final QEPCore core;
+	transient final QEPCore core;
 
 	private QEPComponent(QEPComponent other) {
 		this.predicateIds = new HashMap<>(other.predicateIds);

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPCore.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPCore.java
@@ -17,6 +17,7 @@
 import com.the_qa_company.qendpoint.core.options.HDTOptions;
 import com.the_qa_company.qendpoint.core.storage.converter.NodeConverter;
 import com.the_qa_company.qendpoint.core.storage.iterator.CatQueryCloseable;
+import com.the_qa_company.qendpoint.core.storage.iterator.CloseableIterator;
 import com.the_qa_company.qendpoint.core.storage.iterator.QueryCloseableIterator;
 import com.the_qa_company.qendpoint.core.storage.merge.QEPCoreMergeThread;
 import com.the_qa_company.qendpoint.core.storage.search.QEPComponentTriple;
@@ -31,6 +32,8 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.io.Serial;
+import java.io.Serializable;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
@@ -79,6 +82,7 @@
  * @author Antoine Willerval
  */
 public class QEPCore implements AutoCloseable {
+
 	private static final Logger logger = LoggerFactory.getLogger(QEPCore.class);
 	/**
 	 * the max size of a dataset id

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPDataset.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPDataset.java
@@ -14,6 +14,8 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.Serial;
+import java.io.Serializable;
 import java.nio.file.Path;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -26,7 +28,10 @@
  * @author Antoine Willerval
  * @see QEPCore
  */
-public class QEPDataset implements Closeable {
+public class QEPDataset implements Closeable, Serializable {
+
+	@Serial
+	private static final long serialVersionUID = 7525689572432598258L;
 
 	public record ComponentFind(QEPDataset dataset, TripleComponentRole role, long id, long pid) {
 		public boolean isFind() {
@@ -261,7 +266,7 @@ public void setComponentInDelta(TripleComponentRole role, long component) {
 	public QueryCloseableIterator search(CharSequence subject, CharSequence predicate, CharSequence object)
 			throws QEPCoreException {
 		QEPDatasetContext ctx = createContext();
-		return search(ctx, subject, predicate, object).attach(ctx);
+		return (QueryCloseableIterator) search(ctx, subject, predicate, object).attach(ctx);
 	}
 
 	/**
@@ -278,7 +283,7 @@ public QueryCloseableIterator search(CharSequence subject, CharSequence predicat
 	public QueryCloseableIterator search(QEPComponent subject, QEPComponent predicate, QEPComponent object)
 			throws QEPCoreException {
 		QEPDatasetContext ctx = createContext();
-		return search(ctx, subject, predicate, object).attach(ctx);
+		return (QueryCloseableIterator) search(ctx, subject, predicate, object).attach(ctx);
 	}
 
 	/**
@@ -292,7 +297,7 @@ public QueryCloseableIterator search(QEPComponent subject, QEPComponent predicat
 	 */
 	public QueryCloseableIterator search(QEPComponentTriple pattern) throws QEPCoreException {
 		QEPDatasetContext ctx = createContext();
-		return search(ctx, pattern).attach(ctx);
+		return (QueryCloseableIterator) search(ctx, pattern).attach(ctx);
 	}
 
 	/**

diff --git a/...main/java/com/the_qa_company/qendpoint/core/storage/iterator/CloseableAttachIterator.java b/...main/java/com/the_qa_company/qendpoint/core/storage/iterator/CloseableAttachIterator.java
@@ -11,32 +11,33 @@
  * @param <T>
  * @param <E>
  */
-public class CloseableAttachIterator<T, E extends Exception> implements CloseableIterator<T, E> {
+public class CloseableAttachIterator<T extends RuntimeException> implements CloseableIterator<T> {
 	@SafeVarargs
-	public static <T, E extends Exception> CloseableIterator<T, E> of(CloseableIterator<T, E> it,
-			AutoCloseableGeneric<E>... closeables) {
+	public static <T> CloseableIterator<T> of(CloseableIterator<T> it,
+			AutoCloseableGeneric<? extends RuntimeException>... closeables) {
 		if (closeables.length == 0) {
 			return it;
 		}
-		return new CloseableAttachIterator<>(it, closeables);
+		return new CloseableAttachIterator(it, closeables);
 	}
 
-	private final CloseableIterator<T, E> handle;
-	private final List<AutoCloseableGeneric<E>> closeables;
+	private final CloseableIterator<T> handle;
+	private final List<AutoCloseableGeneric<RuntimeException>> closeables;
 
 	@SafeVarargs
-	private CloseableAttachIterator(CloseableIterator<T, E> handle, AutoCloseableGeneric<E>... closeableGenerics) {
+	private CloseableAttachIterator(CloseableIterator<T> handle,
+			AutoCloseableGeneric<RuntimeException>... closeableGenerics) {
 		this.handle = handle;
 		closeables = new ArrayList<>(List.of(closeableGenerics));
 	}
 
 	@Override
-	public void close() throws E {
+	public void close() {
 		try {
 			handle.close();
 		} catch (Error | Exception t) {
 			try {
-				AutoCloseableGeneric.<E>closeAll(closeables);
+				AutoCloseableGeneric.closeAll(closeables);
 			} catch (RuntimeException | Error err) {
 				err.addSuppressed(t);
 				throw err;
@@ -62,12 +63,6 @@ public void remove() {
 		handle.remove();
 	}
 
-	@Override
-	public CloseableIterator<T, E> attach(AutoCloseableGeneric<E> closeable) {
-		closeables.add(closeable);
-		return this;
-	}
-
 	@Override
 	public void forEachRemaining(Consumer<? super T> action) {
 		handle.forEachRemaining(action);

diff --git a/...java/com/the_qa_company/qendpoint/core/storage/iterator/CloseableAttachQueryIterator.java b/...java/com/the_qa_company/qendpoint/core/storage/iterator/CloseableAttachQueryIterator.java
@@ -12,7 +12,7 @@ public class CloseableAttachQueryIterator implements QueryCloseableIterator {
 
 	@SafeVarargs
 	public static QueryCloseableIterator of(QueryCloseableIterator it,
-			AutoCloseableGeneric<QEPCoreException>... closeables) {
+			AutoCloseableGeneric<? extends RuntimeException>... closeables) {
 		Objects.requireNonNull(it, "it can't be null!");
 		if (closeables.length == 0) {
 			return it;
@@ -21,11 +21,11 @@ public static QueryCloseableIterator of(QueryCloseableIterator it,
 	}
 
 	private final QueryCloseableIterator handle;
-	private final List<AutoCloseableGeneric<QEPCoreException>> closeables;
+	private final List<AutoCloseableGeneric<? extends RuntimeException>> closeables;
 
 	@SafeVarargs
 	private CloseableAttachQueryIterator(QueryCloseableIterator handle,
-			AutoCloseableGeneric<QEPCoreException>... closeableGenerics) {
+			AutoCloseableGeneric<? extends RuntimeException>... closeableGenerics) {
 		this.handle = handle;
 		closeables = new ArrayList<>(List.of(closeableGenerics));
 	}
@@ -68,7 +68,7 @@ public long estimateCardinality() {
 	}
 
 	@Override
-	public QueryCloseableIterator attach(AutoCloseableGeneric<QEPCoreException> closeable) {
+	public QueryCloseableIterator attach(AutoCloseableGeneric<? extends RuntimeException> closeable) {
 		closeables.add(closeable);
 		return this;
 	}