GH-443 HDTq store (#447)

* [skip ci] wip * test quads * add quads compliance tests * run formatter
the-qa-company · Feb 2, 2024 · 42f4d62 · 42f4d62
1 parent ca4fc6b
commit 42f4d62
Show file tree

Hide file tree

Showing 34 changed files with 2,188 additions and 1,060 deletions.
diff --git a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java
@@ -426,7 +426,7 @@ void initializeEndpointStore(boolean finishLoading) throws IOException {
 			CompiledSailOptions options = new CompiledSailOptions();
 			options.setPort(port);
 			options.setEndpointThreshold(threshold);
-			options.setHdtSpec(hdtSpec);
+			options.setHdtSpec(HDTOptions.of(hdtSpec));
 			options.setTimeoutQuery(maxTimeoutCfg);
 			options.setTimeoutUpdate(maxTimeoutUpdateCfg);
 

diff --git a/...core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/MultiLayerBitmap.java b/...core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/MultiLayerBitmap.java
@@ -7,95 +7,11 @@
 import java.io.OutputStream;
 
 public interface MultiLayerBitmap extends Bitmap {
-	static MultiLayerBitmap ofBitmap(Bitmap bitmap) {
+	static MultiLayerBitmap ofBitmap(Bitmap bitmap, long graphs) {
 		if (bitmap instanceof MultiLayerBitmap mlb) {
 			return mlb;
 		}
-		return new MultiLayerBitmap() {
-
-			@Override
-			public boolean access(long layer, long position) {
-				assert layer == 0;
-				return bitmap.access(position);
-			}
-
-			@Override
-			public long rank1(long layer, long position) {
-				return bitmap.rank1(position);
-			}
-
-			@Override
-			public long rank0(long layer, long position) {
-				assert layer == 0;
-				return bitmap.rank0(position);
-			}
-
-			@Override
-			public long selectPrev1(long layer, long start) {
-				assert layer == 0;
-				return bitmap.selectPrev1(start);
-			}
-
-			@Override
-			public long selectNext1(long layer, long start) {
-				assert layer == 0;
-				return bitmap.selectNext1(start);
-			}
-
-			@Override
-			public long select0(long layer, long n) {
-				assert layer == 0;
-				return bitmap.select0(n);
-			}
-
-			@Override
-			public long select1(long layer, long n) {
-				assert layer == 0;
-				return bitmap.select1(n);
-			}
-
-			@Override
-			public long getNumBits() {
-				return bitmap.getNumBits();
-			}
-
-			@Override
-			public long countOnes(long layer) {
-				assert layer == 0;
-				return bitmap.countOnes();
-			}
-
-			@Override
-			public long countZeros(long layer) {
-				assert layer == 0;
-				return bitmap.countZeros();
-			}
-
-			@Override
-			public long getSizeBytes() {
-				return bitmap.getSizeBytes();
-			}
-
-			@Override
-			public void save(OutputStream output, ProgressListener listener) throws IOException {
-				bitmap.save(output, listener);
-			}
-
-			@Override
-			public void load(InputStream input, ProgressListener listener) throws IOException {
-				bitmap.load(input, listener);
-			}
-
-			@Override
-			public String getType() {
-				return bitmap.getType();
-			}
-
-			@Override
-			public long getLayersCount() {
-				return 1;
-			}
-		};
+		return MultiLayerBitmapWrapper.of(bitmap, graphs);
 	}
 
 	/**

diff --git a/...c/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/MultiLayerBitmapWrapper.java b/...c/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/MultiLayerBitmapWrapper.java
@@ -0,0 +1,183 @@
+package com.the_qa_company.qendpoint.core.compact.bitmap;
+
+import com.the_qa_company.qendpoint.core.exceptions.NotImplementedException;
+import com.the_qa_company.qendpoint.core.listener.ProgressListener;
+import com.the_qa_company.qendpoint.core.util.io.IOUtil;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class MultiLayerBitmapWrapper implements MultiLayerBitmap, Closeable {
+	public static class MultiLayerModBitmapWrapper extends MultiLayerBitmapWrapper
+			implements ModifiableMultiLayerBitmap {
+
+		private MultiLayerModBitmapWrapper(ModifiableBitmap handle, long graphs) {
+			super(handle, graphs);
+		}
+
+		@Override
+		public void set(long layer, long position, boolean value) {
+			((ModifiableBitmap) handle).set(graphs * position + layer, value);
+		}
+	}
+
+	public static MultiLayerBitmapWrapper of(Bitmap bitmap, long graphCount) {
+		if (bitmap == null) {
+			return null;
+		}
+
+		if (bitmap instanceof ModifiableBitmap mbm) {
+			return new MultiLayerModBitmapWrapper(mbm, graphCount);
+		}
+
+		return new MultiLayerBitmapWrapper(bitmap, graphCount);
+	}
+
+	public static MultiLayerModBitmapWrapper of(ModifiableBitmap bitmap, long graphCount) {
+		if (bitmap == null) {
+			return null;
+		}
+
+		return new MultiLayerModBitmapWrapper(bitmap, graphCount);
+	}
+
+	protected final Bitmap handle;
+	protected final long graphs;
+
+	private MultiLayerBitmapWrapper(Bitmap handle, long graphs) {
+		this.handle = handle;
+		this.graphs = graphs;
+	}
+
+	@SuppressWarnings("unchecked")
+	public <T extends Bitmap> T getHandle() {
+		return (T) handle;
+	}
+
+	@Override
+	public boolean access(long layer, long position) {
+		return handle.access(graphs * position + layer);
+	}
+
+	@Override
+	public long rank1(long layer, long position) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long rank0(long layer, long position) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long selectPrev1(long layer, long start) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long selectNext1(long layer, long start) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long select0(long layer, long n) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long select1(long layer, long n) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long getNumBits() {
+		return (handle.getNumBits() - 1) / 5 + 1;
+	}
+
+	@Override
+	public long countOnes(long layer) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long countZeros(long layer) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long getSizeBytes() {
+		return handle.getSizeBytes();
+	}
+
+	@Override
+	public void save(OutputStream output, ProgressListener listener) throws IOException {
+		handle.save(output, listener);
+	}
+
+	@Override
+	public void load(InputStream input, ProgressListener listener) throws IOException {
+		handle.load(input, listener);
+	}
+
+	@Override
+	public String getType() {
+		return handle.getType();
+	}
+
+	@Override
+	public long getLayersCount() {
+		return graphs;
+	}
+
+	@Override
+	public boolean access(long position) {
+		return access(0, position);
+	}
+
+	@Override
+	public long rank1(long position) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long rank0(long position) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long selectPrev1(long start) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long selectNext1(long start) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long select0(long n) {
+		throw new NotImplementedException();
+	}
+
+	@Override
+	public long select1(long n) {
+		return MultiLayerBitmap.super.select1(n);
+	}
+
+	@Override
+	public long countOnes() {
+		return MultiLayerBitmap.super.countOnes();
+	}
+
+	@Override
+	public long countZeros() {
+		return MultiLayerBitmap.super.countZeros();
+	}
+
+	@Override
+	public void close() throws IOException {
+		IOUtil.closeObject(handle);
+	}
+}
diff --git a/...ava/com/the_qa_company/qendpoint/core/dictionary/impl/kcat/GroupBySubjectMapIterator.java b/...ava/com/the_qa_company/qendpoint/core/dictionary/impl/kcat/GroupBySubjectMapIterator.java
@@ -132,7 +132,8 @@ public static Iterator<TripleID> fromHDTs(KCatMerger merger, HDT[] hdts, List<?
 					// extract hdt elements for this index
 					HDT hdt = hdts[hdtIndex];
 					MultiLayerBitmap deleteBitmap = deleteBitmaps == null ? null
-							: MultiLayerBitmap.ofBitmap(deleteBitmaps.get(hdtIndex));
+							: MultiLayerBitmap.ofBitmap(deleteBitmaps.get(hdtIndex),
+									hdt.getDictionary().supportGraphs() ? hdt.getDictionary().getNgraphs() : 1);
 
 					if (hdt.getTriples().getNumberOfElements() == 0) {
 						// no triples
@@ -172,7 +173,8 @@ public static Iterator<TripleID> fromHDTs(KCatMerger merger, HDT[] hdts, List<?
 					// extract hdt elements for this index
 					HDT hdt = hdts[hdtIndex];
 					MultiLayerBitmap deleteBitmap = deleteBitmaps == null ? null
-							: MultiLayerBitmap.ofBitmap(deleteBitmaps.get(hdtIndex));
+							: MultiLayerBitmap.ofBitmap(deleteBitmaps.get(hdtIndex),
+									hdt.getDictionary().supportGraphs() ? hdt.getDictionary().getNgraphs() : 1);
 
 					if (hdt.getTriples().getNumberOfElements() == 0) {
 						// no triples

diff --git a/...t-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/kcat/KCatImpl.java b/...t-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/kcat/KCatImpl.java
@@ -260,7 +260,8 @@ private KCatImpl(List<?> hdtFileNames, List<? extends Bitmap> deleteBitmaps, HDT
 					long c = 0;
 
 					@SuppressWarnings("resource")
-					MultiLayerBitmap bm = MultiLayerBitmap.ofBitmap(deleteBitmap);
+					MultiLayerBitmap bm = MultiLayerBitmap.ofBitmap(deleteBitmap,
+							hdt.getDictionary().supportGraphs() ? hdt.getDictionary().getNgraphs() : 1);
 
 					while (searchAll.hasNext()) {
 						TripleID tripleID = searchAll.next();