diff --git a/README.md b/README.md index db0700b8..a00ccba6 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ The QA Company over the social networks ## About The qEndpoint is a highly scalable triple store with full-text and [GeoSPARQL](https://www.ogc.org/standards/geosparql) support. It can be used as a standalone SPARQL endpoint, or as a dependency. -The qEndpoint is for example used in [Kohesio](https://kohesio.ec.europa.eu/) where each interaction with the UI corresponds to an underlying SPARQL query on the qEndpoint. +The qEndpoint is for example used in [Kohesio](https://kohesio.ec.europa.eu/) where each interaction with the UI corresponds to an underlying SPARQL query on the qEndpoint. Also qEndpoint is part of [QAnswer](https://qanswer.ai) enabeling question answering over RDF Graphs. ### Built With diff --git a/pom.xml b/pom.xml index 6ad2ca77..1c97efae 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.6 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index b53618e4..331680f5 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.0 + 1.15.6 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.6 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index 067dbd8b..b90f8f81 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.0 + 1.15.6 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.6 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index 22501494..c847f459 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.0 + 1.15.6 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.6 diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java index 17fcb6e7..1d4ce246 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java @@ -134,6 +134,14 @@ default boolean supportsLanguageOfId() { return false; } + /** + * @return if the dictionary is an MSD. if so, {@link #getObjects()} can't + * be used and the {@link #getAllObjects()} method should be used. + */ + default boolean isMultiSectionDictionary() { + return false; + } + /** * Returns whether the dictionary supports graphs * diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java index f392f4a8..b40bc126 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java @@ -375,6 +375,11 @@ public void reset() { } } + @Override + public boolean isMultiSectionDictionary() { + return true; + } + @Override public OptimizedExtractor createOptimizedMapExtractor() { return new MultDictionaryPFCOptimizedExtractor(this); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java index ded6c74e..25853366 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java @@ -528,6 +528,11 @@ public OptimizedExtractor createOptimizedMapExtractor() { return new MultipleSectionDictionaryLangPFCOptimizedExtractor(this); } + @Override + public boolean isMultiSectionDictionary() { + return true; + } + public int getObjectsSectionCount() { return objectIdLocationsSec.length; } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java new file mode 100644 index 00000000..0a99cd8c --- /dev/null +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java @@ -0,0 +1,20 @@ +package com.the_qa_company.qendpoint.core.exceptions; + +import java.io.IOException; + +public class SignatureIOException extends IOException { + public SignatureIOException() { + } + + public SignatureIOException(String message) { + super(message); + } + + public SignatureIOException(String message, Throwable cause) { + super(message, cause); + } + + public SignatureIOException(Throwable cause) { + super(cause); + } +} diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java index f080ba8d..7d648148 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java @@ -4,7 +4,6 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.internal.Lists; import com.the_qa_company.qendpoint.core.dictionary.DictionarySection; -import com.the_qa_company.qendpoint.core.dictionary.impl.MultipleBaseDictionary; import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; import com.the_qa_company.qendpoint.core.hdt.HDT; import com.the_qa_company.qendpoint.core.hdt.HDTManager; @@ -237,7 +236,7 @@ public void exec() throws Throwable { try (HDT hdt = hdtl) { boolean error; long count = 0; - if (hdt.getDictionary() instanceof MultipleBaseDictionary) { + if (hdt.getDictionary().isMultiSectionDictionary()) { colorTool.log("Checking subject entries"); error = checkDictionarySectionOrder(binary, unicode, colorTool, "subject", hdt.getDictionary().getSubjects(), console); @@ -279,6 +278,12 @@ public void exec() throws Throwable { hdt.getDictionary().getShared(), console); count += hdt.getDictionary().getShared().getNumberOfElements(); } + if (hdt.getDictionary().supportGraphs()) { + colorTool.log("Checking graph entries"); + error |= checkDictionarySectionOrder(binary, unicode, colorTool, "graph", + hdt.getDictionary().getGraphs(), console); + count += hdt.getDictionary().getGraphs().getNumberOfElements(); + } if (error) { colorTool.error("This HDT isn't valid", true); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java index 58d2527c..2b9afacd 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java @@ -33,6 +33,7 @@ import com.the_qa_company.qendpoint.core.dictionary.Dictionary; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException; +import com.the_qa_company.qendpoint.core.exceptions.SignatureIOException; import com.the_qa_company.qendpoint.core.hdt.HDTVocabulary; import com.the_qa_company.qendpoint.core.hdt.impl.HDTDiskImporter; import com.the_qa_company.qendpoint.core.hdt.impl.diskindex.DiskIndexSort; @@ -1347,7 +1348,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene idx.getOrder()); } IOUtil.closeQuietly(old); - } catch (NoSuchFileException ignore) { + } catch (NoSuchFileException | SignatureIOException ignore) { // no index with this name if (!askedOrders.contains(order)) { continue; // not asked by the user, we can ignore diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java index 78f34936..597e011b 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java @@ -11,6 +11,7 @@ import com.the_qa_company.qendpoint.core.compact.sequence.SequenceLog64BigDisk; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException; +import com.the_qa_company.qendpoint.core.exceptions.SignatureIOException; import com.the_qa_company.qendpoint.core.iterator.utils.AsyncIteratorFetcher; import com.the_qa_company.qendpoint.core.iterator.utils.ExceptionIterator; import com.the_qa_company.qendpoint.core.iterator.utils.MapIterator; @@ -96,7 +97,8 @@ public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTripl long currentSignature = signature(triples); if (signature != currentSignature) { - throw new IOException(format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature)); + throw new SignatureIOException( + format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature)); } } diff --git a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java new file mode 100644 index 00000000..12fb7173 --- /dev/null +++ b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java @@ -0,0 +1,74 @@ +package com.the_qa_company.qendpoint.core.triples.impl; + +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.exceptions.ParserException; +import com.the_qa_company.qendpoint.core.hdt.HDTManager; +import com.the_qa_company.qendpoint.core.hdt.HDTVersion; +import com.the_qa_company.qendpoint.core.listener.ProgressListener; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys; +import com.the_qa_company.qendpoint.core.util.LargeFakeDataSetStreamSupplier; +import com.the_qa_company.qendpoint.core.util.crc.CRC32; +import org.apache.commons.io.file.PathUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class BitmapTriplesIndexFileTest { + + @Rule + public TemporaryFolder tempDir = TemporaryFolder.builder().assureDeletion().build(); + + public long crc32(byte[] data) { + CRC32 crc = new CRC32(); + crc.update(data, 0, data.length); + return crc.getValue(); + } + + @Test + public void genTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + HDTOptions spec = HDTOptions.of( + HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "spo,ops", + HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true + ); + try { + Path hdtPath = root.resolve("temp.hdt"); + + LargeFakeDataSetStreamSupplier supplier = LargeFakeDataSetStreamSupplier + .createSupplierWithMaxTriples(1000, 10) + .withMaxLiteralSize(50) + .withMaxElementSplit(20); + + supplier.createAndSaveFakeHDT(spec, hdtPath); + + // should load + HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore()).close(); + assertTrue("ops index doesn't exist", Files.exists(BitmapTriplesIndexFile.getIndexPath(hdtPath, TripleComponentOrder.OPS))); + assertFalse("foq index exists", Files.exists(hdtPath.resolveSibling(hdtPath.getFileName() + HDTVersion.get_index_suffix("-")))); + + long crcold = crc32(Files.readAllBytes(hdtPath)); + + Path hdtPath2 = root.resolve("temp2.hdt"); + + Files.move(hdtPath, hdtPath2); + + supplier.createAndSaveFakeHDT(spec, hdtPath); + // should erase the previous index and generate another one + HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore()).close(); + + long crcnew = crc32(Files.readAllBytes(hdtPath)); + + assertNotEquals("files are the same", crcold, crcnew); + } finally { + PathUtils.deleteDirectory(root); + } + } +} \ No newline at end of file diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index 09a2050c..87667ece 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.0 + 1.15.6 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.6 diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java index 03883ca3..3b839e84 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java @@ -11,16 +11,10 @@ import com.the_qa_company.qendpoint.utils.sail.SourceSailConnectionWrapper; import jakarta.json.Json; import jakarta.json.stream.JsonGenerator; -import org.eclipse.rdf4j.model.Literal; import org.eclipse.rdf4j.model.Namespace; import org.eclipse.rdf4j.model.Statement; -import org.eclipse.rdf4j.model.ValueFactory; -import org.eclipse.rdf4j.model.base.CoreDatatype; -import org.eclipse.rdf4j.model.impl.SimpleValueFactory; import org.eclipse.rdf4j.model.util.Values; import org.eclipse.rdf4j.query.*; -import org.eclipse.rdf4j.query.algebra.Var; -import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor; import org.eclipse.rdf4j.query.explanation.Explanation; import org.eclipse.rdf4j.query.explanation.GenericPlanNode; import org.eclipse.rdf4j.query.parser.*; @@ -104,9 +98,6 @@ public CompiledSailOptions getOptions() { /** * reindex all the lucene sails of this repository - * - * @throws Exception any exception returned by - * {@link org.eclipse.rdf4j.sail.lucene.LuceneSail#reindex()} */ public void reindexLuceneSails() { compiledSail.reindexLuceneSails(); @@ -759,10 +750,36 @@ private ClosableResult execute0(RepositoryConnection customConnection, String * @param out the output stream, can be null */ public void executeUpdate(String sparqlQuery, int timeout, OutputStream out) { + executeUpdate(sparqlQuery, timeout, out, null); + + } + + /** + * execute a sparql update query + * + * @param sparqlQuery the query + * @param timeout query timeout + * @param out the output stream, can be null + * @param customConnection custom connection to use + */ + public void executeUpdate(String sparqlQuery, int timeout, OutputStream out, + RepositoryConnection customConnection) { // logger.info("Running update query:"+sparqlQuery); sparqlQuery = applyPrefixes(sparqlQuery); sparqlQuery = Pattern.compile("MINUS \\{(?s).*?}\\n {2}}").matcher(sparqlQuery).replaceAll(""); - try (SailRepositoryConnection connection = repository.getConnection()) { + + RepositoryConnection connectionCloseable; + RepositoryConnection connection; + + if (customConnection == null) { + connection = repository.getConnection(); + connectionCloseable = connection; + } else { + connectionCloseable = null; + connection = customConnection; + } + + try (connectionCloseable) { connection.setParserConfig(new ParserConfig().set(BasicParserSettings.VERIFY_URI_SYNTAX, false)); Update preparedUpdate = connection.prepareUpdate(QueryLanguage.SPARQL, sparqlQuery); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java index 479ba76a..992bc8e5 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java @@ -85,6 +85,10 @@ public class EndpointStore extends AbstractNotifyingSail { * enable the merge join, default true */ public static final String OPTION_QENDPOINT_MERGE_JOIN = "qendpoint.mergejoin"; + /** + * disable delete bitmaps, default false + */ + public static final String OPTION_QENDPOINT_DELETE_DISABLE = "qendpoint.delete.disable"; private static final AtomicLong ENDPOINT_DEBUG_ID_GEN = new AtomicLong(); private static final Logger logger = LoggerFactory.getLogger(EndpointStore.class); private final long debugId; @@ -113,6 +117,7 @@ public class EndpointStore extends AbstractNotifyingSail { // setting to put the delete map only in memory, i.e don't write to disk private final boolean inMemDeletes; private final boolean loadIntoMemory; + private final boolean deleteDisabled; // bitmaps used to mark if the subject, predicate, object elements in HDT // are used in the rdf4j delta store @@ -175,6 +180,7 @@ public EndpointStore(EndpointFiles files, HDTOptions spec, boolean inMemDeletes, throws IOException { // load HDT file this.spec = (spec = HDTOptions.ofNullable(spec)); + deleteDisabled = spec.getBoolean(OPTION_QENDPOINT_DELETE_DISABLE, false); validOrders = getHDTSpec().getEnumSet(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, TripleComponentOrder.class); validOrders.add(TripleComponentOrder.SPO); // we need at least SPO @@ -1215,6 +1221,10 @@ public long getGraphsCount(HDT hdt) { return hdt.getDictionary().supportGraphs() ? hdt.getDictionary().getNgraphs() : 1; } + public boolean isDeleteDisabled() { + return deleteDisabled; + } + public long getGraphsCount() { return getGraphsCount(this.hdt); } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java index 3e2ef82d..8c512674 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java @@ -503,7 +503,8 @@ protected long sizeInternal(Resource... contexts) throws SailException { long sizeNativeB = connB_read.size(contexts); long sizeHdt = this.endpoint.getHdt().getTriples().getNumberOfElements(); - long sizeDeleted = this.endpoint.getDeleteBitMap(TripleComponentOrder.SPO).getHandle().countOnes(); + long sizeDeleted = endpoint.isDeleteDisabled() ? 0 + : this.endpoint.getDeleteBitMap(TripleComponentOrder.SPO).getHandle().countOnes(); logger.info("---------------------------"); logger.info("Size native A:" + sizeNativeA); logger.info("Size native B:" + sizeNativeB); @@ -519,6 +520,10 @@ public void removeStatement(UpdateContext op, Resource subj, IRI pred, Value obj if (MergeRunnableStopPoint.disableRequest) throw new MergeRunnableStopPoint.MergeRunnableException("connections request disabled"); + if (endpoint.isDeleteDisabled()) { + throw new SailException("This sail doesn't support deletion"); + } + isWriteConnection = true; Resource newSubj; @@ -585,6 +590,9 @@ private boolean tripleDoesntExistInHDT(TripleID tripleID) { // if iterator is empty then the given triple doesn't exist in HDT if (iter.hasNext()) { TripleID tid = iter.next(); + if (endpoint.isDeleteDisabled()) { + return false; + } long index = iter.getLastTriplePosition(); return this.endpoint .getDeleteBitMap( @@ -601,6 +609,9 @@ private boolean quadDoesntExistInHDT(TripleID tripleID) { // if iterator is empty then the given triple 't' doesn't exist in HDT if (iter.hasNext()) { TripleID tid = iter.next(); + if (endpoint.isDeleteDisabled()) { + return false; + } long index = iter.getLastTriplePosition(); return this.endpoint .getDeleteBitMap( @@ -612,6 +623,9 @@ private boolean quadDoesntExistInHDT(TripleID tripleID) { private void assignBitMapDeletes(TripleID tid, Resource subj, IRI pred, Value obj, Resource[] contexts, long[] contextIds) throws SailException { + if (endpoint.isDeleteDisabled()) { + throw new SailException("This endpoint doesn't support deletion"); + } long s = tid.getSubject(); long p = tid.getPredicate(); long o = tid.getObject(); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java index cd2e202a..b655d36c 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java @@ -1,7 +1,9 @@ package com.the_qa_company.qendpoint.store; +import com.the_qa_company.qendpoint.core.compact.bitmap.MultiLayerBitmapWrapper; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.store.exception.EndpointTimeoutException; +import com.the_qa_company.qendpoint.utils.BitArrayDisk; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.IndexReportingIterator; import org.eclipse.rdf4j.model.IRI; @@ -52,10 +54,11 @@ public boolean hasNext() { // iterate over the result of hdt while (iterator.hasNext()) { TripleID tripleID = iterator.next(); - long index = iterator.getLastTriplePosition(); TripleComponentOrder order = iterator.isLastTriplePositionBoundToOrder() ? iterator.getOrder() : TripleComponentOrder.SPO; - if (!endpoint.getDeleteBitMap(order).access(tripleID.isQuad() ? tripleID.getGraph() - 1 : 0, index)) { + MultiLayerBitmapWrapper.MultiLayerModBitmapWrapper dbm = endpoint.getDeleteBitMap(order); + if (endpoint.isDeleteDisabled() || dbm.getHandle().getMaxNumBits() == 0 + || !dbm.access(tripleID.isQuad() ? tripleID.getGraph() - 1 : 0, iterator.getLastTriplePosition())) { Resource subject = endpoint.getHdtConverter().idToSubjectHDTResource(tripleID.getSubject()); IRI predicate = endpoint.getHdtConverter().idToPredicateHDTResource(tripleID.getPredicate()); Value object = endpoint.getHdtConverter().idToObjectHDTResource(tripleID.getObject()); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java index e17e9352..efc35427 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java @@ -404,4 +404,11 @@ public String printInfo() { + (inMemory ? ", inMemory: true" : "\nfile: " + output.getFile().getAbsolutePath()) + (allBits <= 20 ? "\nbits: " + toString(true) : ""); } + + /** + * @return the maximum bit of this bitmap + */ + public long getMaxNumBits() { + return numbits; + } } diff --git a/release/RELEASE.md b/release/RELEASE.md index e2d421c8..185830a4 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1,2 +1 @@ -- add delete bitmap for multiple co-indexes (#440) - +- fix signature check on new other indexes (#458) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index 16cd8f64..c5d2d3e2 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,29 @@ +## Version 1.15.5 + +- disable deletes when not required (#455) +## Version 1.15.4 + +- add option to delete triple (#455) + +## Version 1.15.3 + +- add custom connection to SparqlRepository updates +- add support for MSDL/Graph dict in hdtVerify ci (#451) + +## Version 1.15.2 + +- HDTq store (#443) +- Add compiler config in CompiledSail builder (#448) + +## Version 1.15.1 + +- fix idx after merge +- merge join (#428) + +## Version 1.15.0 + +- add delete bitmap for multiple co-indexes (#440) + ## Version 1.14.1 - add hdt diffcat tool