From 3478bd36d25791c10799654a1103ac62985cd4c2 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Wed, 24 Jan 2024 17:09:06 +0100 Subject: [PATCH 01/12] prepare for version 1.15.1 --- pom.xml | 2 +- qendpoint-backend/pom.xml | 4 ++-- qendpoint-cli/pom.xml | 4 ++-- qendpoint-core/pom.xml | 4 ++-- qendpoint-store/pom.xml | 4 ++-- release/RELEASE.md | 3 ++- release/RELEASE.md_old | 4 ++++ 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 6ad2ca77..6d093bec 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.1 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index b53618e4..f0165a65 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.0 + 1.15.1 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.1 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index 067dbd8b..b7c26652 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.0 + 1.15.1 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.1 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index 22501494..3992e8e3 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.0 + 1.15.1 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.1 diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index 09a2050c..117551b4 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.0 + 1.15.1 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.0 + 1.15.1 diff --git a/release/RELEASE.md b/release/RELEASE.md index e2d421c8..f81481c4 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1,2 +1,3 @@ -- add delete bitmap for multiple co-indexes (#440) +- fix idx after merge +- merge join (#428) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index 16cd8f64..420e8b48 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,7 @@ +## Version 1.15.0 + +- add delete bitmap for multiple co-indexes (#440) + ## Version 1.14.1 - add hdt diffcat tool From 4e692c622e8c9c40202765522e6dc4ebd0e7ace8 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Thu, 8 Feb 2024 13:57:01 +0100 Subject: [PATCH 02/12] prepare for version 1.15.2 --- pom.xml | 2 +- qendpoint-backend/pom.xml | 4 ++-- qendpoint-cli/pom.xml | 4 ++-- qendpoint-core/pom.xml | 4 ++-- qendpoint-store/pom.xml | 4 ++-- release/RELEASE.md | 4 ++-- release/RELEASE.md_old | 5 +++++ 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 6d093bec..9cbd7b22 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.1 + 1.15.2 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index f0165a65..448829fd 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.1 + 1.15.2 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.1 + 1.15.2 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index b7c26652..4785ed0a 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.1 + 1.15.2 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.1 + 1.15.2 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index 3992e8e3..b2dff80a 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.1 + 1.15.2 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.1 + 1.15.2 diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index 117551b4..64106f33 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.1 + 1.15.2 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.1 + 1.15.2 diff --git a/release/RELEASE.md b/release/RELEASE.md index f81481c4..c0471e42 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1,3 +1,3 @@ -- fix idx after merge -- merge join (#428) +- HDTq store (#443) +- Add compiler config in CompiledSail builder (#448) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index 420e8b48..024dc2aa 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,8 @@ +## Version 1.15.1 + +- fix idx after merge +- merge join (#428) + ## Version 1.15.0 - add delete bitmap for multiple co-indexes (#440) From 9d04fd11c70c1e43e1b1a8cb5860b3385618fdd1 Mon Sep 17 00:00:00 2001 From: Dennis Diefenbach Date: Sat, 10 Feb 2024 17:17:29 +0100 Subject: [PATCH 03/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db0700b8..a00ccba6 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ The QA Company over the social networks ## About The qEndpoint is a highly scalable triple store with full-text and [GeoSPARQL](https://www.ogc.org/standards/geosparql) support. It can be used as a standalone SPARQL endpoint, or as a dependency. -The qEndpoint is for example used in [Kohesio](https://kohesio.ec.europa.eu/) where each interaction with the UI corresponds to an underlying SPARQL query on the qEndpoint. +The qEndpoint is for example used in [Kohesio](https://kohesio.ec.europa.eu/) where each interaction with the UI corresponds to an underlying SPARQL query on the qEndpoint. Also qEndpoint is part of [QAnswer](https://qanswer.ai) enabeling question answering over RDF Graphs. ### Built With From 41a94adc916f6277da95e73bb711a6e9d01ceab9 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Mon, 19 Feb 2024 14:54:05 +0100 Subject: [PATCH 04/12] add custom connection to SparqlRepository update --- .../qendpoint/compiler/SparqlRepository.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java index 03883ca3..2819c09c 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java @@ -759,10 +759,34 @@ private ClosableResult execute0(RepositoryConnection customConnection, String * @param out the output stream, can be null */ public void executeUpdate(String sparqlQuery, int timeout, OutputStream out) { + executeUpdate(sparqlQuery, timeout, out, null); + + } + /** + * execute a sparql update query + * + * @param sparqlQuery the query + * @param timeout query timeout + * @param out the output stream, can be null + * @param customConnection custom connection to use + */ + public void executeUpdate(String sparqlQuery, int timeout, OutputStream out, RepositoryConnection customConnection) { // logger.info("Running update query:"+sparqlQuery); sparqlQuery = applyPrefixes(sparqlQuery); sparqlQuery = Pattern.compile("MINUS \\{(?s).*?}\\n {2}}").matcher(sparqlQuery).replaceAll(""); - try (SailRepositoryConnection connection = repository.getConnection()) { + + RepositoryConnection connectionCloseable; + RepositoryConnection connection; + + if (customConnection == null) { + connection = repository.getConnection(); + connectionCloseable = connection; + } else { + connectionCloseable = null; + connection = customConnection; + } + + try (connectionCloseable) { connection.setParserConfig(new ParserConfig().set(BasicParserSettings.VERIFY_URI_SYNTAX, false)); Update preparedUpdate = connection.prepareUpdate(QueryLanguage.SPARQL, sparqlQuery); From 44a47c42fe8d8cb889398ce2f685af794207911b Mon Sep 17 00:00:00 2001 From: qaate47 Date: Tue, 20 Feb 2024 14:13:47 +0100 Subject: [PATCH 05/12] GH-451 Add support for MSDL/GRAPH Dictionary in HDT verify --- .../qendpoint/core/dictionary/Dictionary.java | 8 ++++++++ .../impl/MultipleBaseDictionary.java | 5 +++++ .../impl/MultipleLangBaseDictionary.java | 5 +++++ .../qendpoint/core/tools/HDTVerify.java | 9 +++++++-- .../qendpoint/compiler/SparqlRepository.java | 19 ++++++------------- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java index 17fcb6e7..1d4ce246 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/Dictionary.java @@ -134,6 +134,14 @@ default boolean supportsLanguageOfId() { return false; } + /** + * @return if the dictionary is an MSD. if so, {@link #getObjects()} can't + * be used and the {@link #getAllObjects()} method should be used. + */ + default boolean isMultiSectionDictionary() { + return false; + } + /** * Returns whether the dictionary supports graphs * diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java index f392f4a8..b40bc126 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleBaseDictionary.java @@ -375,6 +375,11 @@ public void reset() { } } + @Override + public boolean isMultiSectionDictionary() { + return true; + } + @Override public OptimizedExtractor createOptimizedMapExtractor() { return new MultDictionaryPFCOptimizedExtractor(this); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java index ded6c74e..25853366 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java @@ -528,6 +528,11 @@ public OptimizedExtractor createOptimizedMapExtractor() { return new MultipleSectionDictionaryLangPFCOptimizedExtractor(this); } + @Override + public boolean isMultiSectionDictionary() { + return true; + } + public int getObjectsSectionCount() { return objectIdLocationsSec.length; } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java index f080ba8d..7d648148 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/tools/HDTVerify.java @@ -4,7 +4,6 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.internal.Lists; import com.the_qa_company.qendpoint.core.dictionary.DictionarySection; -import com.the_qa_company.qendpoint.core.dictionary.impl.MultipleBaseDictionary; import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; import com.the_qa_company.qendpoint.core.hdt.HDT; import com.the_qa_company.qendpoint.core.hdt.HDTManager; @@ -237,7 +236,7 @@ public void exec() throws Throwable { try (HDT hdt = hdtl) { boolean error; long count = 0; - if (hdt.getDictionary() instanceof MultipleBaseDictionary) { + if (hdt.getDictionary().isMultiSectionDictionary()) { colorTool.log("Checking subject entries"); error = checkDictionarySectionOrder(binary, unicode, colorTool, "subject", hdt.getDictionary().getSubjects(), console); @@ -279,6 +278,12 @@ public void exec() throws Throwable { hdt.getDictionary().getShared(), console); count += hdt.getDictionary().getShared().getNumberOfElements(); } + if (hdt.getDictionary().supportGraphs()) { + colorTool.log("Checking graph entries"); + error |= checkDictionarySectionOrder(binary, unicode, colorTool, "graph", + hdt.getDictionary().getGraphs(), console); + count += hdt.getDictionary().getGraphs().getNumberOfElements(); + } if (error) { colorTool.error("This HDT isn't valid", true); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java index 2819c09c..3b839e84 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SparqlRepository.java @@ -11,16 +11,10 @@ import com.the_qa_company.qendpoint.utils.sail.SourceSailConnectionWrapper; import jakarta.json.Json; import jakarta.json.stream.JsonGenerator; -import org.eclipse.rdf4j.model.Literal; import org.eclipse.rdf4j.model.Namespace; import org.eclipse.rdf4j.model.Statement; -import org.eclipse.rdf4j.model.ValueFactory; -import org.eclipse.rdf4j.model.base.CoreDatatype; -import org.eclipse.rdf4j.model.impl.SimpleValueFactory; import org.eclipse.rdf4j.model.util.Values; import org.eclipse.rdf4j.query.*; -import org.eclipse.rdf4j.query.algebra.Var; -import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor; import org.eclipse.rdf4j.query.explanation.Explanation; import org.eclipse.rdf4j.query.explanation.GenericPlanNode; import org.eclipse.rdf4j.query.parser.*; @@ -104,9 +98,6 @@ public CompiledSailOptions getOptions() { /** * reindex all the lucene sails of this repository - * - * @throws Exception any exception returned by - * {@link org.eclipse.rdf4j.sail.lucene.LuceneSail#reindex()} */ public void reindexLuceneSails() { compiledSail.reindexLuceneSails(); @@ -762,15 +753,17 @@ public void executeUpdate(String sparqlQuery, int timeout, OutputStream out) { executeUpdate(sparqlQuery, timeout, out, null); } + /** * execute a sparql update query * - * @param sparqlQuery the query - * @param timeout query timeout - * @param out the output stream, can be null + * @param sparqlQuery the query + * @param timeout query timeout + * @param out the output stream, can be null * @param customConnection custom connection to use */ - public void executeUpdate(String sparqlQuery, int timeout, OutputStream out, RepositoryConnection customConnection) { + public void executeUpdate(String sparqlQuery, int timeout, OutputStream out, + RepositoryConnection customConnection) { // logger.info("Running update query:"+sparqlQuery); sparqlQuery = applyPrefixes(sparqlQuery); sparqlQuery = Pattern.compile("MINUS \\{(?s).*?}\\n {2}}").matcher(sparqlQuery).replaceAll(""); From f68dff8d43e1126687894c2ee872ecc3fdf65078 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Tue, 20 Feb 2024 14:16:45 +0100 Subject: [PATCH 06/12] prepare for version 1.15.3 --- pom.xml | 2 +- qendpoint-backend/pom.xml | 4 ++-- qendpoint-cli/pom.xml | 4 ++-- qendpoint-core/pom.xml | 4 ++-- qendpoint-store/pom.xml | 4 ++-- release/RELEASE.md | 4 ++-- release/RELEASE.md_old | 5 +++++ 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 9cbd7b22..4aa72b41 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.2 + 1.15.3 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index 448829fd..65be74fd 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.2 + 1.15.3 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.2 + 1.15.3 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index 4785ed0a..fd774e20 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.2 + 1.15.3 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.2 + 1.15.3 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index b2dff80a..0e95248e 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.2 + 1.15.3 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.2 + 1.15.3 diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index 64106f33..ff2fafab 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.2 + 1.15.3 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.2 + 1.15.3 diff --git a/release/RELEASE.md b/release/RELEASE.md index c0471e42..3928bf4e 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1,3 +1,3 @@ -- HDTq store (#443) -- Add compiler config in CompiledSail builder (#448) +- add custom connection to SparqlRepository updates +- add support for MSDL/Graph dict in hdtVerify ci (#451) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index 024dc2aa..99b4c547 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,8 @@ +## Version 1.15.2 + +- HDTq store (#443) +- Add compiler config in CompiledSail builder (#448) + ## Version 1.15.1 - fix idx after merge From 9e789aed8f7c40d040b9845229f4dc51e63c3dd6 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Fri, 1 Mar 2024 12:18:54 +0100 Subject: [PATCH 07/12] GH-455 Disable delete bitmap feature --- .../qendpoint/store/EndpointStore.java | 10 ++++++++++ .../qendpoint/store/EndpointStoreConnection.java | 16 +++++++++++++++- .../store/EndpointStoreTripleIterator.java | 10 ++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java index 479ba76a..992bc8e5 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java @@ -85,6 +85,10 @@ public class EndpointStore extends AbstractNotifyingSail { * enable the merge join, default true */ public static final String OPTION_QENDPOINT_MERGE_JOIN = "qendpoint.mergejoin"; + /** + * disable delete bitmaps, default false + */ + public static final String OPTION_QENDPOINT_DELETE_DISABLE = "qendpoint.delete.disable"; private static final AtomicLong ENDPOINT_DEBUG_ID_GEN = new AtomicLong(); private static final Logger logger = LoggerFactory.getLogger(EndpointStore.class); private final long debugId; @@ -113,6 +117,7 @@ public class EndpointStore extends AbstractNotifyingSail { // setting to put the delete map only in memory, i.e don't write to disk private final boolean inMemDeletes; private final boolean loadIntoMemory; + private final boolean deleteDisabled; // bitmaps used to mark if the subject, predicate, object elements in HDT // are used in the rdf4j delta store @@ -175,6 +180,7 @@ public EndpointStore(EndpointFiles files, HDTOptions spec, boolean inMemDeletes, throws IOException { // load HDT file this.spec = (spec = HDTOptions.ofNullable(spec)); + deleteDisabled = spec.getBoolean(OPTION_QENDPOINT_DELETE_DISABLE, false); validOrders = getHDTSpec().getEnumSet(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, TripleComponentOrder.class); validOrders.add(TripleComponentOrder.SPO); // we need at least SPO @@ -1215,6 +1221,10 @@ public long getGraphsCount(HDT hdt) { return hdt.getDictionary().supportGraphs() ? hdt.getDictionary().getNgraphs() : 1; } + public boolean isDeleteDisabled() { + return deleteDisabled; + } + public long getGraphsCount() { return getGraphsCount(this.hdt); } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java index 3e2ef82d..8c512674 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java @@ -503,7 +503,8 @@ protected long sizeInternal(Resource... contexts) throws SailException { long sizeNativeB = connB_read.size(contexts); long sizeHdt = this.endpoint.getHdt().getTriples().getNumberOfElements(); - long sizeDeleted = this.endpoint.getDeleteBitMap(TripleComponentOrder.SPO).getHandle().countOnes(); + long sizeDeleted = endpoint.isDeleteDisabled() ? 0 + : this.endpoint.getDeleteBitMap(TripleComponentOrder.SPO).getHandle().countOnes(); logger.info("---------------------------"); logger.info("Size native A:" + sizeNativeA); logger.info("Size native B:" + sizeNativeB); @@ -519,6 +520,10 @@ public void removeStatement(UpdateContext op, Resource subj, IRI pred, Value obj if (MergeRunnableStopPoint.disableRequest) throw new MergeRunnableStopPoint.MergeRunnableException("connections request disabled"); + if (endpoint.isDeleteDisabled()) { + throw new SailException("This sail doesn't support deletion"); + } + isWriteConnection = true; Resource newSubj; @@ -585,6 +590,9 @@ private boolean tripleDoesntExistInHDT(TripleID tripleID) { // if iterator is empty then the given triple doesn't exist in HDT if (iter.hasNext()) { TripleID tid = iter.next(); + if (endpoint.isDeleteDisabled()) { + return false; + } long index = iter.getLastTriplePosition(); return this.endpoint .getDeleteBitMap( @@ -601,6 +609,9 @@ private boolean quadDoesntExistInHDT(TripleID tripleID) { // if iterator is empty then the given triple 't' doesn't exist in HDT if (iter.hasNext()) { TripleID tid = iter.next(); + if (endpoint.isDeleteDisabled()) { + return false; + } long index = iter.getLastTriplePosition(); return this.endpoint .getDeleteBitMap( @@ -612,6 +623,9 @@ private boolean quadDoesntExistInHDT(TripleID tripleID) { private void assignBitMapDeletes(TripleID tid, Resource subj, IRI pred, Value obj, Resource[] contexts, long[] contextIds) throws SailException { + if (endpoint.isDeleteDisabled()) { + throw new SailException("This endpoint doesn't support deletion"); + } long s = tid.getSubject(); long p = tid.getPredicate(); long o = tid.getObject(); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java index cd2e202a..2427cea5 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java @@ -52,10 +52,16 @@ public boolean hasNext() { // iterate over the result of hdt while (iterator.hasNext()) { TripleID tripleID = iterator.next(); - long index = iterator.getLastTriplePosition(); + long index; + if (endpoint.isDeleteDisabled()) { + index = -1; + } else { + index = iterator.getLastTriplePosition(); + } TripleComponentOrder order = iterator.isLastTriplePositionBoundToOrder() ? iterator.getOrder() : TripleComponentOrder.SPO; - if (!endpoint.getDeleteBitMap(order).access(tripleID.isQuad() ? tripleID.getGraph() - 1 : 0, index)) { + if (endpoint.isDeleteDisabled() || !endpoint.getDeleteBitMap(order) + .access(tripleID.isQuad() ? tripleID.getGraph() - 1 : 0, index)) { Resource subject = endpoint.getHdtConverter().idToSubjectHDTResource(tripleID.getSubject()); IRI predicate = endpoint.getHdtConverter().idToPredicateHDTResource(tripleID.getPredicate()); Value object = endpoint.getHdtConverter().idToObjectHDTResource(tripleID.getObject()); From e42e7f655bcaf292046c82313d8b92bd62e6f378 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Fri, 1 Mar 2024 13:31:24 +0100 Subject: [PATCH 08/12] prepare for version 1.15.4 --- pom.xml | 2 +- qendpoint-backend/pom.xml | 4 ++-- qendpoint-cli/pom.xml | 4 ++-- qendpoint-core/pom.xml | 4 ++-- qendpoint-store/pom.xml | 4 ++-- release/RELEASE.md | 3 +-- release/RELEASE.md_old | 5 +++++ 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 4aa72b41..2376b4ec 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.3 + 1.15.4 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index 65be74fd..c30443b2 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.3 + 1.15.4 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.3 + 1.15.4 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index fd774e20..379ecba9 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.3 + 1.15.4 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.3 + 1.15.4 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index 0e95248e..75f1e463 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.3 + 1.15.4 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.3 + 1.15.4 diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index ff2fafab..cfa901fd 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.3 + 1.15.4 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.3 + 1.15.4 diff --git a/release/RELEASE.md b/release/RELEASE.md index 3928bf4e..0d565fb3 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1,3 +1,2 @@ -- add custom connection to SparqlRepository updates -- add support for MSDL/Graph dict in hdtVerify ci (#451) +- add option to delete triple (#455) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index 99b4c547..adcc378d 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,8 @@ +## Version 1.15.3 + +- add custom connection to SparqlRepository updates +- add support for MSDL/Graph dict in hdtVerify ci (#451) + ## Version 1.15.2 - HDTq store (#443) From c760e9387b393cac06e35f0d8e4b00436adba496 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Fri, 1 Mar 2024 15:31:31 +0100 Subject: [PATCH 09/12] GH-455 Disable delete bitmap when not required --- .../store/EndpointStoreTripleIterator.java | 13 +++++-------- .../qendpoint/utils/BitArrayDisk.java | 7 +++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java index 2427cea5..b655d36c 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java @@ -1,7 +1,9 @@ package com.the_qa_company.qendpoint.store; +import com.the_qa_company.qendpoint.core.compact.bitmap.MultiLayerBitmapWrapper; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.store.exception.EndpointTimeoutException; +import com.the_qa_company.qendpoint.utils.BitArrayDisk; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.IndexReportingIterator; import org.eclipse.rdf4j.model.IRI; @@ -52,16 +54,11 @@ public boolean hasNext() { // iterate over the result of hdt while (iterator.hasNext()) { TripleID tripleID = iterator.next(); - long index; - if (endpoint.isDeleteDisabled()) { - index = -1; - } else { - index = iterator.getLastTriplePosition(); - } TripleComponentOrder order = iterator.isLastTriplePositionBoundToOrder() ? iterator.getOrder() : TripleComponentOrder.SPO; - if (endpoint.isDeleteDisabled() || !endpoint.getDeleteBitMap(order) - .access(tripleID.isQuad() ? tripleID.getGraph() - 1 : 0, index)) { + MultiLayerBitmapWrapper.MultiLayerModBitmapWrapper dbm = endpoint.getDeleteBitMap(order); + if (endpoint.isDeleteDisabled() || dbm.getHandle().getMaxNumBits() == 0 + || !dbm.access(tripleID.isQuad() ? tripleID.getGraph() - 1 : 0, iterator.getLastTriplePosition())) { Resource subject = endpoint.getHdtConverter().idToSubjectHDTResource(tripleID.getSubject()); IRI predicate = endpoint.getHdtConverter().idToPredicateHDTResource(tripleID.getPredicate()); Value object = endpoint.getHdtConverter().idToObjectHDTResource(tripleID.getObject()); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java index e17e9352..efc35427 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/utils/BitArrayDisk.java @@ -404,4 +404,11 @@ public String printInfo() { + (inMemory ? ", inMemory: true" : "\nfile: " + output.getFile().getAbsolutePath()) + (allBits <= 20 ? "\nbits: " + toString(true) : ""); } + + /** + * @return the maximum bit of this bitmap + */ + public long getMaxNumBits() { + return numbits; + } } From ade72545934cc9c5874a7b8e04cdae17be7f9eef Mon Sep 17 00:00:00 2001 From: qaate47 Date: Fri, 1 Mar 2024 15:36:42 +0100 Subject: [PATCH 10/12] prepare for version 1.15.5 --- pom.xml | 2 +- qendpoint-backend/pom.xml | 4 ++-- qendpoint-cli/pom.xml | 4 ++-- qendpoint-core/pom.xml | 4 ++-- qendpoint-store/pom.xml | 4 ++-- release/RELEASE.md | 3 +-- release/RELEASE.md_old | 4 ++++ 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 2376b4ec..8e99acc3 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.4 + 1.15.5 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index c30443b2..a9ef5f50 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.4 + 1.15.5 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.4 + 1.15.5 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index 379ecba9..d2dc31be 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.4 + 1.15.5 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.4 + 1.15.5 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index 75f1e463..7330bf40 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.4 + 1.15.5 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.4 + 1.15.5 diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index cfa901fd..3e480945 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.4 + 1.15.5 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.4 + 1.15.5 diff --git a/release/RELEASE.md b/release/RELEASE.md index 0d565fb3..08d7bb2f 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1,2 +1 @@ -- add option to delete triple (#455) - +- disable deletes when not required (#455) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index adcc378d..494ce9d2 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,7 @@ +## Version 1.15.4 + +- add option to delete triple (#455) + ## Version 1.15.3 - add custom connection to SparqlRepository updates From faeccae06428167c7f676d7abd1bb193c06fdf4b Mon Sep 17 00:00:00 2001 From: qaate47 Date: Mon, 4 Mar 2024 14:24:30 +0100 Subject: [PATCH 11/12] GH-458 Regenerate index if signature is wrong --- .../core/exceptions/SignatureIOException.java | 20 +++++ .../core/triples/impl/BitmapTriples.java | 3 +- .../triples/impl/BitmapTriplesIndexFile.java | 4 +- .../impl/BitmapTriplesIndexFileTest.java | 74 +++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java create mode 100644 qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java new file mode 100644 index 00000000..0a99cd8c --- /dev/null +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/exceptions/SignatureIOException.java @@ -0,0 +1,20 @@ +package com.the_qa_company.qendpoint.core.exceptions; + +import java.io.IOException; + +public class SignatureIOException extends IOException { + public SignatureIOException() { + } + + public SignatureIOException(String message) { + super(message); + } + + public SignatureIOException(String message, Throwable cause) { + super(message, cause); + } + + public SignatureIOException(Throwable cause) { + super(cause); + } +} diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java index 58d2527c..2b9afacd 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java @@ -33,6 +33,7 @@ import com.the_qa_company.qendpoint.core.dictionary.Dictionary; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException; +import com.the_qa_company.qendpoint.core.exceptions.SignatureIOException; import com.the_qa_company.qendpoint.core.hdt.HDTVocabulary; import com.the_qa_company.qendpoint.core.hdt.impl.HDTDiskImporter; import com.the_qa_company.qendpoint.core.hdt.impl.diskindex.DiskIndexSort; @@ -1347,7 +1348,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene idx.getOrder()); } IOUtil.closeQuietly(old); - } catch (NoSuchFileException ignore) { + } catch (NoSuchFileException | SignatureIOException ignore) { // no index with this name if (!askedOrders.contains(order)) { continue; // not asked by the user, we can ignore diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java index 78f34936..597e011b 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java @@ -11,6 +11,7 @@ import com.the_qa_company.qendpoint.core.compact.sequence.SequenceLog64BigDisk; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException; +import com.the_qa_company.qendpoint.core.exceptions.SignatureIOException; import com.the_qa_company.qendpoint.core.iterator.utils.AsyncIteratorFetcher; import com.the_qa_company.qendpoint.core.iterator.utils.ExceptionIterator; import com.the_qa_company.qendpoint.core.iterator.utils.MapIterator; @@ -96,7 +97,8 @@ public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTripl long currentSignature = signature(triples); if (signature != currentSignature) { - throw new IOException(format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature)); + throw new SignatureIOException( + format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature)); } } diff --git a/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java new file mode 100644 index 00000000..12fb7173 --- /dev/null +++ b/qendpoint-core/src/test/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFileTest.java @@ -0,0 +1,74 @@ +package com.the_qa_company.qendpoint.core.triples.impl; + +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.exceptions.ParserException; +import com.the_qa_company.qendpoint.core.hdt.HDTManager; +import com.the_qa_company.qendpoint.core.hdt.HDTVersion; +import com.the_qa_company.qendpoint.core.listener.ProgressListener; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys; +import com.the_qa_company.qendpoint.core.util.LargeFakeDataSetStreamSupplier; +import com.the_qa_company.qendpoint.core.util.crc.CRC32; +import org.apache.commons.io.file.PathUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class BitmapTriplesIndexFileTest { + + @Rule + public TemporaryFolder tempDir = TemporaryFolder.builder().assureDeletion().build(); + + public long crc32(byte[] data) { + CRC32 crc = new CRC32(); + crc.update(data, 0, data.length); + return crc.getValue(); + } + + @Test + public void genTest() throws IOException, ParserException { + Path root = tempDir.newFolder().toPath(); + + HDTOptions spec = HDTOptions.of( + HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "spo,ops", + HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, true + ); + try { + Path hdtPath = root.resolve("temp.hdt"); + + LargeFakeDataSetStreamSupplier supplier = LargeFakeDataSetStreamSupplier + .createSupplierWithMaxTriples(1000, 10) + .withMaxLiteralSize(50) + .withMaxElementSplit(20); + + supplier.createAndSaveFakeHDT(spec, hdtPath); + + // should load + HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore()).close(); + assertTrue("ops index doesn't exist", Files.exists(BitmapTriplesIndexFile.getIndexPath(hdtPath, TripleComponentOrder.OPS))); + assertFalse("foq index exists", Files.exists(hdtPath.resolveSibling(hdtPath.getFileName() + HDTVersion.get_index_suffix("-")))); + + long crcold = crc32(Files.readAllBytes(hdtPath)); + + Path hdtPath2 = root.resolve("temp2.hdt"); + + Files.move(hdtPath, hdtPath2); + + supplier.createAndSaveFakeHDT(spec, hdtPath); + // should erase the previous index and generate another one + HDTManager.mapIndexedHDT(hdtPath, spec, ProgressListener.ignore()).close(); + + long crcnew = crc32(Files.readAllBytes(hdtPath)); + + assertNotEquals("files are the same", crcold, crcnew); + } finally { + PathUtils.deleteDirectory(root); + } + } +} \ No newline at end of file From 32b7a92cbd398772a2fd464badbe5587a807593f Mon Sep 17 00:00:00 2001 From: qaate47 Date: Mon, 4 Mar 2024 14:44:40 +0100 Subject: [PATCH 12/12] prepare for version 1.15.6 --- pom.xml | 2 +- qendpoint-backend/pom.xml | 4 ++-- qendpoint-cli/pom.xml | 4 ++-- qendpoint-core/pom.xml | 4 ++-- qendpoint-store/pom.xml | 4 ++-- release/RELEASE.md | 2 +- release/RELEASE.md_old | 3 +++ 7 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 8e99acc3..1c97efae 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ com.the-qa-company qendpoint-parent - 1.15.5 + 1.15.6 pom diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml index a9ef5f50..331680f5 100644 --- a/qendpoint-backend/pom.xml +++ b/qendpoint-backend/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-backend - 1.15.5 + 1.15.6 jar @@ -15,7 +15,7 @@ com.the-qa-company qendpoint-parent - 1.15.5 + 1.15.6 diff --git a/qendpoint-cli/pom.xml b/qendpoint-cli/pom.xml index d2dc31be..b90f8f81 100644 --- a/qendpoint-cli/pom.xml +++ b/qendpoint-cli/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 qendpoint-cli - 1.15.5 + 1.15.6 qendpoint package Package of the qendpoint. @@ -11,7 +11,7 @@ com.the-qa-company qendpoint-parent - 1.15.5 + 1.15.6 diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml index 7330bf40..c847f459 100644 --- a/qendpoint-core/pom.xml +++ b/qendpoint-core/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint-core - 1.15.5 + 1.15.6 jar @@ -27,7 +27,7 @@ com.the-qa-company qendpoint-parent - 1.15.5 + 1.15.6 diff --git a/qendpoint-store/pom.xml b/qendpoint-store/pom.xml index 3e480945..87667ece 100644 --- a/qendpoint-store/pom.xml +++ b/qendpoint-store/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qendpoint - 1.15.5 + 1.15.6 jar @@ -13,7 +13,7 @@ com.the-qa-company qendpoint-parent - 1.15.5 + 1.15.6 diff --git a/release/RELEASE.md b/release/RELEASE.md index 08d7bb2f..185830a4 100644 --- a/release/RELEASE.md +++ b/release/RELEASE.md @@ -1 +1 @@ -- disable deletes when not required (#455) +- fix signature check on new other indexes (#458) diff --git a/release/RELEASE.md_old b/release/RELEASE.md_old index 494ce9d2..c5d2d3e2 100644 --- a/release/RELEASE.md_old +++ b/release/RELEASE.md_old @@ -1,3 +1,6 @@ +## Version 1.15.5 + +- disable deletes when not required (#455) ## Version 1.15.4 - add option to delete triple (#455)