From ab73967a7bac290244bcde08ee9f954523fcdee6 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Tue, 8 Oct 2024 11:19:21 +0200 Subject: [PATCH] GH-486 Add option to read v0 indexes --- .../core/options/HDTOptionsKeys.java | 3 + .../core/triples/impl/BitmapTriples.java | 5 +- .../triples/impl/BitmapTriplesIndexFile.java | 55 ++++++++++++++----- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java index 45a54c367..b9dad63ad 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java @@ -528,6 +528,9 @@ public class HDTOptionsKeys { @Key(type = Key.Type.STRING, desc = "Create other indexes in bitmaptriples pattern values (spo, ops, etc.), default none") public static final String BITMAPTRIPLES_INDEX_OTHERS = "bitmaptriples.index.others"; + @Key(type = Key.Type.BOOLEAN, desc = "Allow old other indexes, default false") + public static final String BITMAPTRIPLES_INDEX_ALLOW_OLD_OTHERS = "bitmaptriples.index.allowOldOthers"; + @Key(type = Key.Type.BOOLEAN, desc = "No FoQ index generation default false") public static final String BITMAPTRIPLES_INDEX_NO_FOQ = "bitmaptriples.index.noFoQ"; diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java index 2b9afacdb..16dd2b141 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java @@ -1316,6 +1316,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene } String otherIdxs = spec.get(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, ""); + boolean allowOldOthers = spec.getBoolean(HDTOptionsKeys.BITMAPTRIPLES_INDEX_ALLOW_OLD_OTHERS, false); Set askedOrders = Arrays.stream(otherIdxs.toUpperCase().split(",")).map(e -> { if (e.isEmpty() || e.equalsIgnoreCase(TripleComponentOrder.Unknown.name())) { @@ -1340,7 +1341,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) { // load from the path... - BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this); + BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this, allowOldOthers); BitmapTriplesIndex old = indexes.put(order, idx); indexesMask |= idx.getOrder().mask; if (old != null) { @@ -1357,7 +1358,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene BitmapTriplesIndexFile.generateIndex(this, subIndexPath, order, spec, mListener); try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) { // load from the path... - BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this); + BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this, allowOldOthers); BitmapTriplesIndex old = indexes.put(order, idx); indexesMask |= order.mask; if (old != null) { diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java index 597e011b4..ab9a6fa33 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java @@ -28,6 +28,8 @@ import com.the_qa_company.qendpoint.core.util.io.CountInputStream; import com.the_qa_company.qendpoint.core.util.io.IOUtil; import com.the_qa_company.qendpoint.core.util.listener.ListenerUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -50,6 +52,9 @@ * @author Antoine Willerval */ public class BitmapTriplesIndexFile implements BitmapTriplesIndex, Closeable { + + private static final Logger logger = LoggerFactory.getLogger(BitmapTriplesIndexFile.class); + /** * Get the path for an order for a hdt file * @@ -71,39 +76,61 @@ public static long signature(BitmapTriples triples) { return 0x484454802020L ^ triples.getNumberOfElements(); } - public static final byte[] MAGIC = "$HDTIDX1".getBytes(StandardCharsets.US_ASCII); + public static final String MAGIC_STR = "$HDTIDX1"; + public static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII); + public static final byte[] MAGIC_V0 = "$HDTIDX0".getBytes(StandardCharsets.US_ASCII); /** * Map a file from a file * - * @param file file - * @param channel channel - * @param triples triples + * @param file file + * @param channel channel + * @param triples triples + * @param allowOld allow old files * @return index * @throws IOException io */ - public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTriples triples) throws IOException { + public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTriples triples, boolean allowOld) + throws IOException { + + long headerSize = MAGIC.length; try (CloseMappedByteBuffer header = IOUtil.mapChannel(file, channel, FileChannel.MapMode.READ_ONLY, 0, MAGIC.length + 8)) { byte[] magicRead = new byte[MAGIC.length]; header.get(magicRead); - if (!Arrays.equals(magicRead, MAGIC)) { - throw new IOException(format("Can't read %s magic", file)); - } + if (Arrays.equals(magicRead, MAGIC)) { + headerSize += 8; // signature - long signature = header.order(ByteOrder.LITTLE_ENDIAN).getLong(magicRead.length); + long signature = header.order(ByteOrder.LITTLE_ENDIAN).getLong(magicRead.length); - long currentSignature = signature(triples); - if (signature != currentSignature) { - throw new SignatureIOException( - format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature)); + long currentSignature = signature(triples); + if (signature != currentSignature) { + throw new SignatureIOException( + format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature)); + } + + } else { + if (!allowOld) { + throw new IOException( + format("Invalid magic for %s: %s", file, new String(magicRead, StandardCharsets.US_ASCII))); + } + logger.warn("Reading {} with {}!={} magic", file, new String(magicRead, StandardCharsets.US_ASCII), + MAGIC_STR); + + if (Arrays.equals(magicRead, MAGIC_V0)) { + // reading v0 + logger.debug("Use v0 magic for {}", file); + } else { + throw new IOException( + format("Unknown magic for %s: %s", file, new String(magicRead, StandardCharsets.US_ASCII))); + } } } CountInputStream stream = new CountInputStream(new BufferedInputStream(Channels.newInputStream(channel))); - stream.skipNBytes(MAGIC.length + 8); + stream.skipNBytes(headerSize); String orderCfg = IOUtil.readSizedString(stream, ProgressListener.ignore());