Skip to content

Commit

Permalink
Merge pull request #487 from the-qa-company/GH-486-old-index-reading
Browse files Browse the repository at this point in the history 
GH-486 Add option to read v0 indexes
  • Loading branch information
@ate47
ate47 authored Oct 8, 2024
2 parents 93affe4 + ab73967 commit 167d749
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 16 deletions.
3 changes: 3 additions & 0 deletions qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptionsKeys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,9 @@ public class HDTOptionsKeys {
@Key(type = Key.Type.STRING, desc = "Create other indexes in bitmaptriples pattern values (spo, ops, etc.), default none")
public static final String BITMAPTRIPLES_INDEX_OTHERS = "bitmaptriples.index.others";

@Key(type = Key.Type.BOOLEAN, desc = "Allow old other indexes, default false")
public static final String BITMAPTRIPLES_INDEX_ALLOW_OLD_OTHERS = "bitmaptriples.index.allowOldOthers";

@Key(type = Key.Type.BOOLEAN, desc = "No FoQ index generation default false")
public static final String BITMAPTRIPLES_INDEX_NO_FOQ = "bitmaptriples.index.noFoQ";

Expand Down
5 changes: 3 additions & 2 deletions ...oint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1316,6 +1316,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
}

String otherIdxs = spec.get(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, "");
boolean allowOldOthers = spec.getBoolean(HDTOptionsKeys.BITMAPTRIPLES_INDEX_ALLOW_OLD_OTHERS, false);

Set<TripleComponentOrder> askedOrders = Arrays.stream(otherIdxs.toUpperCase().split(",")).map(e -> {
if (e.isEmpty() || e.equalsIgnoreCase(TripleComponentOrder.Unknown.name())) {
Expand All @@ -1340,7 +1341,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) {
// load from the path...

BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this);
BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this, allowOldOthers);
BitmapTriplesIndex old = indexes.put(order, idx);
indexesMask |= idx.getOrder().mask;
if (old != null) {
Expand All @@ -1357,7 +1358,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
BitmapTriplesIndexFile.generateIndex(this, subIndexPath, order, spec, mListener);
try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) {
// load from the path...
BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this);
BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this, allowOldOthers);
BitmapTriplesIndex old = indexes.put(order, idx);
indexesMask |= order.mask;
if (old != null) {
Expand Down
55 changes: 41 additions & 14 deletions .../src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import com.the_qa_company.qendpoint.core.util.io.CountInputStream;
import com.the_qa_company.qendpoint.core.util.io.IOUtil;
import com.the_qa_company.qendpoint.core.util.listener.ListenerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
Expand All @@ -50,6 +52,9 @@
* @author Antoine Willerval
*/
public class BitmapTriplesIndexFile implements BitmapTriplesIndex, Closeable {

private static final Logger logger = LoggerFactory.getLogger(BitmapTriplesIndexFile.class);

/**
* Get the path for an order for a hdt file
*
Expand All @@ -71,39 +76,61 @@ public static long signature(BitmapTriples triples) {
return 0x484454802020L ^ triples.getNumberOfElements();
}

public static final byte[] MAGIC = "$HDTIDX1".getBytes(StandardCharsets.US_ASCII);
public static final String MAGIC_STR = "$HDTIDX1";
public static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII);
public static final byte[] MAGIC_V0 = "$HDTIDX0".getBytes(StandardCharsets.US_ASCII);

/**
* Map a file from a file
*
* @param file file
* @param channel channel
* @param triples triples
* @param file file
* @param channel channel
* @param triples triples
* @param allowOld allow old files
* @return index
* @throws IOException io
*/
public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTriples triples) throws IOException {
public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTriples triples, boolean allowOld)
throws IOException {

long headerSize = MAGIC.length;
try (CloseMappedByteBuffer header = IOUtil.mapChannel(file, channel, FileChannel.MapMode.READ_ONLY, 0,
MAGIC.length + 8)) {
byte[] magicRead = new byte[MAGIC.length];

header.get(magicRead);

if (!Arrays.equals(magicRead, MAGIC)) {
throw new IOException(format("Can't read %s magic", file));
}
if (Arrays.equals(magicRead, MAGIC)) {
headerSize += 8; // signature

long signature = header.order(ByteOrder.LITTLE_ENDIAN).getLong(magicRead.length);
long signature = header.order(ByteOrder.LITTLE_ENDIAN).getLong(magicRead.length);

long currentSignature = signature(triples);
if (signature != currentSignature) {
throw new SignatureIOException(
format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature));
long currentSignature = signature(triples);
if (signature != currentSignature) {
throw new SignatureIOException(
format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature));
}

} else {
if (!allowOld) {
throw new IOException(
format("Invalid magic for %s: %s", file, new String(magicRead, StandardCharsets.US_ASCII)));
}
logger.warn("Reading {} with {}!={} magic", file, new String(magicRead, StandardCharsets.US_ASCII),
MAGIC_STR);

if (Arrays.equals(magicRead, MAGIC_V0)) {
// reading v0
logger.debug("Use v0 magic for {}", file);
} else {
throw new IOException(
format("Unknown magic for %s: %s", file, new String(magicRead, StandardCharsets.US_ASCII)));
}
}
}

CountInputStream stream = new CountInputStream(new BufferedInputStream(Channels.newInputStream(channel)));
stream.skipNBytes(MAGIC.length + 8);
stream.skipNBytes(headerSize);

String orderCfg = IOUtil.readSizedString(stream, ProgressListener.ignore());

Expand Down

0 comments on commit 167d749

Please sign in to comment.