From 85fb67b30e32408e7c463f9275dcce7a44bbe420 Mon Sep 17 00:00:00 2001 From: David Ehrmann Date: Sun, 28 Jan 2018 19:03:58 -0800 Subject: [PATCH] Add support for ByteBuffer dictionaries --- .../davidehrmann/vcdiff/VCDiffDecoder.java | 22 ++++- .../vcdiff/VCDiffDecoderBuilder.java | 5 ++ .../vcdiff/VCDiffStreamingDecoder.java | 26 +++++- .../vcdiff/engine/VCDiffDeltaFileWindow.java | 64 +++++++------- .../engine/VCDiffStreamingDecoderImpl.java | 83 ++++++++++--------- .../vcdiff/io/VCDiffInputStream.java | 25 ++++-- 6 files changed, 139 insertions(+), 86 deletions(-) diff --git a/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoder.java b/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoder.java index 379c270..5bc8b98 100755 --- a/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoder.java +++ b/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoder.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; /** * A simpler (non-streaming) interface to the VCDIFF decoder that can be used @@ -17,6 +18,8 @@ public VCDiffDecoder(VCDiffStreamingDecoder decoder) { } /** + * @deprecated use {@link #decode(byte[], byte[], OutputStream)} + * * decode the contents of encoding using the specified dictionary, writing the decoded data to target * * @param dictionary dictionary @@ -26,14 +29,27 @@ public VCDiffDecoder(VCDiffStreamingDecoder decoder) { * @param target output writer for decoded data * @throws IOException if there was an exception decoding or writing to the output target */ + @Deprecated public void decode(byte[] dictionary, byte[] encoding, int offset, int len, OutputStream target) throws IOException { + decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding, offset, len), target); + } + + /** + * decode the contents of encoding using the specified dictionary, writing the decoded data to target + * + * @param dictionary dictionary + * @param encoding data to decode + * @param target output writer for decoded data + * @throws IOException if there was an exception decoding or writing to the output target + */ + public void decode(ByteBuffer dictionary, ByteBuffer encoding, OutputStream target) throws IOException { decoder.startDecoding(dictionary); - decoder.decodeChunk(encoding, offset, len, target); + decoder.decodeChunk(encoding, target); decoder.finishDecoding(); } /** - * Convenience method equivalent to decode(dictionary, encoding, 0, encoding.length, target) + * Convenience method equivalent to decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding), target) * * @param dictionary dictionary * @param encoding data to decode @@ -41,6 +57,6 @@ public void decode(byte[] dictionary, byte[] encoding, int offset, int len, Outp * @throws IOException if there was an exception decoding or writing to the output target */ public void decode(byte[] dictionary, byte[] encoding, OutputStream target) throws IOException { - decode(dictionary, encoding, 0, encoding.length, target); + decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding), target); } } diff --git a/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoderBuilder.java b/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoderBuilder.java index 524e968..e2b5f34 100755 --- a/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoderBuilder.java +++ b/core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoderBuilder.java @@ -25,6 +25,7 @@ import com.davidehrmann.vcdiff.io.VCDiffInputStream; import java.io.InputStream; +import java.nio.ByteBuffer; public class VCDiffDecoderBuilder { @@ -73,6 +74,10 @@ public synchronized VCDiffStreamingDecoder buildStreaming(VCDiffStreamingDecoder } public VCDiffInputStream buildInputStream(InputStream in, byte[] dictionary) { + return buildInputStream(in, ByteBuffer.wrap(dictionary)); + } + + public VCDiffInputStream buildInputStream(InputStream in, ByteBuffer dictionary) { return new VCDiffInputStream(in, dictionary, buildStreaming()); } diff --git a/core/src/main/java/com/davidehrmann/vcdiff/VCDiffStreamingDecoder.java b/core/src/main/java/com/davidehrmann/vcdiff/VCDiffStreamingDecoder.java index c0f3d90..2abfa7c 100755 --- a/core/src/main/java/com/davidehrmann/vcdiff/VCDiffStreamingDecoder.java +++ b/core/src/main/java/com/davidehrmann/vcdiff/VCDiffStreamingDecoder.java @@ -17,10 +17,17 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; public interface VCDiffStreamingDecoder { + /** + * @deprecated use {@link #startDecoding(ByteBuffer)} + */ + @Deprecated + void startDecoding(byte[] dictionary); + /** * Resets the dictionary to dictionary parameter * and sets up the data structures for decoding. Note that the dictionary @@ -29,9 +36,11 @@ public interface VCDiffStreamingDecoder { * * @param dictionary dictionary the decoder is initialized with */ - void startDecoding(byte[] dictionary); + void startDecoding(ByteBuffer dictionary); /** + * @deprecated use {@link #decodeChunk(ByteBuffer, OutputStream)} + * * Accepts "data[offset,offset+length-1]" as additional data received in the * compressed stream. If any chunks of data can be fully decoded, * they are appended to out. @@ -43,10 +52,23 @@ public interface VCDiffStreamingDecoder { * @throws IOException if an error occurred decoding chunk or writing * the decoded chunk to out */ + @Deprecated void decodeChunk(byte[] data, int offset, int length, OutputStream out) throws IOException; /** - * Convenience method equivilent to decodeChunk(data, 0, data.length, out) + * Accepts "data[offset,offset+length-1]" as additional data received in the + * compressed stream. If any chunks of data can be fully decoded, + * they are appended to out. + * + * @param data data to decoder + * @param out OutputStream to write decoded data to + * @throws IOException if an error occurred decoding chunk or writing + * the decoded chunk to out + */ + void decodeChunk(ByteBuffer data, OutputStream out) throws IOException; + + /** + * Convenience method equivalent to decodeChunk(data, 0, data.length, out) * * @param data data to decoder * @param out OutputStream to write decoded data to diff --git a/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffDeltaFileWindow.java b/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffDeltaFileWindow.java index 4ed843e..d398d17 100755 --- a/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffDeltaFileWindow.java +++ b/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffDeltaFileWindow.java @@ -77,14 +77,14 @@ public int DecodeWindow(ByteBuffer parseable_chunk) throws IOException { reader.updatePointers(instructionsAndSizes); } switch (decodeBody(parseable_chunk)) { - case VCDiffHeaderParser.RESULT_END_OF_DATA: - if (moreDataExpected()) { - return VCDiffHeaderParser.RESULT_END_OF_DATA; - } else { - throw new IOException("End of data reached while decoding VCDIFF delta file"); - } - default: - break; // decodeBody succeeded + case VCDiffHeaderParser.RESULT_END_OF_DATA: + if (moreDataExpected()) { + return VCDiffHeaderParser.RESULT_END_OF_DATA; + } else { + throw new IOException("End of data reached while decoding VCDIFF delta file"); + } + default: + break; // decodeBody succeeded } // Get ready to read a new delta window Reset(); @@ -157,7 +157,7 @@ private int readHeader(ByteBuffer parseableChunk) throws IOException { VCDiffHeaderParser header_parser = new VCDiffHeaderParser(parseableChunk.slice()); VCDiffHeaderParser.DeltaWindowHeader deltaWindowHeader = header_parser.parseWinIndicatorAndSourceSegment( - parent.dictionarySize(), + parent.dictionary_ptr().limit(), decoded_target.size(), parent.allowVcdTarget() ); @@ -191,7 +191,7 @@ private int readHeader(ByteBuffer parseableChunk) throws IOException { // Get a pointer to the start of the source segment. if ((deltaWindowHeader.win_indicator & VCD_SOURCE) != 0) { - sourceSegment = ByteBuffer.wrap(parent.dictionary_ptr()); + sourceSegment = (ByteBuffer) parent.dictionary_ptr().duplicate().rewind(); sourceSegment.position(deltaWindowHeader.source_segment_position); } else if ((deltaWindowHeader.win_indicator & VCD_TARGET) != 0) { // This assignment must happen after the reserve(). @@ -307,11 +307,11 @@ private int decodeBody(ByteBuffer parseable_chunk) throws IOException { final AtomicInteger mode = new AtomicInteger(0); int instruction = reader.getNextInstruction(decoded_size, mode); switch (instruction) { - case VCD_INSTRUCTION_END_OF_DATA: - updateInstructionPointer(parseable_chunk); - return VCDiffHeaderParser.RESULT_END_OF_DATA; - default: - break; + case VCD_INSTRUCTION_END_OF_DATA: + updateInstructionPointer(parseable_chunk); + return VCDiffHeaderParser.RESULT_END_OF_DATA; + default: + break; } final int size = decoded_size.get(); // The value of "size" itself could be enormous (say, INT32_MAX) @@ -326,25 +326,25 @@ private int decodeBody(ByteBuffer parseable_chunk) throws IOException { } int result; switch (instruction) { - case VCD_ADD: - result = decodeAdd(size); - break; - case VCD_RUN: - result = decodeRun(size); - break; - case VCD_COPY: - result = decodeCopy(size, (short)mode.get()); - break; - default: - throw new IOException("Unexpected instruction type " + instruction + " in opcode stream"); + case VCD_ADD: + result = decodeAdd(size); + break; + case VCD_RUN: + result = decodeRun(size); + break; + case VCD_COPY: + result = decodeCopy(size, (short)mode.get()); + break; + default: + throw new IOException("Unexpected instruction type " + instruction + " in opcode stream"); } switch (result) { - case VCDiffHeaderParser.RESULT_END_OF_DATA: - reader.unGetInstruction(); - updateInstructionPointer(parseable_chunk); - return VCDiffHeaderParser.RESULT_END_OF_DATA; - case VCDiffHeaderParser.RESULT_SUCCESS: - break; + case VCDiffHeaderParser.RESULT_END_OF_DATA: + reader.unGetInstruction(); + updateInstructionPointer(parseable_chunk); + return VCDiffHeaderParser.RESULT_END_OF_DATA; + case VCDiffHeaderParser.RESULT_SUCCESS: + break; } } if (targetBytesDecoded() != targetWindowLength) { diff --git a/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffStreamingDecoderImpl.java b/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffStreamingDecoderImpl.java index 1d84209..fe19dcd 100755 --- a/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffStreamingDecoderImpl.java +++ b/core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffStreamingDecoderImpl.java @@ -63,7 +63,7 @@ public class VCDiffStreamingDecoderImpl implements VCDiffStreamingDecoder { public static final int UNLIMITED_BYTES = -3; // Contents and length of the source (dictionary) data. - private byte[] dictionary; + private ByteBuffer dictionary; // This string will be used to store any unparsed bytes left over when // decodeChunk() reaches the end of its input and returns RESULT_END_OF_DATA. @@ -157,10 +157,11 @@ public void reset() { decodedTargetOutputPosition = 0; } - // These functions are identical to their counterparts - // in VCDiffStreamingDecoder. - // public void startDecoding(byte[] dictionary) { + startDecoding(ByteBuffer.wrap(dictionary)); + } + + public void startDecoding(ByteBuffer dictionary) { if (startDecodingWasCalled) { throw new IllegalStateException("startDecoding() called twice without finishDecoding()"); } @@ -173,14 +174,18 @@ public void startDecoding(byte[] dictionary) { } public void decodeChunk(byte[] data, int offset, int len, OutputStream out) throws IOException { + decodeChunk(ByteBuffer.wrap(data, offset, len), out); + } + + public void decodeChunk(ByteBuffer data, OutputStream out) throws IOException { if (!startDecodingWasCalled) { reset(); throw new IOException("decodeChunk() called without startDecoding()"); } // TODO: there's a lot of room for optimization here - ByteBuffer parseable_chunk = ByteBuffer.allocate(unparsedBytes.remaining() + len); + ByteBuffer parseable_chunk = ByteBuffer.allocate(unparsedBytes.remaining() + data.remaining()); parseable_chunk.put(unparsedBytes); - parseable_chunk.put(data, offset, len); + parseable_chunk.put(data); parseable_chunk.flip(); unparsedBytes = parseable_chunk.duplicate(); @@ -217,7 +222,7 @@ public void decodeChunk(byte[] data, int offset, int len, OutputStream out) thro } public void decodeChunk(byte[] data, OutputStream out) throws IOException { - decodeChunk(data, 0, data.length, out); + decodeChunk(ByteBuffer.wrap(data), out); } public void finishDecoding() throws IOException { @@ -372,9 +377,7 @@ private boolean isDecodingComplete() { } } - public byte[] dictionary_ptr() { return dictionary; } - - int dictionarySize() { return dictionary.length; } + public ByteBuffer dictionary_ptr() { return dictionary; } VCDiffAddressCache addrCache() { return addrCache; } @@ -432,35 +435,35 @@ private int readDeltaFileHeader(ByteBuffer data) throws IOException { final DeltaFileHeader header = new DeltaFileHeader(paddedHeaderData); boolean wrong_magic_number = false; switch (data_size) { - // Verify only the bytes that are available. - default: - // Found header contents up to and including VCDIFF version - vcdiffVersionCode = header.header4; - if ((vcdiffVersionCode != 0x00) && // Draft standard VCDIFF (RFC 3284) - (vcdiffVersionCode != 'S')) { // Enhancements for SDCH protocol - throw new IOException("Unrecognized VCDIFF format version"); - } - // fall through - case 3: - if (header.header3 != (byte) 0xC4) { // magic value 'D' | 0x80 - wrong_magic_number = true; - } - // fall through - case 2: - if (header.header2 != (byte) 0xC3) { // magic value 'C' | 0x80 - wrong_magic_number = true; - } - // fall through - case 1: - if (header.header1 != (byte) 0xD6) { // magic value 'V' | 0x80 - wrong_magic_number = true; - } - // fall through - case 0: - if (wrong_magic_number) { - throw new IOException("Did not find VCDIFF header bytes; input is not a VCDIFF delta file"); - } - if (data_size < DeltaFileHeader.SERIALIZED_SIZE) return RESULT_END_OF_DATA; + // Verify only the bytes that are available. + default: + // Found header contents up to and including VCDIFF version + vcdiffVersionCode = header.header4; + if ((vcdiffVersionCode != 0x00) && // Draft standard VCDIFF (RFC 3284) + (vcdiffVersionCode != 'S')) { // Enhancements for SDCH protocol + throw new IOException("Unrecognized VCDIFF format version"); + } + // fall through + case 3: + if (header.header3 != (byte) 0xC4) { // magic value 'D' | 0x80 + wrong_magic_number = true; + } + // fall through + case 2: + if (header.header2 != (byte) 0xC3) { // magic value 'C' | 0x80 + wrong_magic_number = true; + } + // fall through + case 1: + if (header.header1 != (byte) 0xD6) { // magic value 'V' | 0x80 + wrong_magic_number = true; + } + // fall through + case 0: + if (wrong_magic_number) { + throw new IOException("Did not find VCDIFF header bytes; input is not a VCDIFF delta file"); + } + if (data_size < DeltaFileHeader.SERIALIZED_SIZE) return RESULT_END_OF_DATA; } int unrecognizedFlags = header.hdr_indicator & 0xff & ~(VCD_DECOMPRESS | VCD_CODETABLE); @@ -475,7 +478,7 @@ private int readDeltaFileHeader(ByteBuffer data) throws IOException { if ((header.hdr_indicator & VCD_CODETABLE) != 0) { int bytes_parsed = InitCustomCodeTable(data.array(), data.arrayOffset() + data.position() + DeltaFileHeader.SERIALIZED_SIZE, - data.remaining() - DeltaFileHeader.SERIALIZED_SIZE); + data.remaining() - DeltaFileHeader.SERIALIZED_SIZE); if (bytes_parsed == RESULT_END_OF_DATA) { return RESULT_END_OF_DATA; } diff --git a/core/src/main/java/com/davidehrmann/vcdiff/io/VCDiffInputStream.java b/core/src/main/java/com/davidehrmann/vcdiff/io/VCDiffInputStream.java index 181d60e..e923c26 100755 --- a/core/src/main/java/com/davidehrmann/vcdiff/io/VCDiffInputStream.java +++ b/core/src/main/java/com/davidehrmann/vcdiff/io/VCDiffInputStream.java @@ -31,7 +31,7 @@ public class VCDiffInputStream extends InputStream { public static final boolean DEFAULT_ALLOW_VCD_TARGET = false; private final VCDiffStreamingDecoder decoder; - private final byte[] dictionary; + private final ByteBuffer dictionary; private final InputStream in; @@ -50,19 +50,26 @@ public VCDiffInputStream(InputStream in, byte[] dictionary) { public VCDiffInputStream(InputStream in, byte[] dictionary, long maxTargetFileSize, int maxTargetWindowSize, boolean allowVcdTarget) { - this.in = Objects.requireNotNull(in, "in was null"); - this.dictionary = Objects.requireNotNull(dictionary, "dictionary was null").clone(); - decoder = VCDiffDecoderBuilder.builder() - .withMaxTargetFileSize(maxTargetFileSize) - .withMaxTargetWindowSize(maxTargetWindowSize) - .withAllowTargetMatches(allowVcdTarget) - .buildStreaming(); + this( + Objects.requireNotNull(in, "in was null"), + ByteBuffer.wrap(Objects.requireNotNull(dictionary, "dictionary was null").clone()), + VCDiffDecoderBuilder.builder() + .withMaxTargetFileSize(maxTargetFileSize) + .withMaxTargetWindowSize(maxTargetWindowSize) + .withAllowTargetMatches(allowVcdTarget) + .buildStreaming()); } public VCDiffInputStream(InputStream in, byte[] dictionary, VCDiffStreamingDecoder decoder) { + this(in, + ByteBuffer.wrap(Objects.requireNotNull(dictionary, "dictionary was null").clone()), + decoder); + } + + public VCDiffInputStream(InputStream in, ByteBuffer dictionary, VCDiffStreamingDecoder decoder) { this.in = Objects.requireNotNull(in, "in was null"); this.decoder = Objects.requireNotNull(decoder, "decoder was null"); - this.dictionary = Objects.requireNotNull(dictionary, "dictionary was null").clone(); + this.dictionary = Objects.requireNotNull(dictionary, "dictionary was null"); } @Override