Skip to content

Commit

Permalink
Add support for ByteBuffer dictionaries
Browse files Browse the repository at this point in the history
  • Loading branch information
@ehrmann
ehrmann committed Jan 29, 2018
1 parent aa98d7f commit 85fb67b
Show file tree
Hide file tree
Showing 6 changed files with 139 additions and 86 deletions.
22 changes: 19 additions & 3 deletions core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;

/**
* A simpler (non-streaming) interface to the VCDIFF decoder that can be used
Expand All @@ -17,6 +18,8 @@ public VCDiffDecoder(VCDiffStreamingDecoder decoder) {
}

/**
* @deprecated use {@link #decode(byte[], byte[], OutputStream)}
*
* decode the contents of encoding using the specified dictionary, writing the decoded data to target
*
* @param dictionary dictionary
Expand All @@ -26,21 +29,34 @@ public VCDiffDecoder(VCDiffStreamingDecoder decoder) {
* @param target output writer for decoded data
* @throws IOException if there was an exception decoding or writing to the output target
*/
@Deprecated
public void decode(byte[] dictionary, byte[] encoding, int offset, int len, OutputStream target) throws IOException {
decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding, offset, len), target);
}

/**
* decode the contents of encoding using the specified dictionary, writing the decoded data to target
*
* @param dictionary dictionary
* @param encoding data to decode
* @param target output writer for decoded data
* @throws IOException if there was an exception decoding or writing to the output target
*/
public void decode(ByteBuffer dictionary, ByteBuffer encoding, OutputStream target) throws IOException {
decoder.startDecoding(dictionary);
decoder.decodeChunk(encoding, offset, len, target);
decoder.decodeChunk(encoding, target);
decoder.finishDecoding();
}

/**
* Convenience method equivalent to decode(dictionary, encoding, 0, encoding.length, target)
* Convenience method equivalent to decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding), target)
*
* @param dictionary dictionary
* @param encoding data to decode
* @param target output writer for decoded data
* @throws IOException if there was an exception decoding or writing to the output target
*/
public void decode(byte[] dictionary, byte[] encoding, OutputStream target) throws IOException {
decode(dictionary, encoding, 0, encoding.length, target);
decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding), target);
}
}
5 changes: 5 additions & 0 deletions core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoderBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.davidehrmann.vcdiff.io.VCDiffInputStream;

import java.io.InputStream;
import java.nio.ByteBuffer;

public class VCDiffDecoderBuilder {

Expand Down Expand Up @@ -73,6 +74,10 @@ public synchronized VCDiffStreamingDecoder buildStreaming(VCDiffStreamingDecoder
}

public VCDiffInputStream buildInputStream(InputStream in, byte[] dictionary) {
return buildInputStream(in, ByteBuffer.wrap(dictionary));
}

public VCDiffInputStream buildInputStream(InputStream in, ByteBuffer dictionary) {
return new VCDiffInputStream(in, dictionary, buildStreaming());
}

Expand Down
26 changes: 24 additions & 2 deletions core/src/main/java/com/davidehrmann/vcdiff/VCDiffStreamingDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,17 @@

import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;


public interface VCDiffStreamingDecoder {

/**
* @deprecated use {@link #startDecoding(ByteBuffer)}
*/
@Deprecated
void startDecoding(byte[] dictionary);

/**
* Resets the dictionary to dictionary parameter
* and sets up the data structures for decoding. Note that the dictionary
Expand All @@ -29,9 +36,11 @@ public interface VCDiffStreamingDecoder {
*
* @param dictionary dictionary the decoder is initialized with
*/
void startDecoding(byte[] dictionary);
void startDecoding(ByteBuffer dictionary);

/**
* @deprecated use {@link #decodeChunk(ByteBuffer, OutputStream)}
*
* Accepts "data[offset,offset+length-1]" as additional data received in the
* compressed stream. If any chunks of data can be fully decoded,
* they are appended to out.
Expand All @@ -43,10 +52,23 @@ public interface VCDiffStreamingDecoder {
* @throws IOException if an error occurred decoding chunk or writing
* the decoded chunk to out
*/
@Deprecated
void decodeChunk(byte[] data, int offset, int length, OutputStream out) throws IOException;

/**
* Convenience method equivilent to decodeChunk(data, 0, data.length, out)
* Accepts "data[offset,offset+length-1]" as additional data received in the
* compressed stream. If any chunks of data can be fully decoded,
* they are appended to out.
*
* @param data data to decoder
* @param out OutputStream to write decoded data to
* @throws IOException if an error occurred decoding chunk or writing
* the decoded chunk to out
*/
void decodeChunk(ByteBuffer data, OutputStream out) throws IOException;

/**
* Convenience method equivalent to decodeChunk(data, 0, data.length, out)
*
* @param data data to decoder
* @param out OutputStream to write decoded data to
Expand Down
64 changes: 32 additions & 32 deletions core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffDeltaFileWindow.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,14 @@ public int DecodeWindow(ByteBuffer parseable_chunk) throws IOException {
reader.updatePointers(instructionsAndSizes);
}
switch (decodeBody(parseable_chunk)) {
case VCDiffHeaderParser.RESULT_END_OF_DATA:
if (moreDataExpected()) {
return VCDiffHeaderParser.RESULT_END_OF_DATA;
} else {
throw new IOException("End of data reached while decoding VCDIFF delta file");
}
default:
break; // decodeBody succeeded
case VCDiffHeaderParser.RESULT_END_OF_DATA:
if (moreDataExpected()) {
return VCDiffHeaderParser.RESULT_END_OF_DATA;
} else {
throw new IOException("End of data reached while decoding VCDIFF delta file");
}
default:
break; // decodeBody succeeded
}
// Get ready to read a new delta window
Reset();
Expand Down Expand Up @@ -157,7 +157,7 @@ private int readHeader(ByteBuffer parseableChunk) throws IOException {
VCDiffHeaderParser header_parser = new VCDiffHeaderParser(parseableChunk.slice());

VCDiffHeaderParser.DeltaWindowHeader deltaWindowHeader = header_parser.parseWinIndicatorAndSourceSegment(
parent.dictionarySize(),
parent.dictionary_ptr().limit(),
decoded_target.size(),
parent.allowVcdTarget()
);
Expand Down Expand Up @@ -191,7 +191,7 @@ private int readHeader(ByteBuffer parseableChunk) throws IOException {

// Get a pointer to the start of the source segment.
if ((deltaWindowHeader.win_indicator & VCD_SOURCE) != 0) {
sourceSegment = ByteBuffer.wrap(parent.dictionary_ptr());
sourceSegment = (ByteBuffer) parent.dictionary_ptr().duplicate().rewind();
sourceSegment.position(deltaWindowHeader.source_segment_position);
} else if ((deltaWindowHeader.win_indicator & VCD_TARGET) != 0) {
// This assignment must happen after the reserve().
Expand Down Expand Up @@ -307,11 +307,11 @@ private int decodeBody(ByteBuffer parseable_chunk) throws IOException {
final AtomicInteger mode = new AtomicInteger(0);
int instruction = reader.getNextInstruction(decoded_size, mode);
switch (instruction) {
case VCD_INSTRUCTION_END_OF_DATA:
updateInstructionPointer(parseable_chunk);
return VCDiffHeaderParser.RESULT_END_OF_DATA;
default:
break;
case VCD_INSTRUCTION_END_OF_DATA:
updateInstructionPointer(parseable_chunk);
return VCDiffHeaderParser.RESULT_END_OF_DATA;
default:
break;
}
final int size = decoded_size.get();
// The value of "size" itself could be enormous (say, INT32_MAX)
Expand All @@ -326,25 +326,25 @@ private int decodeBody(ByteBuffer parseable_chunk) throws IOException {
}
int result;
switch (instruction) {
case VCD_ADD:
result = decodeAdd(size);
break;
case VCD_RUN:
result = decodeRun(size);
break;
case VCD_COPY:
result = decodeCopy(size, (short)mode.get());
break;
default:
throw new IOException("Unexpected instruction type " + instruction + " in opcode stream");
case VCD_ADD:
result = decodeAdd(size);
break;
case VCD_RUN:
result = decodeRun(size);
break;
case VCD_COPY:
result = decodeCopy(size, (short)mode.get());
break;
default:
throw new IOException("Unexpected instruction type " + instruction + " in opcode stream");
}
switch (result) {
case VCDiffHeaderParser.RESULT_END_OF_DATA:
reader.unGetInstruction();
updateInstructionPointer(parseable_chunk);
return VCDiffHeaderParser.RESULT_END_OF_DATA;
case VCDiffHeaderParser.RESULT_SUCCESS:
break;
case VCDiffHeaderParser.RESULT_END_OF_DATA:
reader.unGetInstruction();
updateInstructionPointer(parseable_chunk);
return VCDiffHeaderParser.RESULT_END_OF_DATA;
case VCDiffHeaderParser.RESULT_SUCCESS:
break;
}
}
if (targetBytesDecoded() != targetWindowLength) {
Expand Down
83 changes: 43 additions & 40 deletions core/src/main/java/com/davidehrmann/vcdiff/engine/VCDiffStreamingDecoderImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class VCDiffStreamingDecoderImpl implements VCDiffStreamingDecoder {
public static final int UNLIMITED_BYTES = -3;

// Contents and length of the source (dictionary) data.
private byte[] dictionary;
private ByteBuffer dictionary;

// This string will be used to store any unparsed bytes left over when
// decodeChunk() reaches the end of its input and returns RESULT_END_OF_DATA.
Expand Down Expand Up @@ -157,10 +157,11 @@ public void reset() {
decodedTargetOutputPosition = 0;
}

// These functions are identical to their counterparts
// in VCDiffStreamingDecoder.
//
public void startDecoding(byte[] dictionary) {
startDecoding(ByteBuffer.wrap(dictionary));
}

public void startDecoding(ByteBuffer dictionary) {
if (startDecodingWasCalled) {
throw new IllegalStateException("startDecoding() called twice without finishDecoding()");
}
Expand All @@ -173,14 +174,18 @@ public void startDecoding(byte[] dictionary) {
}

public void decodeChunk(byte[] data, int offset, int len, OutputStream out) throws IOException {
decodeChunk(ByteBuffer.wrap(data, offset, len), out);
}

public void decodeChunk(ByteBuffer data, OutputStream out) throws IOException {
if (!startDecodingWasCalled) {
reset();
throw new IOException("decodeChunk() called without startDecoding()");
}
// TODO: there's a lot of room for optimization here
ByteBuffer parseable_chunk = ByteBuffer.allocate(unparsedBytes.remaining() + len);
ByteBuffer parseable_chunk = ByteBuffer.allocate(unparsedBytes.remaining() + data.remaining());
parseable_chunk.put(unparsedBytes);
parseable_chunk.put(data, offset, len);
parseable_chunk.put(data);
parseable_chunk.flip();
unparsedBytes = parseable_chunk.duplicate();

Expand Down Expand Up @@ -217,7 +222,7 @@ public void decodeChunk(byte[] data, int offset, int len, OutputStream out) thro
}

public void decodeChunk(byte[] data, OutputStream out) throws IOException {
decodeChunk(data, 0, data.length, out);
decodeChunk(ByteBuffer.wrap(data), out);
}

public void finishDecoding() throws IOException {
Expand Down Expand Up @@ -372,9 +377,7 @@ private boolean isDecodingComplete() {
}
}

public byte[] dictionary_ptr() { return dictionary; }

int dictionarySize() { return dictionary.length; }
public ByteBuffer dictionary_ptr() { return dictionary; }

VCDiffAddressCache addrCache() { return addrCache; }

Expand Down Expand Up @@ -432,35 +435,35 @@ private int readDeltaFileHeader(ByteBuffer data) throws IOException {
final DeltaFileHeader header = new DeltaFileHeader(paddedHeaderData);
boolean wrong_magic_number = false;
switch (data_size) {
// Verify only the bytes that are available.
default:
// Found header contents up to and including VCDIFF version
vcdiffVersionCode = header.header4;
if ((vcdiffVersionCode != 0x00) && // Draft standard VCDIFF (RFC 3284)
(vcdiffVersionCode != 'S')) { // Enhancements for SDCH protocol
throw new IOException("Unrecognized VCDIFF format version");
}
// fall through
case 3:
if (header.header3 != (byte) 0xC4) { // magic value 'D' | 0x80
wrong_magic_number = true;
}
// fall through
case 2:
if (header.header2 != (byte) 0xC3) { // magic value 'C' | 0x80
wrong_magic_number = true;
}
// fall through
case 1:
if (header.header1 != (byte) 0xD6) { // magic value 'V' | 0x80
wrong_magic_number = true;
}
// fall through
case 0:
if (wrong_magic_number) {
throw new IOException("Did not find VCDIFF header bytes; input is not a VCDIFF delta file");
}
if (data_size < DeltaFileHeader.SERIALIZED_SIZE) return RESULT_END_OF_DATA;
// Verify only the bytes that are available.
default:
// Found header contents up to and including VCDIFF version
vcdiffVersionCode = header.header4;
if ((vcdiffVersionCode != 0x00) && // Draft standard VCDIFF (RFC 3284)
(vcdiffVersionCode != 'S')) { // Enhancements for SDCH protocol
throw new IOException("Unrecognized VCDIFF format version");
}
// fall through
case 3:
if (header.header3 != (byte) 0xC4) { // magic value 'D' | 0x80
wrong_magic_number = true;
}
// fall through
case 2:
if (header.header2 != (byte) 0xC3) { // magic value 'C' | 0x80
wrong_magic_number = true;
}
// fall through
case 1:
if (header.header1 != (byte) 0xD6) { // magic value 'V' | 0x80
wrong_magic_number = true;
}
// fall through
case 0:
if (wrong_magic_number) {
throw new IOException("Did not find VCDIFF header bytes; input is not a VCDIFF delta file");
}
if (data_size < DeltaFileHeader.SERIALIZED_SIZE) return RESULT_END_OF_DATA;
}

int unrecognizedFlags = header.hdr_indicator & 0xff & ~(VCD_DECOMPRESS | VCD_CODETABLE);
Expand All @@ -475,7 +478,7 @@ private int readDeltaFileHeader(ByteBuffer data) throws IOException {

if ((header.hdr_indicator & VCD_CODETABLE) != 0) {
int bytes_parsed = InitCustomCodeTable(data.array(), data.arrayOffset() + data.position() + DeltaFileHeader.SERIALIZED_SIZE,
data.remaining() - DeltaFileHeader.SERIALIZED_SIZE);
data.remaining() - DeltaFileHeader.SERIALIZED_SIZE);
if (bytes_parsed == RESULT_END_OF_DATA) {
return RESULT_END_OF_DATA;
}
Expand Down
Loading

0 comments on commit 85fb67b

Please sign in to comment.