Skip to content

Commit

Permalink
Use ByteBuffers in decoder
Browse files Browse the repository at this point in the history
  • Loading branch information
ehrmann committed Jan 26, 2018
1 parent 3ac935d commit 49af855
Show file tree
Hide file tree
Showing 7 changed files with 77 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private static OutputStream OpenFileForWriting(String file_name, String file_typ
// Opens the dictionary file and reads it into a newly allocated buffer.
// If successful, returns true and populates dictionary with the dictionary
// contents; otherwise, returns the buffer.
protected static byte[] OpenDictionary(String dictionary) throws IOException {
protected static byte[] openDictionary(String dictionary) throws IOException {
InputStream in = OpenFileForReading(dictionary, "dictionary");
try {
return IOUtils.toByteArray(in);
Expand Down Expand Up @@ -295,7 +295,7 @@ private static class EncodeCommand extends VCDiffFileBasedCoder {
private OptionalTargetAndDeltaOptions targetAndDeltaOptions = new OptionalTargetAndDeltaOptions();

public void Encode() throws IOException {
byte[] dictionary = OpenDictionary(globalOptions.dictionary);
byte[] dictionary = openDictionary(globalOptions.dictionary);

// FIXME: support encodeOptions.json
/*
Expand Down Expand Up @@ -362,7 +362,7 @@ private static class DecodeCommand extends VCDiffFileBasedCoder {
private OptionalTargetAndDeltaOptions targetAndDeltaFlags = new OptionalTargetAndDeltaOptions();

public void Decode() throws IOException {
byte[] dictionary = OpenDictionary(globalOptions.dictionary);
byte[] dictionary = openDictionary(globalOptions.dictionary);

boolean useStdin = (targetAndDeltaFlags.delta == null || targetAndDeltaFlags.delta.isEmpty());
boolean useStdout = (targetAndDeltaFlags.target == null || targetAndDeltaFlags.target.isEmpty());
Expand Down Expand Up @@ -418,7 +418,7 @@ private static class DecodeAndCompareCommand extends VCDiffFileBasedCoder {
private RequiredTargetAndDeltaOptions targetAndDeltaOptions = new RequiredTargetAndDeltaOptions();

public void DecodeAndCompare() throws IOException {
byte[] dictionary = OpenDictionary(globalOptions.dictionary);
byte[] dictionary = openDictionary(globalOptions.dictionary);

CountingInputStream countedIn = new CountingInputStream(OpenFileForReading(targetAndDeltaOptions.delta, "delta"));
try {
Expand Down
22 changes: 19 additions & 3 deletions core/src/main/java/com/davidehrmann/vcdiff/VCDiffDecoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;

/**
* A simpler (non-streaming) interface to the VCDIFF decoder that can be used
Expand All @@ -17,6 +18,8 @@ public VCDiffDecoder(VCDiffStreamingDecoder decoder) {
}

/**
* @deprecated use {@link #decode(byte[], byte[], OutputStream)}
*
* decode the contents of encoding using the specified dictionary, writing the decoded data to target
*
* @param dictionary dictionary
Expand All @@ -26,21 +29,34 @@ public VCDiffDecoder(VCDiffStreamingDecoder decoder) {
* @param target output writer for decoded data
* @throws IOException if there was an exception decoding or writing to the output target
*/
@Deprecated
public void decode(byte[] dictionary, byte[] encoding, int offset, int len, OutputStream target) throws IOException {
decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding, offset, len), target);
}

/**
* decode the contents of encoding using the specified dictionary, writing the decoded data to target
*
* @param dictionary dictionary
* @param encoding data to decode
* @param target output writer for decoded data
* @throws IOException if there was an exception decoding or writing to the output target
*/
public void decode(ByteBuffer dictionary, ByteBuffer encoding, OutputStream target) throws IOException {
decoder.startDecoding(dictionary);
decoder.decodeChunk(encoding, offset, len, target);
decoder.decodeChunk(encoding, target);
decoder.finishDecoding();
}

/**
* Convenience method equivalent to decode(dictionary, encoding, 0, encoding.length, target)
* Convenience method equivalent to decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding), target)
*
* @param dictionary dictionary
* @param encoding data to decode
* @param target output writer for decoded data
* @throws IOException if there was an exception decoding or writing to the output target
*/
public void decode(byte[] dictionary, byte[] encoding, OutputStream target) throws IOException {
decode(dictionary, encoding, 0, encoding.length, target);
decode(ByteBuffer.wrap(dictionary), ByteBuffer.wrap(encoding), target);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.davidehrmann.vcdiff.io.VCDiffInputStream;

import java.io.InputStream;
import java.nio.ByteBuffer;

public class VCDiffDecoderBuilder {

Expand Down Expand Up @@ -73,6 +74,10 @@ public synchronized VCDiffStreamingDecoder buildStreaming(VCDiffStreamingDecoder
}

public VCDiffInputStream buildInputStream(InputStream in, byte[] dictionary) {
return buildInputStream(in, ByteBuffer.wrap(dictionary));
}

public VCDiffInputStream buildInputStream(InputStream in, ByteBuffer dictionary) {
return new VCDiffInputStream(in, dictionary, buildStreaming());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,17 @@

import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;


public interface VCDiffStreamingDecoder {

/**
* @deprecated use {@link #startDecoding(ByteBuffer)}
*/
@Deprecated
void startDecoding(byte[] dictionary);

/**
* Resets the dictionary to dictionary parameter
* and sets up the data structures for decoding. Note that the dictionary
Expand All @@ -29,9 +36,11 @@ public interface VCDiffStreamingDecoder {
*
* @param dictionary dictionary the decoder is initialized with
*/
void startDecoding(byte[] dictionary);
void startDecoding(ByteBuffer dictionary);

/**
* @deprecated use {@link #decodeChunk(ByteBuffer, OutputStream)}
*
* Accepts "data[offset,offset+length-1]" as additional data received in the
* compressed stream. If any chunks of data can be fully decoded,
* they are appended to out.
Expand All @@ -43,10 +52,23 @@ public interface VCDiffStreamingDecoder {
* @throws IOException if an error occurred decoding chunk or writing
* the decoded chunk to out
*/
@Deprecated
void decodeChunk(byte[] data, int offset, int length, OutputStream out) throws IOException;

/**
* Convenience method equivilent to decodeChunk(data, 0, data.length, out)
* Accepts "data[offset,offset+length-1]" as additional data received in the
* compressed stream. If any chunks of data can be fully decoded,
* they are appended to out.
*
* @param data data to decoder
* @param out OutputStream to write decoded data to
* @throws IOException if an error occurred decoding chunk or writing
* the decoded chunk to out
*/
void decodeChunk(ByteBuffer data, OutputStream out) throws IOException;

/**
* Convenience method equivalent to decodeChunk(data, 0, data.length, out)
*
* @param data data to decoder
* @param out OutputStream to write decoded data to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ private int readHeader(ByteBuffer parseableChunk) throws IOException {
VCDiffHeaderParser header_parser = new VCDiffHeaderParser(parseableChunk.slice());

VCDiffHeaderParser.DeltaWindowHeader deltaWindowHeader = header_parser.parseWinIndicatorAndSourceSegment(
parent.dictionarySize(),
parent.dictionary_ptr().limit(),
decoded_target.size(),
parent.allowVcdTarget()
);
Expand Down Expand Up @@ -191,7 +191,7 @@ private int readHeader(ByteBuffer parseableChunk) throws IOException {

// Get a pointer to the start of the source segment.
if ((deltaWindowHeader.win_indicator & VCD_SOURCE) != 0) {
sourceSegment = ByteBuffer.wrap(parent.dictionary_ptr());
sourceSegment = (ByteBuffer) parent.dictionary_ptr().duplicate().rewind();
sourceSegment.position(deltaWindowHeader.source_segment_position);
} else if ((deltaWindowHeader.win_indicator & VCD_TARGET) != 0) {
// This assignment must happen after the reserve().
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class VCDiffStreamingDecoderImpl implements VCDiffStreamingDecoder {
public static final int UNLIMITED_BYTES = -3;

// Contents and length of the source (dictionary) data.
private byte[] dictionary;
private ByteBuffer dictionary;

// This string will be used to store any unparsed bytes left over when
// decodeChunk() reaches the end of its input and returns RESULT_END_OF_DATA.
Expand Down Expand Up @@ -157,10 +157,11 @@ public void reset() {
decodedTargetOutputPosition = 0;
}

// These functions are identical to their counterparts
// in VCDiffStreamingDecoder.
//
public void startDecoding(byte[] dictionary) {
startDecoding(ByteBuffer.wrap(dictionary));
}

public void startDecoding(ByteBuffer dictionary) {
if (startDecodingWasCalled) {
throw new IllegalStateException("startDecoding() called twice without finishDecoding()");
}
Expand All @@ -173,14 +174,18 @@ public void startDecoding(byte[] dictionary) {
}

public void decodeChunk(byte[] data, int offset, int len, OutputStream out) throws IOException {
decodeChunk(ByteBuffer.wrap(data, offset, len), out);
}

public void decodeChunk(ByteBuffer data, OutputStream out) throws IOException {
if (!startDecodingWasCalled) {
reset();
throw new IOException("decodeChunk() called without startDecoding()");
}
// TODO: there's a lot of room for optimization here
ByteBuffer parseable_chunk = ByteBuffer.allocate(unparsedBytes.remaining() + len);
ByteBuffer parseable_chunk = ByteBuffer.allocate(unparsedBytes.remaining() + data.remaining());
parseable_chunk.put(unparsedBytes);
parseable_chunk.put(data, offset, len);
parseable_chunk.put(data);
parseable_chunk.flip();
unparsedBytes = parseable_chunk.duplicate();

Expand Down Expand Up @@ -217,7 +222,7 @@ public void decodeChunk(byte[] data, int offset, int len, OutputStream out) thro
}

public void decodeChunk(byte[] data, OutputStream out) throws IOException {
decodeChunk(data, 0, data.length, out);
decodeChunk(ByteBuffer.wrap(data), out);
}

public void finishDecoding() throws IOException {
Expand Down Expand Up @@ -372,9 +377,7 @@ private boolean isDecodingComplete() {
}
}

public byte[] dictionary_ptr() { return dictionary; }

int dictionarySize() { return dictionary.length; }
public ByteBuffer dictionary_ptr() { return dictionary; }

VCDiffAddressCache addrCache() { return addrCache; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public class VCDiffInputStream extends InputStream {
public static final boolean DEFAULT_ALLOW_VCD_TARGET = false;

private final VCDiffStreamingDecoder decoder;
private final byte[] dictionary;
private final ByteBuffer dictionary;

private final InputStream in;

Expand All @@ -51,7 +51,8 @@ public VCDiffInputStream(InputStream in, byte[] dictionary) {
public VCDiffInputStream(InputStream in, byte[] dictionary,
long maxTargetFileSize, int maxTargetWindowSize, boolean allowVcdTarget) {
this.in = Objects.requireNotNull(in, "in was null");
this.dictionary = Objects.requireNotNull(dictionary, "dictionary was null").clone();
this.dictionary = ByteBuffer.wrap(
Objects.requireNotNull(dictionary, "dictionary was null").clone());
decoder = VCDiffDecoderBuilder.builder()
.withMaxTargetFileSize(maxTargetFileSize)
.withMaxTargetWindowSize(maxTargetWindowSize)
Expand All @@ -60,9 +61,15 @@ public VCDiffInputStream(InputStream in, byte[] dictionary,
}

public VCDiffInputStream(InputStream in, byte[] dictionary, VCDiffStreamingDecoder decoder) {
this(in,
ByteBuffer.wrap(Objects.requireNotNull(dictionary, "dictionary was null").clone()),
decoder);
}

public VCDiffInputStream(InputStream in, ByteBuffer dictionary, VCDiffStreamingDecoder decoder) {
this.in = Objects.requireNotNull(in, "in was null");
this.decoder = Objects.requireNotNull(decoder, "decoder was null");
this.dictionary = Objects.requireNotNull(dictionary, "dictionary was null").clone();
this.dictionary = Objects.requireNotNull(dictionary, "dictionary was null");
}

@Override
Expand Down

0 comments on commit 49af855

Please sign in to comment.