Skip to content

Commit

Permalink
Merge branch 'master' into socket_get_port_op
Browse files Browse the repository at this point in the history
  • Loading branch information
@cono
cono committed May 23, 2017
2 parents b5f29e9 + 6f0b5b4 commit bd57598
Show file tree
Hide file tree
Showing 5 changed files with 257 additions and 68 deletions.
2 changes: 1 addition & 1 deletion src/vm/jvm/QAST/Compiler.nqp
View file
Expand Up @@ -2399,7 +2399,7 @@ QAST::OperationsJAST.map_classlib_core_op('decoderaddbytes', $TYPE_OPS, 'decoder
QAST::OperationsJAST.map_classlib_core_op('decodertakechars', $TYPE_OPS, 'decodertakechars', [$RT_OBJ, $RT_INT], $RT_STR, :tc);
QAST::OperationsJAST.map_classlib_core_op('decodertakeallchars', $TYPE_OPS, 'decodertakeallchars', [$RT_OBJ], $RT_STR, :tc);
QAST::OperationsJAST.map_classlib_core_op('decodertakeavailablechars', $TYPE_OPS, 'decodertakeavailablechars', [$RT_OBJ], $RT_STR, :tc);
QAST::OperationsJAST.map_classlib_core_op('decodertakeline', $TYPE_OPS, 'decodertakeline', [$RT_OBJ, $RT_INT, $RT_INT], $RT_OBJ, :tc);
QAST::OperationsJAST.map_classlib_core_op('decodertakeline', $TYPE_OPS, 'decodertakeline', [$RT_OBJ, $RT_INT, $RT_INT], $RT_STR, :tc);
QAST::OperationsJAST.map_classlib_core_op('decoderbytesavailable', $TYPE_OPS, 'decoderbytesavailable', [$RT_OBJ], $RT_INT, :tc);
QAST::OperationsJAST.map_classlib_core_op('decodertakebytes', $TYPE_OPS, 'decodertakebytes', [$RT_OBJ, $RT_OBJ, $RT_INT], $RT_OBJ, :tc);
QAST::OperationsJAST.map_classlib_core_op('decoderempty', $TYPE_OPS, 'decoderempty', [$RT_OBJ], $RT_INT, :tc);
Expand Down
34 changes: 34 additions & 0 deletions src/vm/jvm/runtime/org/perl6/nqp/runtime/Ops.java
View file
Expand Up @@ -4441,6 +4441,40 @@ public static String decodertakeallchars(SixModelObject decoder, ThreadContext t
throw ExceptionHandling.dieInternal(tc, "decodertakeallchars requires an instance with the Decoder REPR");
}

public static String decodertakeline(SixModelObject decoder, long chomp, long eof,
ThreadContext tc) {
if (decoder instanceof DecoderInstance)
return ((DecoderInstance)decoder).takeLine(tc, chomp != 0, eof != 0);
else
throw ExceptionHandling.dieInternal(tc, "decodertakeline requires an instance with the Decoder REPR");
}

public static SixModelObject decodersetlineseps(SixModelObject decoder, SixModelObject seps,
ThreadContext tc) {
if (decoder instanceof DecoderInstance) {
((DecoderInstance)decoder).setLineSeps(tc, seps);
return decoder;
}
else {
throw ExceptionHandling.dieInternal(tc, "decodersetlineseps requires an instance with the Decoder REPR");
}
}

public static long decoderbytesavailable(SixModelObject decoder, ThreadContext tc) {
if (decoder instanceof DecoderInstance)
return ((DecoderInstance)decoder).bytesAvailable(tc);
else
throw ExceptionHandling.dieInternal(tc, "decoderbytesavailable requires an instance with the Decoder REPR");
}

public static SixModelObject decodertakebytes(SixModelObject decoder, SixModelObject bufType,
long bytes, ThreadContext tc) {
if (decoder instanceof DecoderInstance)
return ((DecoderInstance)decoder).takeBytes(tc, bufType, bytes);
else
throw ExceptionHandling.dieInternal(tc, "decodertakebytes requires an instance with the Decoder REPR");
}

private static final int CCLASS_ANY = 65535;
private static final int CCLASS_UPPERCASE = 1;
private static final int CCLASS_LOWERCASE = 2;
Expand Down
168 changes: 155 additions & 13 deletions src/vm/jvm/runtime/org/perl6/nqp/sixmodel/reprs/DecoderInstance.java
View file
Expand Up @@ -7,20 +7,44 @@
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.List;
import org.perl6.nqp.runtime.Buffers;
import org.perl6.nqp.runtime.ExceptionHandling;
import org.perl6.nqp.runtime.ThreadContext;
import org.perl6.nqp.sixmodel.SixModelObject;
import org.perl6.nqp.sixmodel.StorageSpec;

public class DecoderInstance extends SixModelObject {
private Charset charset;
private CharsetDecoder decoder;
private List<ByteBuffer> toDecode;
private List<CharBuffer> decoded;
private List<String> lineSeps;

public void configure(ThreadContext tc, String encoding, SixModelObject config) {
if (decoder == null)
decoder = Charset.forName(encoding).newDecoder();
else
if (decoder == null) {
charset = Charset.forName(encoding);
decoder = charset.newDecoder();
decoded = new ArrayList<CharBuffer>();
lineSeps = new ArrayList<String>();
lineSeps.add("\n");
lineSeps.add("\r\n");
}
else {
throw ExceptionHandling.dieInternal(tc, "Decoder already configured");
}
}

public void setLineSeps(ThreadContext tc, SixModelObject seps) {
final int prim = seps.st.REPR.get_value_storage_spec(tc, seps.st).boxed_primitive;
if (prim != StorageSpec.BP_STR)
ExceptionHandling.dieInternal(tc,
"Line separators must be provided as an array of native strings");
lineSeps.clear();
long numSeps = seps.elems(tc);
for (long i = 0; i < numSeps; i++) {
seps.at_pos_native(tc, i);
lineSeps.add(tc.native_s);
}
}

public void addBytes(ThreadContext tc, ByteBuffer bytes) {
Expand All @@ -37,8 +61,8 @@ public String takeChars(ThreadContext tc, long chars) {
return "";

CharBuffer target = CharBuffer.allocate((int)chars + 1);
eatDecodedChars(target);
if (target.position() != chars)
eatDecodedChars(target, (int)(chars + 1));
if (target.position() != chars + 1)
eatUndecodedBytes(target, false);

String normalized = Normalizer.normalize(
Expand All @@ -48,8 +72,6 @@ public String takeChars(ThreadContext tc, long chars) {
String result = normalized.substring(0, (int)chars);
String remaining = normalized.substring((int)chars, normalized.length());
if (remaining.length() > 0) {
if (decoded == null)
decoded = new ArrayList<CharBuffer>();
decoded.add(CharBuffer.wrap(remaining));
}
return result;
Expand All @@ -68,7 +90,7 @@ public String takeAvailableChars(ThreadContext tc) {

int maxChars = availableDecodedChars() + availableUndecodedBytes();
CharBuffer target = CharBuffer.allocate(maxChars);
eatDecodedChars(target);
eatAllDecodedChars(target);
eatUndecodedBytes(target, true);

String normalized = Normalizer.normalize(
Expand All @@ -86,7 +108,7 @@ public String takeAllChars(ThreadContext tc) {
ensureConfigured(tc);
int maxChars = availableDecodedChars() + availableUndecodedBytes();
CharBuffer target = CharBuffer.allocate(maxChars);
eatDecodedChars(target);
eatAllDecodedChars(target);
if (toDecode != null) {
if (toDecode.size() == 0)
toDecode.add(ByteBuffer.allocate(0));
Expand All @@ -97,6 +119,101 @@ public String takeAllChars(ThreadContext tc) {
return Normalizer.normalize(decodedBuffer(target), Normalizer.Form.NFC);
}

public String takeLine(ThreadContext tc, boolean chomp, boolean eof) {
ensureConfigured(tc);
while (true) {
/* See if we can find the separator in any of the decoded chars. */
int charsToTake = 0;
for (int i = 0; i < (decoded == null ? 0 : decoded.size()); i++) {
CharBuffer search = decoded.get(i);
for (int j = 0; j < search.remaining(); j++) {
char c = search.charAt(j);
for (int k = 0; k < lineSeps.size(); k++) {
String sep = lineSeps.get(k);
if (sep.charAt(0) == c) {
if (sep.length() == 1 || sepMatchAt(i, j, sep)) {
return takeCharsSkipChars(
chomp ? charsToTake : charsToTake + sep.length(),
chomp ? sep.length() : 0);
}
}
}
charsToTake++;
}
}

/* If there are no more buffers to decode then we're done. */
if (toDecode == null || toDecode.size() == 0)
break;

/* Otherwise decode one of them. */
ByteBuffer decodee = toDecode.get(0);
CharBuffer target = CharBuffer.allocate(decodee.limit());
decoder.decode(decodee, target, eof && toDecode.size() == 1);
target.rewind();
decoded.add(target);
toDecode.remove(0);
}

return eof ? takeAllChars(tc) : null;
}

public long bytesAvailable(ThreadContext tc) {
ensureConfigured(tc);
forceDecodedBackToBytes();
return availableUndecodedBytes();
}

public SixModelObject takeBytes(ThreadContext tc, SixModelObject bufType, long lBytes) {
int available = (int)bytesAvailable(tc); // Implicitly forces decoded back to bytes
SixModelObject res = bufType.st.REPR.allocate(tc, bufType.st);
byte[] resBytes = new byte[available];
int bytes = (int)lBytes;
if (bytes > available)
bytes = available;
int need = bytes;
while (need > 0) {
ByteBuffer takeFrom = toDecode.get(0);
int fromAvailable = takeFrom.remaining();
if (need >= fromAvailable) {
takeFrom.get(resBytes, bytes - need, fromAvailable);
need -= fromAvailable;
toDecode.remove(0);
}
else {
takeFrom.get(resBytes, bytes - need, need);
need = 0;
}
}
Buffers.stashBytes(tc, res, resBytes);
return res;
}

private boolean sepMatchAt(int decStart, int charStart, String sep) {
int sepIndex = 0;
boolean firstBuffer = true;
for (int i = decStart; i < decoded.size(); i++) {
CharBuffer search = decoded.get(i);
for (int j = firstBuffer ? charStart : 0; j < search.remaining(); j++) {
if (search.charAt(j) != sep.charAt(sepIndex++))
return false;
if (sepIndex == sep.length())
return true;
}
firstBuffer = false;
}
return false;
}

private String takeCharsSkipChars(int take, int skip) {
CharBuffer target = CharBuffer.allocate(take);
eatDecodedChars(target, take);
if (skip > 0)
eatDecodedChars(CharBuffer.allocate(skip), skip);
target.rewind();
return Normalizer.normalize(target, Normalizer.Form.NFC);
}

private int availableDecodedChars() {
int available = 0;
if (decoded != null)
Expand All @@ -109,18 +226,36 @@ private int availableUndecodedBytes() {
int available = 0;
if (toDecode != null)
for (int i = 0; i < toDecode.size(); i++)
available += toDecode.get(i).capacity();
available += toDecode.get(i).remaining();
return available;
}

private void eatDecodedChars(CharBuffer target) {
private void eatAllDecodedChars(CharBuffer target) {
if (decoded != null) {
for (int i = 0; i < decoded.size(); i++)
for (int i = 0; i < decoded.size(); i++) {
target.append(decoded.get(i));
}
decoded.clear();
}
}

private void eatDecodedChars(CharBuffer target, int n) {
int remaining = n;
while (remaining > 0 && decoded.size() > 0) {
CharBuffer source = decoded.get(0);
if (source.remaining() <= remaining) {
target.append(source);
remaining -= source.remaining();
decoded.remove(0);
}
else {
target.append(source.subSequence(0, remaining));
decoded.set(0, source.subSequence(remaining, source.remaining()));
remaining = 0;
}
}
}

private void eatUndecodedBytes(CharBuffer target, boolean eof) {
if (toDecode != null) {
while (toDecode.size() > 0) {
Expand All @@ -145,6 +280,13 @@ private CharBuffer decodedBuffer(CharBuffer buf) {
return buf.subSequence(0, pos);
}

private void forceDecodedBackToBytes() {
for (int i = decoded.size() - 1; i >= 0; i--) {
toDecode.add(0, charset.encode(decoded.get(i)));
decoded.remove(i);
}
}

public long isEmpty(ThreadContext tc) {
ensureConfigured(tc);
if (toDecode != null && toDecode.size() > 0)
Expand All @@ -156,6 +298,6 @@ public long isEmpty(ThreadContext tc) {

private void ensureConfigured(ThreadContext tc) {
if (decoder == null)
throw ExceptionHandling.dieInternal(tc, "Docder not yet configured");
throw ExceptionHandling.dieInternal(tc, "Decoder not yet configured");
}
}

0 comments on commit bd57598

Please sign in to comment.