Skip to content

Commit

Permalink
ndjson handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@rmannibucau
rmannibucau committed Mar 26, 2023
1 parent c20ece0 commit e9d1883
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 28 deletions.
106 changes: 82 additions & 24 deletions src/main/java/io/yupiik/yuc/command/DefaultCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,13 @@
import io.yupiik.yuc.io.IO;
import org.fusesource.jansi.internal.CLibrary;

import java.io.FilterReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.function.Supplier;

@Command(name = "default", description = "" +
"Format the output as a prettified JSON." +
Expand All @@ -52,39 +56,91 @@ public DefaultCommand(final Conf conf, final IO io, final JsonMapper jsonMapper)
@Override
public void run() {
final var charset = Charset.forName(conf.charset());
final var input = io.openInput(charset, conf.input());
try (final var parser = new JsonParser(input, conf.bufferProviderSize(), new BufferProvider(conf.bufferProviderSize()), true);
final var bufferProvider = new BufferProvider(conf.bufferProviderSize());
try (final var input = io.openInput(charset, conf.input());
final var writer = io.openOutput(charset, conf.output())) {
final var visitor = newVisitor(writer, charset);
while (parser.hasNext()) {
switch (parser.next()) {
case START_ARRAY -> visitor.onStartArray();
case END_ARRAY -> visitor.onEndArray();
case START_OBJECT -> visitor.onStartObject();
case END_OBJECT -> visitor.onEndObject();
case KEY_NAME -> visitor.onKey(parser.getString());
case VALUE_STRING -> visitor.onString(parser.getString());
case VALUE_TRUE -> visitor.onBoolean(true);
case VALUE_FALSE -> visitor.onBoolean(false);
case VALUE_NUMBER -> visitor.onNumber(parser.getString());
case VALUE_NULL -> visitor.onNull();
final var visitorFactory = newVisitor(writer, charset);
if (!conf.ndjson()) {
try (final var parser = newParser(bufferProvider, input)) {
onData(parser, writer, visitorFactory.get());
}
}
visitor.onEnd();
if (conf.appendEol()) {
writer.write('\n');
afterLine(writer);
} else {
String line;
boolean addEol = true;
while ((line = input.readLine()) != null) {
if (addEol) {
addEol = false;
} else {
writer.write('\n');
}
try {
if (isDataLine(line)) {
try (final var parser = newParser(bufferProvider, new StringReader(line.strip()))) {
onData(parser, writer, visitorFactory.get());
}
} else if (!conf.ndjsonIgnoreUnknownLines()) {
writer.write(line);
} else {
addEol = true;
}
} catch (final RuntimeException | IOException e) {
if (!conf.ndjsonIgnoreUnknownLines()) {
writer.write(line);
}
}
}
afterLine(writer);
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}
}

private JsonVisitor newVisitor(final Writer writer, final Charset charset) {
private boolean isDataLine(final String line) {
return (line.startsWith("{") && line.endsWith("}")) ||
(line.startsWith("<") && line.endsWith(">"));
}

private JsonParser newParser(final BufferProvider bufferProvider, final Reader input) {
return new JsonParser(new FilterReader(input) {
@Override
public void close() {
// no-op
}
}, conf.bufferProviderSize(), bufferProvider, true);
}

private void onData(final JsonParser parser, final Writer writer, final JsonVisitor visitor) throws IOException {
while (parser.hasNext()) {
switch (parser.next()) {
case START_ARRAY -> visitor.onStartArray();
case END_ARRAY -> visitor.onEndArray();
case START_OBJECT -> visitor.onStartObject();
case END_OBJECT -> visitor.onEndObject();
case KEY_NAME -> visitor.onKey(parser.getString());
case VALUE_STRING -> visitor.onString(parser.getString());
case VALUE_TRUE -> visitor.onBoolean(true);
case VALUE_FALSE -> visitor.onBoolean(false);
case VALUE_NUMBER -> visitor.onNumber(parser.getString());
case VALUE_NULL -> visitor.onNull();
}
}
visitor.onEnd();
}

private void afterLine(final Writer writer) throws IOException {
if (conf.appendEol()) {
writer.write('\n');
}
}

private Supplier<JsonVisitor> newVisitor(final Writer writer, final Charset charset) {
final var output = new SimpleWriter(writer);
return switch (conf.outputType()) {
case HANDLEBARS -> new HandlebarFormatter(output, conf.handlebars(), jsonMapper, charset);
case PRETTY -> new PrettyFormatter(output, getColorScheme());
default -> new DefaultFormatter(output, getColorScheme());
case HANDLEBARS -> () -> new HandlebarFormatter(output, conf.handlebars(), jsonMapper, charset);
case PRETTY -> () -> new PrettyFormatter(output, getColorScheme());
default -> () -> new DefaultFormatter(output, getColorScheme());
};
}

Expand Down Expand Up @@ -118,6 +174,8 @@ public record Conf(
@Property(defaultValue = "true", documentation = "Should the JSON be prettified. Default: `true`.") boolean pretty,
@Property(defaultValue = "\"UTF-8\"", documentation = "Charset to use to read the input stream. Default: `UTF-8`.") String charset,
@Property(defaultValue = "\"-\"", documentation = "Output the command should use, default to `stdout` if set to `-` else a file path. Default: `-`.") String output,
@Property(defaultValue = "\"-\"", documentation = "Input the command should format, default to `stdin` if set to `-` else a file path. Default: `-`.") String input) {
@Property(defaultValue = "\"-\"", documentation = "Input the command should format, default to `stdin` if set to `-` else a file path. Default: `-`.") String input,
@Property(defaultValue = "false", documentation = "If `true`, input is handled per line instead of globally.") boolean ndjson,
@Property(value = "ndjson-ignore-unknown", defaultValue = "false", documentation = "If `true`, not JSON/XML lines are swallowed.") boolean ndjsonIgnoreUnknownLines) {
}
}
9 changes: 5 additions & 4 deletions src/main/java/io/yupiik/yuc/io/IO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,23 @@

import io.yupiik.fusion.framework.api.scope.ApplicationScoped;

import java.io.BufferedReader;
import java.io.FilterInputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PushbackReader;
import java.io.Reader;
import java.io.Writer;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;

@ApplicationScoped
public class IO {
public Reader openInput(final Charset charset, final String value) {
// todo: optimize buffer usages (+ config from CLI - already there anyway, just needs to be propagated)
public BufferedReader openInput(final Charset charset, final String value) {
try {
final var rawReader = switch (value) {
case "&0", "-" -> new InputStreamReader(new FilterInputStream(System.in) {
Expand All @@ -47,9 +48,9 @@ public void close() {
final int first = pushbackReader.read();
pushbackReader.unread(first);
if (first == '<') { // assume xml
return new Xml2JsonReader(pushbackReader);
return new BufferedReader(new Xml2JsonReader(pushbackReader));
}
return pushbackReader;
return new BufferedReader(pushbackReader);
} catch (final IOException e) {
throw new IllegalArgumentException("Invalid input: '" + value + "'", e);
}
Expand Down
47 changes: 47 additions & 0 deletions src/test/java/io/yupiik/yuc/command/DefaultCommandTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,53 @@
import static org.junit.jupiter.api.Assertions.assertEquals;

class DefaultCommandTest {
@FusionCLITest(args = {"default", "--input", "src/test/resources/nd.json", "--ndjson", "true"})
void ndjson(final Stdout stdout) {
assertEquals("""
Picked some options - line to passthru
{
"a-string": "value1",
"a-number": 1234,
"a-boolean": true,
"a-null": null,
"a-nested-object": {
"nested": true
},
"a-list": [
"s1"
]
}
garbage ignored
{
"another": true
}
""", stdout.content());
}

@FusionCLITest(args = {
"default", "--input", "src/test/resources/nd.json",
"--ndjson", "true", "--ndjson-ignore-unknown", "true"
})
void ndjsonIgnoreUnknown(final Stdout stdout) {
assertEquals("""
{
"a-string": "value1",
"a-number": 1234,
"a-boolean": true,
"a-null": null,
"a-nested-object": {
"nested": true
},
"a-list": [
"s1"
]
}
{
"another": true
}
""", stdout.content());
}

@FusionCLITest(args = {
"default", "--input", "src/test/resources/some.json",
"--append-eol", "false",
Expand Down
4 changes: 4 additions & 0 deletions src/test/resources/nd.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Picked some options - line to passthru
{"a-string": "value1", "a-number": 1234, "a-boolean": true, "a-null": null, "a-nested-object": {"nested": true}, "a-list": ["s1"]}
garbage ignored
{"another": true}

0 comments on commit e9d1883

Please sign in to comment.