From 6198ffe3e25477cefc15ce93b6824caba8c4ebf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sun, 7 Mar 2021 14:02:27 -0500 Subject: [PATCH 1/5] Add FastInfoSet exporter --- src/main/java/co/zeroae/gate/Utils.java | 3 +++ src/test/java/co/zeroae/gate/AppTest.java | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/main/java/co/zeroae/gate/Utils.java b/src/main/java/co/zeroae/gate/Utils.java index 5ab5428..e915f84 100644 --- a/src/main/java/co/zeroae/gate/Utils.java +++ b/src/main/java/co/zeroae/gate/Utils.java @@ -81,9 +81,12 @@ static void loadDocumentFormats() { static Map loadExporters() { final GateXMLExporter gateXMLExporter = new GateXMLExporter(); final GATEJsonExporter gateJsonExporter = new GATEJsonExporter(); + final FastInfosetExporter fastInfosetExporter = new FastInfosetExporter(); + final Map rv = new HashMap<>(); rv.put("application/gate+xml", gateXMLExporter); rv.put("application/gate+json", gateJsonExporter); + rv.put("application/fastinfoset", fastInfosetExporter); return Collections.unmodifiableMap(rv); } } diff --git a/src/test/java/co/zeroae/gate/AppTest.java b/src/test/java/co/zeroae/gate/AppTest.java index bd46cb7..35def43 100644 --- a/src/test/java/co/zeroae/gate/AppTest.java +++ b/src/test/java/co/zeroae/gate/AppTest.java @@ -95,6 +95,17 @@ public void testGateJSONResponse() throws Exception { } } + @Test + public void testFastInfosetResponse() { + input_headers.put("Accept", "application/fastinfoset"); + + final APIGatewayProxyResponseEvent result = app.handleRequest(input, context); + assertEquals(200, result.getStatusCode().intValue()); + + // Ensure we get back application/fastinfoset back + assertEquals("application/fastinfoset", result.getHeaders().get("Content-Type")); + } + @Test public void testCache() { input.withBody(input.getBody() + new Random().nextInt()); From cd88dd972b1172e683bb1028f884f8bc08e82d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sun, 7 Mar 2021 14:05:40 -0500 Subject: [PATCH 2/5] Add application/json back as export type --- src/main/java/co/zeroae/gate/Utils.java | 1 + src/test/java/co/zeroae/gate/AppTest.java | 24 ++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/co/zeroae/gate/Utils.java b/src/main/java/co/zeroae/gate/Utils.java index e915f84..6dc5cee 100644 --- a/src/main/java/co/zeroae/gate/Utils.java +++ b/src/main/java/co/zeroae/gate/Utils.java @@ -86,6 +86,7 @@ static Map loadExporters() { final Map rv = new HashMap<>(); rv.put("application/gate+xml", gateXMLExporter); rv.put("application/gate+json", gateJsonExporter); + rv.put("application/json", gateJsonExporter); rv.put("application/fastinfoset", fastInfosetExporter); return Collections.unmodifiableMap(rv); } diff --git a/src/test/java/co/zeroae/gate/AppTest.java b/src/test/java/co/zeroae/gate/AppTest.java index 35def43..619cbac 100644 --- a/src/test/java/co/zeroae/gate/AppTest.java +++ b/src/test/java/co/zeroae/gate/AppTest.java @@ -81,17 +81,19 @@ public void testMissingContentType() { @Test public void testGateJSONResponse() throws Exception { - input_headers.put("Accept", "application/gate+json"); - - final APIGatewayProxyResponseEvent result = app.handleRequest(input, context); - assertEquals(200, result.getStatusCode().intValue()); - - // Ensure we get back application/gate+json back - assertEquals("application/gate+json", result.getHeaders().get("Content-Type")); - final JsonFactory factory = new JsonFactory(); - final JsonParser parser = factory.createParser(result.getBody()); - while (!parser.isClosed()) { - parser.nextToken(); + for (String responseType: new String[]{"application/gate+json", "application/json"}) { + input_headers.put("Accept", responseType); + + final APIGatewayProxyResponseEvent result = app.handleRequest(input, context); + assertEquals(200, result.getStatusCode().intValue()); + + // Ensure we get back application/gate+json back + assertEquals(responseType, result.getHeaders().get("Content-Type")); + final JsonFactory factory = new JsonFactory(); + final JsonParser parser = factory.createParser(result.getBody()); + while (!parser.isClosed()) { + parser.nextToken(); + } } } From 990520e5ceb516cfc6081686ba929410a036a364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sun, 7 Mar 2021 15:13:02 -0500 Subject: [PATCH 3/5] Ensure application/fastinfoset format exports as Base64Encoded body. --- src/main/java/co/zeroae/gate/App.java | 21 +++++++++++++++++---- src/test/java/co/zeroae/gate/AppTest.java | 1 + 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/co/zeroae/gate/App.java b/src/main/java/co/zeroae/gate/App.java index 2837997..392dc13 100644 --- a/src/main/java/co/zeroae/gate/App.java +++ b/src/main/java/co/zeroae/gate/App.java @@ -5,6 +5,7 @@ import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.amazonaws.util.Base64; import com.amazonaws.xray.AWSXRay; import com.amazonaws.xray.entities.Subsegment; @@ -108,7 +109,7 @@ public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent in AWSXRay.beginSubsegment("Gate Export"); AWSXRay.getCurrentSubsegment().putMetadata("Content-Type", responseType); try { - return response.withBody(export(doc, exporter)).withStatusCode(200); + return export(exporter, doc, response).withStatusCode(200); } finally { Factory.deleteResource(doc); AWSXRay.endSubsegment(); @@ -181,10 +182,16 @@ private String computeMessageDigest(FeatureMap featureMap) { } /** - * @param doc an instance of gate.Document * @param exporter The document exporter + * @param doc an instance of gate.Document + * @param response The response where we put the exported Document as body + * @return the modified response */ - private String export(Document doc, DocumentExporter exporter) throws IOException { + private APIGatewayProxyResponseEvent export( + DocumentExporter exporter, + Document doc, + APIGatewayProxyResponseEvent response + ) throws IOException { final FeatureMap exportOptions = Factory.newFeatureMap(); // Take *all* annotation types. @@ -202,7 +209,13 @@ private String export(Document doc, DocumentExporter exporter) throws IOExceptio AWSXRay.getCurrentSubsegment().addException(e); throw e; } - return baos.toString(); + // If we add a second type, then we should create a "Set" at the Utils level and test against it. + if (exporter.getMimeType().equals("application/fastinfoset")) { + response.withIsBase64Encoded(true).setBody(Base64.encodeAsString(baos.toByteArray())); + } else { + response.setBody(baos.toString()); + } + return response; } private static CorpusController loadApplication() { diff --git a/src/test/java/co/zeroae/gate/AppTest.java b/src/test/java/co/zeroae/gate/AppTest.java index 619cbac..5cabf79 100644 --- a/src/test/java/co/zeroae/gate/AppTest.java +++ b/src/test/java/co/zeroae/gate/AppTest.java @@ -106,6 +106,7 @@ public void testFastInfosetResponse() { // Ensure we get back application/fastinfoset back assertEquals("application/fastinfoset", result.getHeaders().get("Content-Type")); + assertTrue(result.getIsBase64Encoded()); } @Test From d461070b429016d370c376c8b5ab2a82ae024064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Mon, 8 Mar 2021 01:49:02 -0500 Subject: [PATCH 4/5] Support FastInfoset input --- src/main/java/co/zeroae/gate/App.java | 28 +++++++++----- src/main/java/co/zeroae/gate/b64/Handler.java | 35 ++++++++++++++++++ src/test/java/co/zeroae/gate/AppTest.java | 17 +++++++++ .../resources/co/zeroae/gate/example.finf | Bin 0 -> 1212 bytes 4 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 src/main/java/co/zeroae/gate/b64/Handler.java create mode 100644 src/test/resources/co/zeroae/gate/example.finf diff --git a/src/main/java/co/zeroae/gate/App.java b/src/main/java/co/zeroae/gate/App.java index 392dc13..08af9c8 100644 --- a/src/main/java/co/zeroae/gate/App.java +++ b/src/main/java/co/zeroae/gate/App.java @@ -1,5 +1,6 @@ package co.zeroae.gate; +import co.zeroae.gate.b64.Handler; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; @@ -71,11 +72,18 @@ public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent in response.getHeaders().put("Content-Type", responseType); } - final String bodyType = input.getHeaders().get("Content-Type"); final FeatureMap featureMap = Factory.newFeatureMap(); - featureMap.put(Document.DOCUMENT_STRING_CONTENT_PARAMETER_NAME, input.getBody()); + final String bodyType = input.getHeaders().get("Content-Type"); if (bodyType != null) featureMap.put(Document.DOCUMENT_MIME_TYPE_PARAMETER_NAME, bodyType); + if (input.getIsBase64Encoded() != null && input.getIsBase64Encoded()) + featureMap.put( + Document.DOCUMENT_URL_PARAMETER_NAME, + new URL("b64", "localhost", 64, input.getBody(), + new Handler())); + else + featureMap.put(Document.DOCUMENT_STRING_CONTENT_PARAMETER_NAME, input.getBody()); + final String inputDigest = AWSXRay.createSubsegment("Message Digest", (subsegment) -> { String rv = computeMessageDigest(featureMap); @@ -116,12 +124,12 @@ public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent in } } catch (GateException e) { logger.error(e); - AWSXRay.getCurrentSegment().addException(e); + AWSXRay.getCurrentSubsegmentOptional().ifPresent((segment -> segment.addException(e))); response.getHeaders().put("Content-Type", "text/plain"); return response.withBody(e.getMessage()).withStatusCode(400); } catch (IOException e) { logger.error(e); - AWSXRay.getCurrentSegment().addException(e); + AWSXRay.getCurrentSubsegmentOptional().ifPresent((segment -> segment.addException(e))); response.getHeaders().put("Content-Type", "text/plain"); return response.withBody(e.getMessage()).withStatusCode(406); } @@ -166,19 +174,21 @@ private Document cacheComputeIfNull(String key, Utils.TextProcessor processor) t } private String computeMessageDigest(FeatureMap featureMap) { - final String sha256; try { final MessageDigest md = MessageDigest.getInstance("SHA-256"); - final String bodyContent = (String)featureMap.get(Document.DOCUMENT_STRING_CONTENT_PARAMETER_NAME); final String mimeType = (String)featureMap.get(Document.DOCUMENT_MIME_TYPE_PARAMETER_NAME); - md.update(bodyContent.getBytes()); + final String bodyContent = (String)featureMap.get(Document.DOCUMENT_STRING_CONTENT_PARAMETER_NAME); + final URL sourceUrl = (URL)featureMap.get(Document.DOCUMENT_URL_PARAMETER_NAME); if (mimeType != null) md.update(mimeType.getBytes()); - sha256 = Hex.encode(md.digest()); + if (bodyContent != null) + md.update(bodyContent.getBytes()); + if (sourceUrl != null) + md.update(sourceUrl.toString().getBytes()); + return Hex.encode(md.digest()); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); } - return sha256; } /** diff --git a/src/main/java/co/zeroae/gate/b64/Handler.java b/src/main/java/co/zeroae/gate/b64/Handler.java new file mode 100644 index 0000000..3ff9726 --- /dev/null +++ b/src/main/java/co/zeroae/gate/b64/Handler.java @@ -0,0 +1,35 @@ +package co.zeroae.gate.b64; + +import com.amazonaws.util.StringInputStream; +import org.apache.commons.codec.binary.Base64InputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +public class Handler extends URLStreamHandler { + private class Connection extends URLConnection { + /** + * Constructs a URL connection to the specified URL. A connection to + * the object referenced by the URL is not created. + * + * @param url the specified URL. + */ + protected Connection(URL url) { + super(url); + } + @Override + public void connect() { + } + @Override + public InputStream getInputStream() throws IOException { + return new Base64InputStream(new StringInputStream(url.getPath())); + } + } + @Override + protected URLConnection openConnection(URL u) { + return new Connection(u); + } +} diff --git a/src/test/java/co/zeroae/gate/AppTest.java b/src/test/java/co/zeroae/gate/AppTest.java index 5cabf79..b66cad1 100644 --- a/src/test/java/co/zeroae/gate/AppTest.java +++ b/src/test/java/co/zeroae/gate/AppTest.java @@ -2,6 +2,7 @@ import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.amazonaws.util.Base64; import com.amazonaws.util.IOUtils; import com.amazonaws.xray.AWSXRay; import com.fasterxml.jackson.core.JsonFactory; @@ -170,6 +171,22 @@ public void testMediaWikiInput() throws IOException { assertFalse(result.getBody().contains("{{Short Description|")); } + @Test + public void testFastInfosetInput() throws IOException { + final byte[] body = IOUtils.toByteArray(getClass().getResourceAsStream("example.finf")); + input.withIsBase64Encoded(true) + .withBody(Base64.encodeAsString(body)) + .getHeaders().put("Content-Type", "application/fastinfoset"); + + final APIGatewayProxyResponseEvent result = app.handleRequest(input, context); + assertEquals(200, result.getStatusCode().intValue()); + assertEquals("MISS", result.getHeaders().get("x-zae-gate-cache")); + + final APIGatewayProxyResponseEvent cachedResult = app.handleRequest(input, context); + assertEquals(200, cachedResult.getStatusCode().intValue()); + assertEquals("HIT", cachedResult.getHeaders().get("x-zae-gate-cache")); + } + @Test public void testCacheCoherenceWithContentType() { final int cacheBust= new Random().nextInt(); diff --git a/src/test/resources/co/zeroae/gate/example.finf b/src/test/resources/co/zeroae/gate/example.finf new file mode 100644 index 0000000000000000000000000000000000000000..9037c02d1427496d3f9ed6a1a20d9314bbe4b241 GIT binary patch literal 1212 zcmZWp&2rN)5RMbaOs!=?qW$A+zR@H$I#9uSn+L)ohp+de>!JPzy8+SON0a-O0A^yNW5N3<$(Tmd;`Iv^F2AIG z(mC@;blp(jOOk~0vFBpSqU_Wv7-*5fK4sb2IfSF>P36&F4w1!~ggzh)vS>hM7x_d` z?9#VT1LHRF5m>*|0huF+wiE&;VF@*A*EZ&pqDfqrH;c8c`53|S44@+}T)SDUSF7{! z1GR3>LEBu5waGCKT##gpODv)JpeYwJ^;@V-9W~sKRn_An$7Fm>DzA#2BI0?PrE0a$hoJR!F!Dt=2&fP9$d~g)K}x1-VP(!#{E} zJxfa1zh+3ve${H_z|>B~3z#K?#So~llDW`#_BrnbUisX}uG8q_6E1R-Uv>l`QoW}n zZ=^Tco=auE#M{1=DU7*b@f|PBJlA*IF!+R71cbnpLhIEHy?Mxbj1O48vlC4OZu;i* z Date: Mon, 8 Mar 2021 01:50:19 -0500 Subject: [PATCH 5/5] Add application/fastinfoset as BinaryMediaType --- examples/complete/template.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/complete/template.yaml b/examples/complete/template.yaml index 41244f4..5477949 100644 --- a/examples/complete/template.yaml +++ b/examples/complete/template.yaml @@ -7,6 +7,9 @@ Description: > # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst Globals: + Api: + BinaryMediaTypes: + - application~1fastinfoset Function: Timeout: 120