From cf0038ffbfa7196f27462e7fa528df3018e90f31 Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Mon, 28 Jun 2021 07:57:54 +0000 Subject: [PATCH 1/2] unblock local development with the pubsub emulator Currently, the functions frameworks are dependent on some private dataplane event marshalling logic in order to correctly pass PubSub events to background functions. This commit implements the same marshalling logic in the FF in order to enable local development using the PubSub emulator. We have already made this change in other languages: https://github.com/GoogleCloudPlatform/functions-framework-nodejs/pull/272 https://github.com/GoogleCloudPlatform/functions-framework-ruby/pull/100 https://github.com/GoogleCloudPlatform/functions-framework-python/pull/121 https://github.com/GoogleCloudPlatform/functions-framework-go/pull/70 --- .../google/cloud/functions/invoker/Event.java | 35 ++++++++++++++++ .../BackgroundFunctionExecutorTest.java | 42 +++++++++++++++++++ .../src/test/resources/pubsub_background.json | 19 +++++++++ .../src/test/resources/pubsub_emulator.json | 10 +++++ 4 files changed, 106 insertions(+) create mode 100644 invoker/core/src/test/resources/pubsub_background.json create mode 100644 invoker/core/src/test/resources/pubsub_emulator.json diff --git a/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java b/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java index 6f52f20b..a0d0a0b2 100644 --- a/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java +++ b/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java @@ -21,6 +21,8 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParseException; import java.lang.reflect.Type; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; /** * Represents an event that should be handled by a background function. This is an internal format @@ -53,6 +55,31 @@ public Event deserialize( context = jsonDeserializationContext.deserialize( adjustContextResource(contextCopy), CloudFunctionsContext.class); + } else if (isPubSubEmulatorPayload(root)) { + JsonObject message = root.getAsJsonObject("message"); + + String timestampString = message.has("publishTime") + ? message.get("publishTime").getAsString() + : DateTimeFormatter.ISO_INSTANT.format(OffsetDateTime.now()); + + context = CloudFunctionsContext.builder() + .setEventType("google.pubsub.topic.publish") + .setTimestamp(timestampString) + .setEventId(message.get("messageId").getAsString()) + .setResource("{" + + "\"name\":null," + + "\"service\":\"pubsub.googleapis.com\"," + + "\"type\":\"type.googleapis.com/google.pubsub.v1.PubsubMessage\""+ + "}") + .build(); + + JsonObject marshalledData = new JsonObject(); + marshalledData.addProperty("@type", "type.googleapis.com/google.pubsub.v1.PubsubMessage"); + marshalledData.add("data", message.get("data")); + if (message.has("attributes")) { + marshalledData.add("attributes", message.get("attributes")); + } + data = marshalledData; } else { JsonObject rootCopy = root.deepCopy(); rootCopy.remove("data"); @@ -63,6 +90,14 @@ public Event deserialize( return Event.of(data, context); } + private boolean isPubSubEmulatorPayload(JsonObject root) { + if (root.has("subscription") && root.has("message") && root.get("message").isJsonObject()) { + JsonObject message = root.getAsJsonObject("message"); + return message.has("data") && message.has("messageId"); + } + return false; + } + /** * Replaces 'resource' member from context JSON with its string equivalent. The original * 'resource' member can be a JSON object itself while {@link CloudFunctionsContext} requires it diff --git a/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java b/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java index 02c5d630..52b00fd2 100644 --- a/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java +++ b/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java @@ -2,14 +2,22 @@ import static com.google.cloud.functions.invoker.BackgroundFunctionExecutor.backgroundFunctionTypeArgument; import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import com.google.cloud.functions.BackgroundFunction; import com.google.cloud.functions.Context; +import com.google.gson.JsonObject; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; + @RunWith(JUnit4.class) public class BackgroundFunctionExecutorTest { private static class PubSubMessage { @@ -62,4 +70,38 @@ public void backgroundFunctionTypeArgument_raw() { (Class>) (Class) ForgotTypeParameter.class; assertThat(backgroundFunctionTypeArgument(c)).isEmpty(); } + + @Test + public void parseLegacyEventPubSub() throws IOException { + try (Reader reader = new InputStreamReader(getClass().getResourceAsStream("/pubsub_background.json"))) { + Event event = BackgroundFunctionExecutor.parseLegacyEvent(reader); + + Context context = event.getContext(); + assertEquals("google.pubsub.topic.publish", context.eventType()); + assertEquals("1", context.eventId()); + assertEquals("2021-06-28T05:46:32.390Z", context.timestamp()); + + JsonObject data = event.getData().getAsJsonObject(); + assertEquals(data.get("data").getAsString(), "eyJmb28iOiJiYXIifQ=="); + String attr = data.get("attributes").getAsJsonObject().get("test").getAsString(); + assertEquals(attr, "123"); + } + } + + @Test + public void parseLegacyEventPubSubEmulator() throws IOException { + try (Reader reader = new InputStreamReader(getClass().getResourceAsStream("/pubsub_emulator.json"))) { + Event event = BackgroundFunctionExecutor.parseLegacyEvent(reader); + + Context context = event.getContext(); + assertEquals("google.pubsub.topic.publish", context.eventType()); + assertEquals("1", context.eventId()); + assertNotNull(context.timestamp()); + + JsonObject data = event.getData().getAsJsonObject(); + assertEquals(data.get("data").getAsString(), "eyJmb28iOiJiYXIifQ=="); + String attr = data.get("attributes").getAsJsonObject().get("test").getAsString(); + assertEquals(attr, "123"); + } + } } diff --git a/invoker/core/src/test/resources/pubsub_background.json b/invoker/core/src/test/resources/pubsub_background.json new file mode 100644 index 00000000..5f9927cb --- /dev/null +++ b/invoker/core/src/test/resources/pubsub_background.json @@ -0,0 +1,19 @@ +{ + "data": { + "@type": "type.googleapis.com/google.pubsub.v1.PubsubMessage", + "data": "eyJmb28iOiJiYXIifQ==", + "attributes": { + "test": "123" + } + }, + "context": { + "eventId": "1", + "eventType": "google.pubsub.topic.publish", + "resource": { + "name": "projects/FOO/topics/BAR_TOPIC", + "service": "pubsub.googleapis.com", + "type": "type.googleapis.com/google.pubsub.v1.PubsubMessage" + }, + "timestamp": "2021-06-28T05:46:32.390Z" + } +} \ No newline at end of file diff --git a/invoker/core/src/test/resources/pubsub_emulator.json b/invoker/core/src/test/resources/pubsub_emulator.json new file mode 100644 index 00000000..cdfe340a --- /dev/null +++ b/invoker/core/src/test/resources/pubsub_emulator.json @@ -0,0 +1,10 @@ +{ + "subscription": "projects/FOO/subscriptions/BAR_SUB", + "message": { + "data": "eyJmb28iOiJiYXIifQ==", + "messageId": "1", + "attributes": { + "test": "123" + } + } +} \ No newline at end of file From dec93ac433d6ad4bd22f935609312920b3bdff16 Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Fri, 2 Jul 2021 22:19:44 +0000 Subject: [PATCH 2/2] fix code style issues --- .../google/cloud/functions/invoker/Event.java | 29 ++++++------ .../BackgroundFunctionExecutorTest.java | 44 ++++++++++--------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java b/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java index a0d0a0b2..ed12efe2 100644 --- a/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java +++ b/invoker/core/src/main/java/com/google/cloud/functions/invoker/Event.java @@ -58,20 +58,23 @@ public Event deserialize( } else if (isPubSubEmulatorPayload(root)) { JsonObject message = root.getAsJsonObject("message"); - String timestampString = message.has("publishTime") - ? message.get("publishTime").getAsString() - : DateTimeFormatter.ISO_INSTANT.format(OffsetDateTime.now()); + String timestampString = + message.has("publishTime") + ? message.get("publishTime").getAsString() + : DateTimeFormatter.ISO_INSTANT.format(OffsetDateTime.now()); - context = CloudFunctionsContext.builder() - .setEventType("google.pubsub.topic.publish") - .setTimestamp(timestampString) - .setEventId(message.get("messageId").getAsString()) - .setResource("{" + - "\"name\":null," + - "\"service\":\"pubsub.googleapis.com\"," + - "\"type\":\"type.googleapis.com/google.pubsub.v1.PubsubMessage\""+ - "}") - .build(); + context = + CloudFunctionsContext.builder() + .setEventType("google.pubsub.topic.publish") + .setTimestamp(timestampString) + .setEventId(message.get("messageId").getAsString()) + .setResource( + "{" + + "\"name\":null," + + "\"service\":\"pubsub.googleapis.com\"," + + "\"type\":\"type.googleapis.com/google.pubsub.v1.PubsubMessage\"" + + "}") + .build(); JsonObject marshalledData = new JsonObject(); marshalledData.addProperty("@type", "type.googleapis.com/google.pubsub.v1.PubsubMessage"); diff --git a/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java b/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java index 52b00fd2..d00b0b4f 100644 --- a/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java +++ b/invoker/core/src/test/java/com/google/cloud/functions/invoker/BackgroundFunctionExecutorTest.java @@ -1,14 +1,12 @@ package com.google.cloud.functions.invoker; import static com.google.cloud.functions.invoker.BackgroundFunctionExecutor.backgroundFunctionTypeArgument; +import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import com.google.cloud.functions.BackgroundFunction; import com.google.cloud.functions.Context; import com.google.gson.JsonObject; - import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; @@ -17,7 +15,6 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; - @RunWith(JUnit4.class) public class BackgroundFunctionExecutorTest { private static class PubSubMessage { @@ -28,7 +25,8 @@ private static class PubSubMessage { } private static class PubSubFunction implements BackgroundFunction { - @Override public void accept(PubSubMessage payload, Context context) {} + @Override + public void accept(PubSubMessage payload, Context context) {} } @Test @@ -39,7 +37,8 @@ public void backgroundFunctionTypeArgument_simple() { private abstract static class Parent implements BackgroundFunction {} private static class Child extends Parent { - @Override public void accept(PubSubMessage payload, Context context) {} + @Override + public void accept(PubSubMessage payload, Context context) {} } @Test @@ -50,7 +49,8 @@ public void backgroundFunctionTypeArgument_superclass() { private interface GenericParent extends BackgroundFunction {} private static class GenericChild implements GenericParent { - @Override public void accept(PubSubMessage payload, Context context) {} + @Override + public void accept(PubSubMessage payload, Context context) {} } @Test @@ -60,7 +60,8 @@ public void backgroundFunctionTypeArgument_genericInterface() { @SuppressWarnings("rawtypes") private static class ForgotTypeParameter implements BackgroundFunction { - @Override public void accept(Object payload, Context context) {} + @Override + public void accept(Object payload, Context context) {} } @Test @@ -73,35 +74,38 @@ public void backgroundFunctionTypeArgument_raw() { @Test public void parseLegacyEventPubSub() throws IOException { - try (Reader reader = new InputStreamReader(getClass().getResourceAsStream("/pubsub_background.json"))) { + try (Reader reader = + new InputStreamReader(getClass().getResourceAsStream("/pubsub_background.json"))) { Event event = BackgroundFunctionExecutor.parseLegacyEvent(reader); Context context = event.getContext(); - assertEquals("google.pubsub.topic.publish", context.eventType()); - assertEquals("1", context.eventId()); - assertEquals("2021-06-28T05:46:32.390Z", context.timestamp()); + assertThat(context.eventType()).isEqualTo("google.pubsub.topic.publish"); + assertThat(context.eventId()).isEqualTo("1"); + assertThat(context.timestamp()).isEqualTo("2021-06-28T05:46:32.390Z"); JsonObject data = event.getData().getAsJsonObject(); - assertEquals(data.get("data").getAsString(), "eyJmb28iOiJiYXIifQ=="); + assertThat(data.get("data").getAsString()).isEqualTo("eyJmb28iOiJiYXIifQ=="); String attr = data.get("attributes").getAsJsonObject().get("test").getAsString(); - assertEquals(attr, "123"); + assertThat(attr).isEqualTo("123"); } } @Test public void parseLegacyEventPubSubEmulator() throws IOException { - try (Reader reader = new InputStreamReader(getClass().getResourceAsStream("/pubsub_emulator.json"))) { + try (Reader reader = + new InputStreamReader(getClass().getResourceAsStream("/pubsub_emulator.json"))) { Event event = BackgroundFunctionExecutor.parseLegacyEvent(reader); Context context = event.getContext(); - assertEquals("google.pubsub.topic.publish", context.eventType()); - assertEquals("1", context.eventId()); - assertNotNull(context.timestamp()); + assertThat(context.eventType()).isEqualTo("google.pubsub.topic.publish"); + assertThat(context.eventId()).isEqualTo("1"); + assertThat(context.timestamp()).isNotNull(); + ; JsonObject data = event.getData().getAsJsonObject(); - assertEquals(data.get("data").getAsString(), "eyJmb28iOiJiYXIifQ=="); + assertThat(data.get("data").getAsString()).isEqualTo("eyJmb28iOiJiYXIifQ=="); String attr = data.get("attributes").getAsJsonObject().get("test").getAsString(); - assertEquals(attr, "123"); + assertThat(attr).isEqualTo("123"); } } }