From ed9a87484531b41e63fa3db9aa7918ba616346e0 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 31 Oct 2019 10:18:42 -0700 Subject: [PATCH] updates --- .../samples/gcp-pubsub-source/README.md | 186 +++++++++--------- .../gcp-pubsub-source/event-display.yaml | 13 ++ .../gcp-pubsub-source/gcp-pubsub-source.yaml | 13 +- .../samples/gcp-pubsub-source/trigger.yaml | 20 +- docs/eventing/sources/README.md | 10 +- docs/eventing/sources/sources.yaml | 10 +- 6 files changed, 127 insertions(+), 125 deletions(-) create mode 100644 docs/eventing/samples/gcp-pubsub-source/event-display.yaml diff --git a/docs/eventing/samples/gcp-pubsub-source/README.md b/docs/eventing/samples/gcp-pubsub-source/README.md index 9b076d23a1c..d44a6b51c1d 100644 --- a/docs/eventing/samples/gcp-pubsub-source/README.md +++ b/docs/eventing/samples/gcp-pubsub-source/README.md @@ -17,65 +17,75 @@ source is most useful as a bridge from other GCP services, such as [Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) and install the `gcloud` CLI and run `gcloud auth login`. This sample will use a mix of `gcloud` and `kubectl` commands. The rest of the sample assumes - that you've set the `$PROJECT_ID` environment variable to your Google Cloud + that you've set the `PROJECT_ID` environment variable to your Google Cloud project id, and also set your project ID as default using `gcloud config set project $PROJECT_ID`. -1. Setup [Knative Serving](../../../install) - -1. Setup [Knative Eventing](../../../eventing). In addition, install the GCP - PubSub event source from `release-gcppubsub.yaml`: - - ```shell - kubectl apply --filename https://github.com/knative/eventing-contrib/releases/download/{{< version >}}/gcppubsub.yaml - ``` - 1. Enable the `Cloud Pub/Sub API` on your project: ```shell gcloud services enable pubsub.googleapis.com ``` -1. Create a - [GCP Service Account](https://console.cloud.google.com/iam-admin/serviceaccounts/project). - This sample creates one service account for both registration and receiving - messages, but you can also create a separate service account for receiving - messages if you want additional privilege separation. - - 1. Create a new service account named `knative-source` with the following - command: - ```shell - gcloud iam service-accounts create knative-source - ``` - 1. Give that Service Account the `Pub/Sub Editor` role on your GCP project: - ```shell - gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member=serviceAccount:knative-source@$PROJECT_ID.iam.gserviceaccount.com \ - --role roles/pubsub.editor - ``` - 1. Download a new JSON private key for that Service Account. **Be sure not to - check this key into source control!** - ```shell - gcloud iam service-accounts keys create knative-source.json \ - --iam-account=knative-source@$PROJECT_ID.iam.gserviceaccount.com - ``` - 1. Create two secrets on the kubernetes cluster with the downloaded key: - - ```shell - # Note that the first secret may already have been created when installing - # Knative Eventing. The following command will overwrite it. If you don't - # want to overwrite it, then skip this command. - kubectl --namespace knative-sources create secret generic gcppubsub-source-key --from-file=key.json=knative-source.json --dry-run --output yaml | kubectl apply --filename - - - # The second secret should not already exist, so just try to create it. - kubectl --namespace default create secret generic google-cloud-key --from-file=key.json=knative-source.json - ``` - - `gcppubsub-source-key` and `key.json` are pre-configured values in the - `controller-manager` StatefulSet which manages your Eventing sources. - - `google-cloud-key` and `key.json` are pre-configured values in - [`gcp-pubsub-source.yaml`](./gcp-pubsub-source.yaml). +1. Setup [Knative Serving](../../../install) + +1. Setup [Knative Eventing](../../../eventing) + +1. In addition, install the PubSub event source from `cloud-run-events.yaml`: + + 1. To install the PubSub source, first install the CRDs by running the `kubectl apply` + command with the `--selector events.cloud.google.com/crd-install=true` flag. This prevents + race conditions during the install, which cause intermittent errors: + + ```shell + kubectl apply --selector events.cloud.google.com/crd-install=true \ + --filename https://github.com/google/knative-gcp/releases/download/{{< version >}}/cloud-run-events.yaml + ``` + + 1. To complete the install of the PubSub source and its dependencies, run the + `kubectl apply` command again, this time without the `--selector` flag: + + ```shell + kubectl apply --filename https://github.com/google/knative-gcp/releases/download/{{< version >}}/cloud-run-events.yaml + ``` + +1. Create a + [Google Cloud Service Account](https://console.cloud.google.com/iam-admin/serviceaccounts/project). + This sample creates one Service Account for both registration and receiving + messages, but you can also create a separate Service Account for receiving + messages if you want additional privilege separation. + + 1. Create a new Service Account named `gcp-source` with the following command: + + ```shell + gcloud iam service-accounts create gcp-source + ``` + + 1. Give that Service Account the `Pub/Sub Editor` role on your Google Cloud + project: + + ```shell + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member=serviceAccount:gcp-source@$PROJECT_ID.iam.gserviceaccount.com \ + --role roles/pubsub.editor + ``` + + 1. Download a new JSON private key for that Service Account. **Be sure not + to check this key into source control!** + + ```shell + gcloud iam service-accounts keys create gcp-source.json \ + --iam-account=gcp-source@$PROJECT_ID.iam.gserviceaccount.com + ``` + + 1. Create a Secret on the Kubernetes cluster with the downloaded key: + + ```shell + # The Secret should not already exist, so just try to create it. + kubectl --namespace default create secret generic google-cloud-key --from-file=key.json=gcp-source.json + ``` + + `google-cloud-key` and `key.json` are default values expected by the PubSub source. ## Deployment @@ -95,26 +105,21 @@ source is most useful as a bridge from other GCP services, such as gcloud pubsub topics create testing ``` -1. Replace the - [`MY_GCP_PROJECT` placeholder](https://cloud.google.com/resource-manager/docs/creating-managing-projects) - in [`gcp-pubsub-source.yaml`](./gcp-pubsub-source.yaml) and apply it. - - If you're in the samples directory, you can replace `MY_GCP_PROJECT` and - apply in one command: +1. If you are *not* running on GKE, uncomment the project line in [`gcp-pubsub-source.yaml`](./gcp-pubsub-source.yaml) + and replace `MY_GCP_PROJECT` with your `PROJECT_ID`. If you are running on GKE, we use GKE's metadata server to + automatically set the project information. Make sure to apply the yaml: ```shell - sed "s/MY_GCP_PROJECT/$PROJECT_ID/g" gcp-pubsub-source.yaml | \ - kubectl apply --filename - + kubectl apply --filename gcp-pubsub-source.yaml ``` - If you are replacing `MY_GCP_PROJECT` manually, then make sure you apply the - resulting YAML: +1. Create a function that will receive the event: ```shell - kubectl apply --filename gcp-pubsub-source.yaml + kubectl apply --filename event-display.yaml ``` -1. Create a function and create a Trigger that will send all events from the +1. Create a Trigger that will send all events from the Broker to the function: ```shell @@ -123,24 +128,20 @@ source is most useful as a bridge from other GCP services, such as ## Publish -Publish messages to your GCP PubSub Topic: +Publish messages to your GCP PubSub topic: ```shell -gcloud pubsub topics publish testing --message="Hello world" +gcloud pubsub topics publish testing --message='{"Hello": "world"}' ``` ## Verify We will verify that the published message was sent into the Knative eventing -system by looking at the logs of the function subscribed to the `pubsub-test` -channel. - -The function and the subscription were created by applying the -[`trigger.yaml`](./trigger.yaml) manifest in the [deployment](#deployment) -section above. +mesh by looking at the logs of the function subscribed, through a Trigger, +to the `default` Broker. 1. We need to wait for the downstream pods to get started and receive our event, - wait 60 seconds. + wait a few seconds. - You can check the status of the downstream pods with: @@ -158,24 +159,33 @@ section above. You should see log lines similar to: -```json -{ - "ID": "284375451531353", - "Data": "SGVsbG8sIHdvcmxk", - "Attributes": null, - "PublishTime": "2018-10-31T00:00:00.00Z" -} -``` - -The log message is a dump of the message sent by `GCP PubSub`. In particular, if -you [base-64 decode](https://www.base64decode.org/) the `Data` field, you should -see the sent message: - ```shell -echo "SGVsbG8sIHdvcmxk" | base64 --decode +☁️ CloudEvent: valid ✅ +Context Attributes, + SpecVersion: 0.3 + Type: com.google.cloud.pubsub.topic.publish + Source: //pubsub.googleapis.com/projects/PROJECT_ID/topics/testing + ID: 815117146007971 + Time: 2019-10-31T04:49:12.582Z + DataContentType: application/json + Extensions: + knativecemode: binary + knativearrivaltime: 2019-10-31T04:49:12Z + knativehistory: default-kne-trigger-kn-channel.default.svc.cluster.local + traceparent: 00-c9659e66c0ed05d6f4fac3e57e62287c-05a289c3e928e698-00 +Transport Context, + URI: / + Host: event-display.default.svc.cluster.local + Method: POST +Data, + { + "Hello": "world" + } ``` -Results in: "Hello world" - -For more information about the format of the message, see the +For more information about the format of the `Data,` see +the `data` field of [PubsubMessage documentation](https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage). + +For more information about CloudEvents, see the +[HTTP transport bindings documentation](https://github.com/cloudevents/spec). diff --git a/docs/eventing/samples/gcp-pubsub-source/event-display.yaml b/docs/eventing/samples/gcp-pubsub-source/event-display.yaml new file mode 100644 index 00000000000..607d1d257eb --- /dev/null +++ b/docs/eventing/samples/gcp-pubsub-source/event-display.yaml @@ -0,0 +1,13 @@ +# This is a very simple Knative Service that writes the input request to its log. + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: event-display +spec: + template: + spec: + containers: + - # This corresponds to + # https://github.com/knative/eventing-contrib/blob/release-0.5/cmd/event_display/main.go + image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display@sha256:bf45b3eb1e7fc4cb63d6a5a6416cf696295484a7662e0cf9ccdf5c080542c21d diff --git a/docs/eventing/samples/gcp-pubsub-source/gcp-pubsub-source.yaml b/docs/eventing/samples/gcp-pubsub-source/gcp-pubsub-source.yaml index 53775059f46..94d269648ce 100644 --- a/docs/eventing/samples/gcp-pubsub-source/gcp-pubsub-source.yaml +++ b/docs/eventing/samples/gcp-pubsub-source/gcp-pubsub-source.yaml @@ -1,17 +1,12 @@ -# Replace the following before applying this file: -# MY_GCP_PROJECT: Replace with the GCP Project's ID. - -apiVersion: sources.eventing.knative.dev/v1alpha1 -kind: GcpPubSubSource +apiVersion: events.cloud.google.com/v1alpha1 +kind: PubSub metadata: name: testing-source spec: - gcpCredsSecret: # A secret in the knative-sources namespace - name: google-cloud-key - key: key.json - googleCloudProject: MY_GCP_PROJECT # Replace this topic: testing sink: apiVersion: eventing.knative.dev/v1alpha1 kind: Broker name: default + # If running in GKE, we will ask the metadata server for the project. + #project: MY_GCP_PROJECT diff --git a/docs/eventing/samples/gcp-pubsub-source/trigger.yaml b/docs/eventing/samples/gcp-pubsub-source/trigger.yaml index f614624788e..a09b4d6b7f7 100644 --- a/docs/eventing/samples/gcp-pubsub-source/trigger.yaml +++ b/docs/eventing/samples/gcp-pubsub-source/trigger.yaml @@ -1,26 +1,10 @@ -# This is a very simple Knative Service that writes the input request to its log. - -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: event-display -spec: - template: - spec: - containers: - - # This corresponds to - # https://github.com/knative/eventing-contrib/blob/release-0.5/cmd/event_display/main.go - image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display@sha256:bf45b3eb1e7fc4cb63d6a5a6416cf696295484a7662e0cf9ccdf5c080542c21d - ---- - -# The GcpPubSubSource's output goes to the default Broker. This Trigger subscribes to events in the +# The PubSub source output goes to the default Broker. This Trigger subscribes to events in the # default Broker. apiVersion: eventing.knative.dev/v1alpha1 kind: Trigger metadata: - name: gcppubsub-source-sample + name: trigger-gcp-pubsub spec: subscriber: ref: diff --git a/docs/eventing/sources/README.md b/docs/eventing/sources/README.md index 6f073416681..dd55600d306 100644 --- a/docs/eventing/sources/README.md +++ b/docs/eventing/sources/README.md @@ -34,17 +34,17 @@ These are sources that are installed as `CRD`s. Name | Status | Support | Description --- | --- | --- | --- -[AWS SQS](https://github.com/knative/eventing-contrib/blob/master/contrib/awssqs/pkg/apis/sources/v1alpha1/aws_sqs_types.go) | Proof of Concept | None | Brings [AWS Simple Queue Service](https://aws.amazon.com/sqs/) messages into Knative. +[AWS SQS](https://github.com/knative/eventing-contrib/blob/master/awssqs/pkg/apis/sources/v1alpha1/aws_sqs_types.go) | Proof of Concept | None | Brings [AWS Simple Queue Service](https://aws.amazon.com/sqs/) messages into Knative. [Apache Camel](https://github.com/knative/eventing-contrib/blob/master/camel/source/pkg/apis/sources/v1alpha1/camelsource_types.go) | Proof of Concept | None | Allows to use [Apache Camel](https://github.com/apache/camel) components for pushing events into Knative. [Apache CouchDB](https://github.com/knative/eventing-contrib/tree/{{< branch >}}/couchdb) | Active Development | None | Brings [Apache CouchDB](https://couchdb.apache.org/) messages into Knative. [Apache Kafka](https://github.com/knative/eventing-contrib/blob/master/kafka/source/pkg/apis/sources/v1alpha1/kafka_types.go) | Proof of Concept | None | Brings [Apache Kafka](https://kafka.apache.org/) messages into Knative. [BitBucket](https://github.com/nachocano/bitbucket-source) | Proof of Concept | None | Registers for events of the specified types on the specified BitBucket organization/repository. Brings those events into Knative. [Cron Job](https://github.com/knative/eventing/blob/master/pkg/apis/sources/v1alpha1/cron_job_types.go) | Proof of Concept | None | Uses an in-memory timer to produce events on the specified Cron schedule. -[GCP PubSub](https://github.com/knative/eventing-contrib/blob/master/contrib/gcppubsub/pkg/apis/sources/v1alpha1/gcp_pubsub_types.go) | Proof of Concept | None | Brings [GCP PubSub](https://cloud.google.com/pubsub/) messages into Knative. -[GitHub](https://github.com/knative/eventing-contrib/blob/master/contrib/github/pkg/apis/sources/v1alpha1/githubsource_types.go) | Proof of Concept | None | Registers for events of the specified types on the specified GitHub organization/repository. Brings those events into Knative. +[GCP PubSub](https://github.com/google/knative-gcp/blob/master/pkg/apis/events/v1alpha1/pubsub_types.go) | Proof of Concept | None | Brings [GCP PubSub](https://cloud.google.com/pubsub/) messages into Knative. +[GitHub](https://github.com/knative/eventing-contrib/blob/master/github/pkg/apis/sources/v1alpha1/githubsource_types.go) | Proof of Concept | None | Registers for events of the specified types on the specified GitHub organization/repository. Brings those events into Knative. [GitLab](https://gitlab.com/triggermesh/gitlabsource) | Proof of Concept | None | Registers for events of the specified types on the specified GitLab repository. Brings those events into Knative. -[Google Cloud Scheduler](https://github.com/vaikas-google/csr) | Active Development | None | Create, update, and delete [Google Cloud Scheduler](https://cloud.google.com/scheduler/) Jobs. When those jobs are triggered, receive the event inside Knative. -[Google Cloud Storage](https://github.com/vaikas-google/gcs) | Active Development | None | Registers for events of the specified types on the specified Google Cloud Storage bucket and optional object prefix. Brings those events into Knative. +[Google Cloud Scheduler](https://github.com/google/knative-gcp/blob/master/pkg/apis/events/v1alpha1/scheduler_types.go) | Active Development | None | Create, update, and delete [Google Cloud Scheduler](https://cloud.google.com/scheduler/) Jobs. When those jobs are triggered, receive the event inside Knative. +[Google Cloud Storage](https://github.com/google/knative-gcp/blob/master/pkg/apis/events/v1alpha1/storage_types.go) | Active Development | None | Registers for events of the specified types on the specified Google Cloud Storage bucket and optional object prefix. Brings those events into Knative. [Kubernetes](https://github.com/knative/eventing/blob/master/pkg/apis/sources/v1alpha1/apiserver_types.go) | Active Development | Knative | Brings Kubernetes API server events into Knative. diff --git a/docs/eventing/sources/sources.yaml b/docs/eventing/sources/sources.yaml index f44cb0c8934..d9157938f80 100644 --- a/docs/eventing/sources/sources.yaml +++ b/docs/eventing/sources/sources.yaml @@ -34,7 +34,7 @@ sources: description: > Allows to use [Apache Camel](https://github.com/apache/camel) components for pushing events into Knative. - name: AWS SQS - url: https://github.com/knative/eventing-contrib/blob/master/contrib/awssqs/pkg/apis/sources/v1alpha1/aws_sqs_types.go + url: https://github.com/knative/eventing-contrib/blob/master/awssqs/pkg/apis/sources/v1alpha1/aws_sqs_types.go status: Proof of Concept support: None description: > @@ -46,13 +46,13 @@ sources: description: > Uses an in-memory timer to produce events on the specified Cron schedule. - name: GCP PubSub - url: https://github.com/knative/eventing-contrib/blob/master/contrib/gcppubsub/pkg/apis/sources/v1alpha1/gcp_pubsub_types.go + url: https://github.com/google/knative-gcp/blob/master/pkg/apis/events/v1alpha1/pubsub_types.go status: Proof of Concept support: None description: > Brings [GCP PubSub](https://cloud.google.com/pubsub/) messages into Knative. - name: GitHub - url: https://github.com/knative/eventing-contrib/blob/master/contrib/github/pkg/apis/sources/v1alpha1/githubsource_types.go + url: https://github.com/knative/eventing-contrib/blob/master/github/pkg/apis/sources/v1alpha1/githubsource_types.go status: Proof of Concept support: None description: > @@ -79,14 +79,14 @@ sources: description: > Brings Kubernetes API server events into Knative. - name: Google Cloud Scheduler - url: https://github.com/vaikas-google/csr + url: https://github.com/google/knative-gcp/blob/master/pkg/apis/events/v1alpha1/scheduler_types.go status: Active Development support: None description: > Create, update, and delete [Google Cloud Scheduler](https://cloud.google.com/scheduler/) Jobs. When those jobs are triggered, receive the event inside Knative. - name: Google Cloud Storage - url: https://github.com/vaikas-google/gcs + url: https://github.com/google/knative-gcp/blob/master/pkg/apis/events/v1alpha1/storage_types.go status: Active Development support: None description: >