From 9ff37c61ba7754838eea61f9f7800c08850b4eee Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Mon, 18 May 2020 14:17:53 +0300 Subject: [PATCH 1/7] Make sample source guide up-to-date --- .../01-theory.md | 26 +++-- .../02-lifecycle-and-types.md | 36 ++++--- .../03-controller.md | 38 ++++--- .../04-reconciler.md | 101 ++++++++++++------ .../05-receive-adapter.md | 81 ++++---------- .../writing-receive-adapter-source/06-yaml.md | 4 +- .../writing-receive-adapter-source/README.md | 21 +++- 7 files changed, 173 insertions(+), 134 deletions(-) diff --git a/docs/eventing/samples/writing-receive-adapter-source/01-theory.md b/docs/eventing/samples/writing-receive-adapter-source/01-theory.md index efda187f920..49b704740f4 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/01-theory.md +++ b/docs/eventing/samples/writing-receive-adapter-source/01-theory.md @@ -70,7 +70,7 @@ Specifically, the `clientset`, `cache`, `informers`, and `listers` can all be ge import ( // ... sampleSourceClient "knative.dev/sample-source/pkg/client/injection/client" - samplesourceinformer “knative.dev/sample-source/pkg/client/injection/informers/samples/v1alpha1/samplesource" + samplesourceinformer "knative.dev/sample-source/pkg/client/injection/informers/samples/v1alpha1/samplesource" ) // ... func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl { @@ -83,20 +83,32 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl // ... } ``` -Ensure that the specific source subdirectory has been added to the injection portion of the `hack/update-codegen.sh` script. + +To have these generated, ensure that the specific source subdirectories has been added to the injection portion of the +[`hack/update-codegen.sh`](https://github.com/knative/eventing-contrib/blob/master/hack/update-codegen.sh) script. ```patch # Sources -+API_DIRS_SOURCES=(github/pkg camel/source/pkg kafka/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg YourSourceHere/pkg) --API_DIRS_SOURCES=(github/pkg camel/source/pkg kafka/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg) ++API_DIRS_SOURCES=(camel/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg YourSourceHere/pkg) +-API_DIRS_SOURCES=(camel/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg) + +# Knative Injection + +chmod +x ${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh +${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh "injection" \ +- knative.dev/sample-source/pkg/client knative.dev/sample-source/pkg/apis \ +- "samples:v1alpha1" \ ++ knative.dev/your-source/pkg/client knative.dev/your-source/pkg/apis \ ++ "your-name:v1alpha1" \ + --go-header-file ${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt ``` and ```patch - -i knative.dev/eventing-contrib/camel/source/pkg/apis \ -- -i knative.dev/eventing-contrib/github/pkg/apis -+ -i knative.dev/eventing-contrib/github/pkg/apis \ + -i knative.dev/eventing-contrib/github/pkg/apis \ +- -i knative.dev/eventing-contrib/gitlab/pkg/apis ++ -i knative.dev/eventing-contrib/gitlab/pkg/apis \ + -i knative.dev/eventing-contrib/YourSourceHere/pkg/apis ``` diff --git a/docs/eventing/samples/writing-receive-adapter-source/02-lifecycle-and-types.md b/docs/eventing/samples/writing-receive-adapter-source/02-lifecycle-and-types.md index 10f6c1ccbad..b60703f7b94 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/02-lifecycle-and-types.md +++ b/docs/eventing/samples/writing-receive-adapter-source/02-lifecycle-and-types.md @@ -32,23 +32,27 @@ type SampleSource struct { // SampleSourceSpec holds the desired state of the SampleSource (from the client). type SampleSourceSpec struct { - // ServiceAccountName holds the name of the Kubernetes service account - // as which the underlying K8s resources should be run. If unspecified - // this will default to the "default" service account for the namespace - // in which the SampleSource exists. - // +optional - ServiceAccountName string `json:"serviceAccountName,omitempty"` + // inherits duck/v1 SourceSpec, which currently provides: + // * Sink - a reference to an object that will resolve to a domain name or + // a URI directly to use as the sink. + // * CloudEventOverrides - defines overrides to control the output format + // and modifications of the event sent to the sink. + duckv1.SourceSpec `json:",inline"` - // Interval is the time interval between events. - // - // The string format is a sequence of decimal numbers, each with optional - // fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time - // units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - Interval string `json:"interval"` + // ServiceAccountName holds the name of the Kubernetes service account + // as which the underlying K8s resources should be run. If unspecified + // this will default to the "default" service account for the namespace + // in which the SampleSource exists. + // +optional + ServiceAccountName string `json:"serviceAccountName,omitempty"` - // Sink is a reference to an object that will resolve to a host - // name to use as the sink. - Sink *duckv1.Destination `json:"sink"` + // Interval is the time interval between events. + // + // The string format is a sequence of decimal numbers, each with optional + // fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time + // units are "ns", "us" (or "µs"), "ms", "s", "m", "h". If unspecified + // this will default to "10s". + Interval string `json:"interval"` } // SampleSourceStatus communicates the observed state of the SampleSource (from the controller). @@ -72,7 +76,7 @@ const ( ``` Define the functions that will be called from the Reconciler functions to set the lifecycle conditions. This is typically done in -`pkg/apis/samples/VERSION/sampleservice_lifecycle.go` +`pkg/apis/samples/VERSION/samplesource_lifecycle.go` ```go // InitializeConditions sets relevant unset conditions to Unknown state. diff --git a/docs/eventing/samples/writing-receive-adapter-source/03-controller.md b/docs/eventing/samples/writing-receive-adapter-source/03-controller.md index 1c5546802a9..6e309807501 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/03-controller.md +++ b/docs/eventing/samples/writing-receive-adapter-source/03-controller.md @@ -11,40 +11,39 @@ Pass the new controller implementation to the shared main ```go import ( // The set of controllers this controller process runs. - "knative.dev/sample-source/pkg/reconciler" + "knative.dev/sample-source/pkg/reconciler/sample" // This defines the shared main for injected controllers. "knative.dev/pkg/injection/sharedmain" ) func main() { - sharedmain.Main("sample-source-controller", - reconciler.NewController - ) + sharedmain.Main("sample-source-controller", sample.NewController) } ``` -Define the NewController implementation, it will be passed a configmap.Watcher, as well as a context which the injected listers will use for the reconciler struct arguments +Define the NewController implementation, it will be passed a `configmap.Watcher`, as well as a context which the injected listers will use for the reconciler struct arguments ```go func NewController( ctx context.Context, cmw configmap.Watcher, ) *controller.Impl { // ... - sampleSourceInformer := samplesourceinformer.Get(ctx) + deploymentInformer := deploymentinformer.Get(ctx) + sinkBindingInformer := sinkbindinginformer.Get(ctx) + sampleSourceInformer := samplesourceinformer.Get(ctx) r := &Reconciler{ - KubeClientSet: kubeclient.Get(ctx), - EventingClientSet: eventingclient.Get(ctx), - samplesourceLister: sampleSourceInformer.Lister(), - deploymentLister: deploymentInformer.Lister(), - samplesourceClientSet: samplesourceClient.Get(ctx), - } + dr: &reconciler.DeploymentReconciler{KubeClientSet: kubeclient.Get(ctx)}, + sbr: &reconciler.SinkBindingReconciler{EventingClientSet: eventingclient.Get(ctx)}, + // Config accessor takes care of tracing/config/logging config propagation to the receive adapter + configAccessor: reconcilersource.WatchConfigurations(ctx, "sample-source", cmw), + } ``` The base reconciler is imported from the knative.dev/pkg dependency: ```go import ( // ... - "knative.dev/eventing/pkg/reconciler" + reconcilersource "knative.dev/eventing/pkg/reconciler/source" // ... ) ``` @@ -52,3 +51,16 @@ Ensure the correct informers have EventHandlers filtered to them ```go sampleSourceInformer.Informer().AddEventHandler(controller.HandleAll(impl.Enqueue)) ``` +Controller for the `SampleSource` uses `Deployment` and `SinkBinding` resources to deploy and also bind the event source and the receive adapter. Also ensure the informers are set up correctly for these secondary resources +```go + deploymentInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: controller.FilterGroupKind(v1alpha1.Kind("SampleSource")), + Handler: controller.HandleAll(impl.EnqueueControllerOf), + }) + + sinkBindingInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: controller.FilterGroupKind(v1alpha1.Kind("SampleSource")), + Handler: controller.HandleAll(impl.EnqueueControllerOf), + }) +``` + diff --git a/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md b/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md index cb661727397..ff9f69de155 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md +++ b/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md @@ -7,64 +7,101 @@ type: "docs" ## Reconciler Functionality General steps the reconciliation process needs to cover: -1. Target the specific samplesource via the `sampleServiceClientSet`: +1. Update the `ObservedGeneration` and initialize the `Status` conditions (as defined in `samplesource_lifecycle.go` and `samplesource_types.go`) ```go -// Get the resource with this namespace/name. -original, err := r.Lister.SampleSources(namespace).Get(name) +src.Status.InitializeConditions() +src.Status.ObservedGeneration = src.Generation +``` +2. Create/reconcile the Receive Adapter (detailed below) +3. If successful, update the `Status` and `MarkDeployed` +```go +src.Status.PropagateDeploymentAvailability(ra) +``` +4. Create/reconcile the `SinkBinding` for the Receive Adapter targeting the `Sink` (detailed below) +5. MarkSink with the result +```go +src.Status.MarkSink(sb.Status.SinkURI) +``` +6. Return a new reconciler event stating that the process is done +```go +return pkgreconciler.NewEvent(corev1.EventTypeNormal, "SampleSourceReconciled", "SampleSource reconciled: \"%s/%s\"", namespace, name) ``` -2. Update the ObservedGeneration -Initialize the Status conditions (as defined in `samplesource_lifecycle.go` and `samplesource_types.go`) -3. Reconcile the Sink and MarkSink with the result -Create the Receive Adapter (detailed below) - 3. If successful, update the Status and MarkDeployed -4. Reconcile the EventTypes and corresponding Status -Creation and deletion of the events is done with the inherited `EventingClientSet().EventingV1alpha1()` api -5. Update the full status field from the resulting reconcile attempt via the source’s clientset and api -`r.samplesourceClientSet.SamplesV1alpha1().SampleSources(desired.Namespace).UpdateStatus(existing)` - ## Reconcile/Create The Receive Adapter As part of the source reconciliation, we have to create and deploy -(and update if necessary) the underlying receive adapter. The two -client sets used in this process is the `kubeClientSet` for the -Deployment tracking, and the `EventingClientSet` for the event -recording. +(and update if necessary) the underlying receive adapter. Verify the specified kubernetes resources are valid, and update the `Status` accordingly Assemble the ReceiveAdapterArgs ```go raArgs := resources.ReceiveAdapterArgs{ - EventSource: eventSource, - Image: r.receiveAdapterImage, - Source: src, - Labels: resources.GetLabels(src.Name), - SinkURI: sinkURI, + EventSource: src.Namespace + "/" + src.Name, + Image: r.ReceiveAdapterImage, + Source: src, + Labels: resources.Labels(src.Name), + AdditionalEnvs: r.configAccessor.ToEnvVars(), // Grab config envs for tracing/logging/metrics } ``` NB The exact arguments may change based on functional requirements Create the underlying deployment from the arguments provided, matching pod templates, labels, owner references, etc as needed to fill out the deployment -Example: [pkg/reconciler/resources/receive_adapter.go](https://github.com/knative/sample-source/tree/master/pkg/reconciler/resources/receive_adapter.go) +Example: [pkg/reconciler/sample/resources/receive_adapter.go](https://github.com/knative/sample-source/blob/master/pkg/reconciler/sample/resources/receive_adapter.go) 1. Fetch the existing receive adapter deployment ```go - ra, err := r.KubeClientSet.AppsV1().Deployments(src.Namespace).Get(expected.Name, metav1.GetOptions{}) +namespace := owner.GetObjectMeta().GetNamespace() +ra, err := r.KubeClientSet.AppsV1().Deployments(namespace).Get(expected.Name, metav1.GetOptions{}) ``` 2. Otherwise, create the deployment ```go -ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Create(expected) +ra, err = r.KubeClientSet.AppsV1().Deployments(namespace).Create(expected) ``` 3. Check if the expected vs existing spec is different, and update the deployment if required ```go -} else if podSpecChanged(ra.Spec.Template.Spec, expected.Spec.Template.Spec) { - ra.Spec.Template.Spec = expected.Spec.Template.Spec - if ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Update(ra); err != nil { - return ra, err - } +} else if r.podSpecImageSync(expected.Spec.Template.Spec, ra.Spec.Template.Spec) { + ra.Spec.Template.Spec = expected.Spec.Template.Spec + if ra, err = r.KubeClientSet.AppsV1().Deployments(namespace).Update(ra); err != nil { + return ra, err + } ``` 4. If updated, record the event ```go - r.Recorder.Eventf(src, corev1.EventTypeNormal, samplesourceDeploymentUpdated, "Deployment updated") - return ra, nil +return pkgreconciler.NewEvent(corev1.EventTypeNormal, "DeploymentUpdated", "updated deployment: \"%s/%s\"", namespace, name) +``` + +## Reconcile/Create The SinkBinding +Instead of directly giving the details of the sink to the receive adapter, use a `SinkBinding` to bind the receive adapter with the sink. + +Steps here are almost the same with the `Deployment` reconciliation above, but it is for another resource, `SinkBinding`. + +1. Create a `Reference` for the receive adapter deployment. This deployment will be `SinkBinding`'s source: +```go +tracker.Reference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Namespace: ra.Namespace, + Name: ra.Name, +} ``` +2. Fetch the existing `SinkBinding` +```go +namespace := owner.GetObjectMeta().GetNamespace() +sb, err := r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Get(expected.Name, metav1.GetOptions{}) +``` +2. If it doesn't exist, create it +```go +sb, err = r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Create(expected) +``` +3. Check if the expected vs existing spec is different, and update the `SinkBinding` if required +```go +else if r.specChanged(sb.Spec, expected.Spec) { + sb.Spec = expected.Spec + if sb, err = r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Update(sb); err != nil { + return sb, err + } +``` +4. If updated, record the event +```go +return pkgreconciler.NewEvent(corev1.EventTypeNormal, "SinkBindingUpdated", "updated SinkBinding: \"%s/%s\"", namespace, name) +``` diff --git a/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md b/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md index c5a43ff5203..c0bae379f76 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md +++ b/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md @@ -6,8 +6,7 @@ type: "docs" --- ## Receive Adapter cmd -Similar to the controller, we'll need an injection based main.go similar to the controller -under `cmd/receiver_adapter/main.go` +Similar to the controller, we'll need an injection based `main.go` similar to the controller under `cmd/receiver_adapter/main.go` ```go // This Adapter generates events at a regular interval. package main @@ -23,10 +22,12 @@ func main() { ``` -## Defining NewAdapter implmentation and Start function -The adapter's `pkg` implemenation constists of two main functions; +## Defining NewAdapter implementation and Start function +The adapter's `pkg` implementation consists of two main functions; -a `NewAdapter(ctx context.Context, aEnv adapter.EnvConfigAccessor, sink cloudevents.Client, reporter source.StatsReporter) adapter.Adapter {}` call. Which creates the new adapter with passed variables via the EnvConfigAccessor, and sets up the cloudevents client (which is where the events are forwarded to). This is sometimes refered to as a sink, or ceClient in the knative ecosystem. The return value is a refernce to the adapter as defined by the adapter's local struct. +1. A `NewAdapter(ctx context.Context, aEnv adapter.EnvConfigAccessor, ceClient cloudevents.Client) adapter.Adapter {}` call, which creates the +new adapter with passed variables via the `EnvConfigAccessor`. The created adapter will be passed the cloudevents client (which is where the events are forwarded to). This is sometimes refered +to as a sink, or `ceClient` in the Knative ecosystem. The return value is a reference to the adapter as defined by the adapter's local struct. In our `sample-source`'s case; ```go @@ -35,31 +36,30 @@ type Adapter struct { logger *zap.Logger interval time.Duration nextID int - sink cloudevents.Client + client cloudevents.Client } ``` -The second required function is the `Start` function implmented as an interface to the adapter struct. -for example: +2. `Start` function implemented as an interface to the adapter struct. ```go func (a *Adapter) Start(stopCh <-chan struct{}) error { ``` -Where `stopCh` is the signal to stop the Adapter. Otherwise the role of the funtion is to process the next -event. In the case of the `sample-source`, it creates an event to forward to the specificed cloudevent sink/client -every X interval, as specified by the loaded EnvConfigAccessor (loaded via the resource yaml). +`stopCh` is the signal to stop the Adapter. Otherwise the role of the function is to process the next +event. In the case of the `sample-source`, it creates an event to forward to the specified cloudevent sink/client +every X interval, as specified by the loaded `EnvConfigAccessor` (loaded via the resource yaml). ```go func (a *Adapter) Start(stopCh <-chan struct{}) error { - a.logger.Info("Starting with: ", - zap.String("Interval: ", a.interval.String())) + a.logger.Infow("Starting heartbeat", zap.String("interval", a.interval.String())) for { select { case <-time.After(a.interval): event := a.newEvent() - a.logger.Info("Sending new event: ", zap.String("event", event.String())) - _, _, err := a.sink.Send(context.Background(), event) - if err != nil { - return err - } + a.logger.Infow("Sending new event", zap.String("event", event.String())) + if result := a.client.Send(context.Background(), event); !cloudevents.IsACK(result) { + a.logger.Infow("failed to send event", zap.String("event", event.String()), zap.Error(result)) + // We got an error but it could be transient, try again next interval. + continue + } case <-stopCh: a.logger.Info("Shutting down...") return nil @@ -67,48 +67,3 @@ func (a *Adapter) Start(stopCh <-chan struct{}) error { } } ``` - -## Reconcile/Create The Receive Adapter -As part of the source reconciliation, we have to create and deploy -(and update if necessary) the underlying receive adapter. The two -client sets used in this process is the `kubeClientSet` for the -Deployment tracking, and the `EventingClientSet` for the event -recording. - -Verify the specified kubernetes resources are valid, and update the `Status` accordingly - -Assemble the ReceiveAdapterArgs -```go -raArgs := resources.ReceiveAdapterArgs{ - EventSource: eventSource, - Image: r.receiveAdapterImage, - Source: src, - Labels: resources.GetLabels(src.Name), - SinkURI: sinkURI, - } -``` -NB The exact arguments may change based on functional requirements -Create the underlying deployment from the arguments provided, matching pod templates, labels, owner references, etc as needed to fill out the deployment -Example: [pkg/reconciler/resources/receive_adapter.go](https://github.com/knative/sample-source/tree/master/pkg/reconciler/resources/receive_adapter.go) - -1. Fetch the existing receive adapter deployment -```go - ra, err := r.KubeClientSet.AppsV1().Deployments(src.Namespace).Get(expected.Name, metav1.GetOptions{}) -``` -2. Otherwise, create the deployment -```go -ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Create(expected) -``` -3. Check if the expected vs existing spec is different, and update the deployment if required -```go -} else if podSpecChanged(ra.Spec.Template.Spec, expected.Spec.Template.Spec) { - ra.Spec.Template.Spec = expected.Spec.Template.Spec - if ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Update(ra); err != nil { - return ra, err - } -``` -4. If updated, record the event -```go - r.Recorder.Eventf(src, corev1.EventTypeNormal, samplesourceDeploymentUpdated, "Deployment updated") - return ra, nil -``` diff --git a/docs/eventing/samples/writing-receive-adapter-source/06-yaml.md b/docs/eventing/samples/writing-receive-adapter-source/06-yaml.md index bba3ce126e0..f55b66594a6 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/06-yaml.md +++ b/docs/eventing/samples/writing-receive-adapter-source/06-yaml.md @@ -10,13 +10,13 @@ type: "docs" Start a minikube cluster. _If you already have a Kubernetes cluster running, you can skip this step. The -cluster must be 1.14+_ +cluster must be 1.15+_ ```sh minikube start ``` -Setup ko to use the minikube docker instance and local registery +Setup `ko` to use the minikube docker instance and local registry ```sh eval $(minikube docker-env) export KO_DOCKER_REPO=ko.local diff --git a/docs/eventing/samples/writing-receive-adapter-source/README.md b/docs/eventing/samples/writing-receive-adapter-source/README.md index d6ccfc487d9..980f47046c7 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/README.md +++ b/docs/eventing/samples/writing-receive-adapter-source/README.md @@ -1,20 +1,37 @@ +## Introduction + This tutorial will walk you though writing a new event source for Knative Eventing using a sample repository and explaining the key concepts used throughout each component. After completing the tutorial, you'll have a basic event source controller as well as receive adapter, which events can be viewed through a basic -event_display Knative Service. +`event_display` Knative Service. Just want to see the code? The reference project is [https://github.com/knative/sample-source](https://github.com/knative/sample-source). +## Other ways + +With the approach in this tutorial, you will create a CRD and a controller for the event source which makes it reusable. + +You can also write your own event source using a [ContainerSource](../../../eventing/sources/README.md#meta-sources) which +is an easy way to turn any dispatcher container into an Event Source. Similarly, another option is using [SinkBinding](../../../eventing/sources/README.md#meta-sources) +which provides a framework for injecting environment variables into any Kubernetes resource which has a `spec.template` that looks like a Pod (aka PodSpecable). + + ## Target Audience The target audience is already familiar with Kubernetes and Go development and wants to develop a new event source, importing their custom events via Knative Eventing into the Knative system. +This tutorial is for adding your event source in Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/). + +If you would like to keep your event source in an independent location, steps are almost the same. +The reference [sample source](https://github.com/knative/sample-source) lives in a separate repository and it is a good starting +point as it has all the tooling for building, installing and publishing. + ## Before You Begin You'll need these tools installed: @@ -25,6 +42,8 @@ You'll need these tools installed: - [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (optional) - [minikube](https://github.com/kubernetes/minikube) (optional) +You're encouraged to clone the [sample source](https://github.com/knative/sample-source) and make changes there. + ## Steps 1. [Separation of Concerns](./01-theory.md) From d784136e932890a32b323018d463d0471536ef5d Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Mon, 18 May 2020 14:39:12 +0300 Subject: [PATCH 2/7] Docs for adding event source to eventing-contrib --- .../01-theory.md | 35 +++++----------- .../07-eventing-contrib.md | 41 +++++++++++++++++++ .../writing-receive-adapter-source/README.md | 14 ++++--- 3 files changed, 61 insertions(+), 29 deletions(-) create mode 100644 docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md diff --git a/docs/eventing/samples/writing-receive-adapter-source/01-theory.md b/docs/eventing/samples/writing-receive-adapter-source/01-theory.md index 49b704740f4..e32e6f545ff 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/01-theory.md +++ b/docs/eventing/samples/writing-receive-adapter-source/01-theory.md @@ -84,33 +84,20 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl } ``` -To have these generated, ensure that the specific source subdirectories has been added to the injection portion of the -[`hack/update-codegen.sh`](https://github.com/knative/eventing-contrib/blob/master/hack/update-codegen.sh) script. - -```patch - -# Sources -+API_DIRS_SOURCES=(camel/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg YourSourceHere/pkg) --API_DIRS_SOURCES=(camel/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg) - -# Knative Injection +Sample source's [`update-codegen.sh`](https://github.com/knative/sample-source/blob/master/hack/update-codegen.sh) have the configuration +to have the required things above generated and injected: +```bash +# Generation +${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ + knative.dev/sample-source/pkg/client knative.dev/sample-source/pkg/apis \ + "samples:v1alpha1" \ + --go-header-file ${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt -chmod +x ${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh +# Injection ${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh "injection" \ -- knative.dev/sample-source/pkg/client knative.dev/sample-source/pkg/apis \ -- "samples:v1alpha1" \ -+ knative.dev/your-source/pkg/client knative.dev/your-source/pkg/apis \ -+ "your-name:v1alpha1" \ + knative.dev/sample-source/pkg/client knative.dev/sample-source/pkg/apis \ + "samples:v1alpha1" \ --go-header-file ${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt - -``` -and -```patch - -i knative.dev/eventing-contrib/github/pkg/apis \ -- -i knative.dev/eventing-contrib/gitlab/pkg/apis -+ -i knative.dev/eventing-contrib/gitlab/pkg/apis \ -+ -i knative.dev/eventing-contrib/YourSourceHere/pkg/apis - ``` File Layout & Hierarchy: diff --git a/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md b/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md new file mode 100644 index 00000000000..715fff706bc --- /dev/null +++ b/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md @@ -0,0 +1,41 @@ +--- +title: "Adding the event source to eventing-contrib" +linkTitle: "Adding to eventing-contrib" +weight: 10 +type: "docs" +--- + +If you would like to contribute Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/), as a starting point you can +have a look at different sources there, such as +[`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), +[`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and +[`AWSSQSSource`](https://github.com/knative/eventing-contrib/tree/master/awssqs). + +To generate and inject `clientset`, `cache`, `informers`, and `listers`, ensure that the specific source subdirectories has been added to the injection portion of the +[`hack/update-codegen.sh`](https://github.com/knative/eventing-contrib/blob/master/hack/update-codegen.sh) script. + +```patch + +# Sources ++API_DIRS_SOURCES=(camel/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg YourSourceHere/pkg) +-API_DIRS_SOURCES=(camel/source/pkg awssqs/pkg couchdb/source/pkg prometheus/pkg) + +# Knative Injection + +chmod +x ${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh +${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh "injection" \ +- knative.dev/sample-source/pkg/client knative.dev/sample-source/pkg/apis \ +- "samples:v1alpha1" \ ++ knative.dev/your-source/pkg/client knative.dev/your-source/pkg/apis \ ++ "your-name:v1alpha1" \ + --go-header-file ${REPO_ROOT}/hack/boilerplate/boilerplate.go.txt + +``` +and +```patch + -i knative.dev/eventing-contrib/github/pkg/apis \ +- -i knative.dev/eventing-contrib/gitlab/pkg/apis ++ -i knative.dev/eventing-contrib/gitlab/pkg/apis \ ++ -i knative.dev/eventing-contrib/YourSourceHere/pkg/apis + +``` diff --git a/docs/eventing/samples/writing-receive-adapter-source/README.md b/docs/eventing/samples/writing-receive-adapter-source/README.md index 980f47046c7..b8ff38d2096 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/README.md +++ b/docs/eventing/samples/writing-receive-adapter-source/README.md @@ -11,6 +11,11 @@ well as receive adapter, which events can be viewed through a basic Just want to see the code? The reference project is [https://github.com/knative/sample-source](https://github.com/knative/sample-source). +A variety of event sources are available in Knative [`eventing-contrib](https://github.com/knative/eventing-contrib/) such as +[`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), +[`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and +[`AWSSQSSource`](https://github.com/knative/eventing-contrib/tree/master/awssqs) that can be used as a reference. + ## Other ways With the approach in this tutorial, you will create a CRD and a controller for the event source which makes it reusable. @@ -26,11 +31,9 @@ The target audience is already familiar with Kubernetes and Go development and wants to develop a new event source, importing their custom events via Knative Eventing into the Knative system. -This tutorial is for adding your event source in Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/). - -If you would like to keep your event source in an independent location, steps are almost the same. -The reference [sample source](https://github.com/knative/sample-source) lives in a separate repository and it is a good starting -point as it has all the tooling for building, installing and publishing. +This tutorial is for having your event source in an independent repository. +If you would like to contribute Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/) and have your source +there, there are some instructions at [Adding the event source to `eventing-contrib`](./07-eventing-contrib.md). ## Before You Begin @@ -52,3 +55,4 @@ You're encouraged to clone the [sample source](https://github.com/knative/sample 4. [Reconciler](./04-reconciler.md) 5. [Receive Adapter](./05-receive-adapter.md) 6. [Example YAML](./06-yaml.md) +7. [Adding the event source to `eventing-contrib`](./07-eventing-contrib.md) From b0b56ed5f58c5e8ebd1146d03e7c961ac2dd4166 Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Mon, 18 May 2020 14:49:06 +0300 Subject: [PATCH 3/7] Get rid of some whitespace --- .../writing-receive-adapter-source/01-theory.md | 2 +- .../04-reconciler.md | 16 ++++++++-------- .../05-receive-adapter.md | 4 ++-- .../07-eventing-contrib.md | 10 +++++----- .../writing-receive-adapter-source/README.md | 12 ++++++------ 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/eventing/samples/writing-receive-adapter-source/01-theory.md b/docs/eventing/samples/writing-receive-adapter-source/01-theory.md index e32e6f545ff..bd43bc59e50 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/01-theory.md +++ b/docs/eventing/samples/writing-receive-adapter-source/01-theory.md @@ -86,7 +86,7 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl Sample source's [`update-codegen.sh`](https://github.com/knative/sample-source/blob/master/hack/update-codegen.sh) have the configuration to have the required things above generated and injected: -```bash +```bash # Generation ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ knative.dev/sample-source/pkg/client knative.dev/sample-source/pkg/apis \ diff --git a/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md b/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md index ff9f69de155..572d6c7cfea 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md +++ b/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md @@ -14,16 +14,16 @@ src.Status.ObservedGeneration = src.Generation ``` 2. Create/reconcile the Receive Adapter (detailed below) 3. If successful, update the `Status` and `MarkDeployed` -```go +```go src.Status.PropagateDeploymentAvailability(ra) ``` 4. Create/reconcile the `SinkBinding` for the Receive Adapter targeting the `Sink` (detailed below) 5. MarkSink with the result -```go +```go src.Status.MarkSink(sb.Status.SinkURI) ``` 6. Return a new reconciler event stating that the process is done -```go +```go return pkgreconciler.NewEvent(corev1.EventTypeNormal, "SampleSourceReconciled", "SampleSource reconciled: \"%s/%s\"", namespace, name) ``` @@ -50,7 +50,7 @@ Example: [pkg/reconciler/sample/resources/receive_adapter.go](https://github.com 1. Fetch the existing receive adapter deployment ```go namespace := owner.GetObjectMeta().GetNamespace() -ra, err := r.KubeClientSet.AppsV1().Deployments(namespace).Get(expected.Name, metav1.GetOptions{}) +ra, err := r.KubeClientSet.AppsV1().Deployments(namespace).Get(expected.Name, metav1.GetOptions{}) ``` 2. Otherwise, create the deployment ```go @@ -84,17 +84,17 @@ tracker.Reference{ } ``` 2. Fetch the existing `SinkBinding` -```go +```go namespace := owner.GetObjectMeta().GetNamespace() sb, err := r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Get(expected.Name, metav1.GetOptions{}) ``` 2. If it doesn't exist, create it -```go +```go sb, err = r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Create(expected) ``` 3. Check if the expected vs existing spec is different, and update the `SinkBinding` if required -```go +```go else if r.specChanged(sb.Spec, expected.Spec) { sb.Spec = expected.Spec if sb, err = r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Update(sb); err != nil { @@ -104,4 +104,4 @@ else if r.specChanged(sb.Spec, expected.Spec) { 4. If updated, record the event ```go return pkgreconciler.NewEvent(corev1.EventTypeNormal, "SinkBindingUpdated", "updated SinkBinding: \"%s/%s\"", namespace, name) -``` +``` diff --git a/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md b/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md index c0bae379f76..840e584d073 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md +++ b/docs/eventing/samples/writing-receive-adapter-source/05-receive-adapter.md @@ -25,8 +25,8 @@ func main() { ## Defining NewAdapter implementation and Start function The adapter's `pkg` implementation consists of two main functions; -1. A `NewAdapter(ctx context.Context, aEnv adapter.EnvConfigAccessor, ceClient cloudevents.Client) adapter.Adapter {}` call, which creates the -new adapter with passed variables via the `EnvConfigAccessor`. The created adapter will be passed the cloudevents client (which is where the events are forwarded to). This is sometimes refered +1. A `NewAdapter(ctx context.Context, aEnv adapter.EnvConfigAccessor, ceClient cloudevents.Client) adapter.Adapter {}` call, which creates the +new adapter with passed variables via the `EnvConfigAccessor`. The created adapter will be passed the cloudevents client (which is where the events are forwarded to). This is sometimes referred to as a sink, or `ceClient` in the Knative ecosystem. The return value is a reference to the adapter as defined by the adapter's local struct. In our `sample-source`'s case; diff --git a/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md b/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md index 715fff706bc..5a3ab28bdea 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md +++ b/docs/eventing/samples/writing-receive-adapter-source/07-eventing-contrib.md @@ -5,13 +5,13 @@ weight: 10 type: "docs" --- -If you would like to contribute Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/), as a starting point you can -have a look at different sources there, such as -[`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), -[`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and +If you would like to contribute Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/), as a starting point you can +have a look at different sources there, such as +[`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), +[`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and [`AWSSQSSource`](https://github.com/knative/eventing-contrib/tree/master/awssqs). -To generate and inject `clientset`, `cache`, `informers`, and `listers`, ensure that the specific source subdirectories has been added to the injection portion of the +To generate and inject `clientset`, `cache`, `informers`, and `listers`, ensure that the specific source subdirectories has been added to the injection portion of the [`hack/update-codegen.sh`](https://github.com/knative/eventing-contrib/blob/master/hack/update-codegen.sh) script. ```patch diff --git a/docs/eventing/samples/writing-receive-adapter-source/README.md b/docs/eventing/samples/writing-receive-adapter-source/README.md index b8ff38d2096..9670c3cf22b 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/README.md +++ b/docs/eventing/samples/writing-receive-adapter-source/README.md @@ -11,16 +11,16 @@ well as receive adapter, which events can be viewed through a basic Just want to see the code? The reference project is [https://github.com/knative/sample-source](https://github.com/knative/sample-source). -A variety of event sources are available in Knative [`eventing-contrib](https://github.com/knative/eventing-contrib/) such as -[`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), -[`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and -[`AWSSQSSource`](https://github.com/knative/eventing-contrib/tree/master/awssqs) that can be used as a reference. +A variety of event sources are available in Knative [`eventing-contrib](https://github.com/knative/eventing-contrib/) such as +[`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), +[`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and +[`AWSSQSSource`](https://github.com/knative/eventing-contrib/tree/master/awssqs) that can be used as a reference. ## Other ways With the approach in this tutorial, you will create a CRD and a controller for the event source which makes it reusable. -You can also write your own event source using a [ContainerSource](../../../eventing/sources/README.md#meta-sources) which +You can also write your own event source using a [ContainerSource](../../../eventing/sources/README.md#meta-sources) which is an easy way to turn any dispatcher container into an Event Source. Similarly, another option is using [SinkBinding](../../../eventing/sources/README.md#meta-sources) which provides a framework for injecting environment variables into any Kubernetes resource which has a `spec.template` that looks like a Pod (aka PodSpecable). @@ -31,7 +31,7 @@ The target audience is already familiar with Kubernetes and Go development and wants to develop a new event source, importing their custom events via Knative Eventing into the Knative system. -This tutorial is for having your event source in an independent repository. +This tutorial is for having your event source in an independent repository. If you would like to contribute Knative's [`eventing-contrib`](https://github.com/knative/eventing-contrib/) and have your source there, there are some instructions at [Adding the event source to `eventing-contrib`](./07-eventing-contrib.md). From 3699dc0c74eca246edc7193cfdc6ab83fca77819 Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Wed, 20 May 2020 13:48:38 +0300 Subject: [PATCH 4/7] Fix broken markdown --- docs/eventing/samples/writing-receive-adapter-source/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/eventing/samples/writing-receive-adapter-source/README.md b/docs/eventing/samples/writing-receive-adapter-source/README.md index 9670c3cf22b..469472b0912 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/README.md +++ b/docs/eventing/samples/writing-receive-adapter-source/README.md @@ -11,7 +11,7 @@ well as receive adapter, which events can be viewed through a basic Just want to see the code? The reference project is [https://github.com/knative/sample-source](https://github.com/knative/sample-source). -A variety of event sources are available in Knative [`eventing-contrib](https://github.com/knative/eventing-contrib/) such as +A variety of event sources are available in Knative [`eventing-contrib`](https://github.com/knative/eventing-contrib/) such as [`KafkaSource`](https://github.com/knative/eventing-contrib/tree/master/kafka/source), [`GithubSource`](https://github.com/knative/eventing-contrib/tree/master/github) and [`AWSSQSSource`](https://github.com/knative/eventing-contrib/tree/master/awssqs) that can be used as a reference. From b75ace155f2a8fa6e112f2e6a0ebc8343422a934 Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Wed, 27 May 2020 11:16:55 +0300 Subject: [PATCH 5/7] Update docs/eventing/samples/writing-receive-adapter-source/03-controller.md Co-authored-by: Evan Anderson --- .../samples/writing-receive-adapter-source/03-controller.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/eventing/samples/writing-receive-adapter-source/03-controller.md b/docs/eventing/samples/writing-receive-adapter-source/03-controller.md index 6e309807501..7c41ac44a1b 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/03-controller.md +++ b/docs/eventing/samples/writing-receive-adapter-source/03-controller.md @@ -29,8 +29,8 @@ func NewController( ) *controller.Impl { // ... deploymentInformer := deploymentinformer.Get(ctx) - sinkBindingInformer := sinkbindinginformer.Get(ctx) - sampleSourceInformer := samplesourceinformer.Get(ctx) + sinkBindingInformer := sinkbindinginformer.Get(ctx) + sampleSourceInformer := samplesourceinformer.Get(ctx) r := &Reconciler{ dr: &reconciler.DeploymentReconciler{KubeClientSet: kubeclient.Get(ctx)}, @@ -63,4 +63,3 @@ Controller for the `SampleSource` uses `Deployment` and `SinkBinding` resources Handler: controller.HandleAll(impl.EnqueueControllerOf), }) ``` - From b888c3570aa1dacc39565f4fc07469dc73b5e7ff Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Wed, 27 May 2020 11:17:07 +0300 Subject: [PATCH 6/7] Update docs/eventing/samples/writing-receive-adapter-source/03-controller.md Co-authored-by: Evan Anderson --- .../writing-receive-adapter-source/03-controller.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/eventing/samples/writing-receive-adapter-source/03-controller.md b/docs/eventing/samples/writing-receive-adapter-source/03-controller.md index 7c41ac44a1b..1d627b84a46 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/03-controller.md +++ b/docs/eventing/samples/writing-receive-adapter-source/03-controller.md @@ -33,11 +33,11 @@ func NewController( sampleSourceInformer := samplesourceinformer.Get(ctx) r := &Reconciler{ - dr: &reconciler.DeploymentReconciler{KubeClientSet: kubeclient.Get(ctx)}, - sbr: &reconciler.SinkBindingReconciler{EventingClientSet: eventingclient.Get(ctx)}, - // Config accessor takes care of tracing/config/logging config propagation to the receive adapter - configAccessor: reconcilersource.WatchConfigurations(ctx, "sample-source", cmw), - } + dr: &reconciler.DeploymentReconciler{KubeClientSet: kubeclient.Get(ctx)}, + sbr: &reconciler.SinkBindingReconciler{EventingClientSet: eventingclient.Get(ctx)}, + // Config accessor takes care of tracing/config/logging config propagation to the receive adapter + configAccessor: reconcilersource.WatchConfigurations(ctx, "sample-source", cmw), +} ``` The base reconciler is imported from the knative.dev/pkg dependency: ```go From 76851e27e2ddd0c218e9be045e0ba3954db81071 Mon Sep 17 00:00:00 2001 From: Ali Ok Date: Wed, 27 May 2020 11:18:36 +0300 Subject: [PATCH 7/7] Remove an empty line --- .../samples/writing-receive-adapter-source/04-reconciler.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md b/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md index 572d6c7cfea..c722fa350b1 100644 --- a/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md +++ b/docs/eventing/samples/writing-receive-adapter-source/04-reconciler.md @@ -91,7 +91,6 @@ sb, err := r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Get(exp 2. If it doesn't exist, create it ```go sb, err = r.EventingClientSet.SourcesV1alpha2().SinkBindings(namespace).Create(expected) - ``` 3. Check if the expected vs existing spec is different, and update the `SinkBinding` if required ```go