Ingestion Controller (#53)

* ingestion spec acc to v3

* task creation

* supprt native batch

* fix router url

* revert license change

* revert go mod change

* fix main

* fix: made some changes as per review comments

* fix: removed unused package from ingestion reconciler

* rebase 1

* add example

* add review

* update dockerfile

---------

Co-authored-by: avtarOPS <avtarsingh12015@gmail.com>

Dockerfile

@@ -15,6 +15,7 @@ RUN go mod download
 COPY main.go main.go
 COPY apis/ apis/
 COPY controllers/ controllers/
+COPY pkg/ pkg/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

PROJECT

@@ -18,4 +18,13 @@ resources:
   kind: Druid
   path: github.com/datainfrahq/druid-operator/apis/druid/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: apache.org
+  group: druid
+  kind: DruidIngestion
+  path: github.com/datainfrahq/druid-operator/apis/druid/v1alpha1
+  version: v1alpha1
 version: "3"

apis/druid/v1alpha1/druidingestion_types.go

@@ -0,0 +1,98 @@
+/*
+DataInfra 2023 Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type DruidIngestionMethod string
+
+const (
+	Kafka                    DruidIngestionMethod = "kafka"
+	Kinesis                  DruidIngestionMethod = "kinesis"
+	NativeBatchIndexParallel DruidIngestionMethod = "native-batch"
+	QueryControllerSQL       DruidIngestionMethod = "sql"
+	HadoopIndexHadoop        DruidIngestionMethod = "index-hadoop"
+)
+
+type DruidIngestionSpec struct {
+	// +optional
+	Suspend bool `json:"suspend"`
+	// +required
+	DruidClusterName string `json:"druidCluster"`
+	// +required
+	Ingestion IngestionSpec `json:"ingestion"`
+	// +optional
+	Auth Auth `json:"auth"`
+}
+
+type IngestionSpec struct {
+	// +required
+	Type DruidIngestionMethod `json:"type"`
+	// +required
+	Spec string `json:"spec,omitempty"`
+}
+
+type DruidIngestionStatus struct {
+	TaskId               string             `json:"taskId"`
+	Type                 string             `json:"type,omitempty"`
+	Status               v1.ConditionStatus `json:"status,omitempty"`
+	Reason               string             `json:"reason,omitempty"`
+	Message              string             `json:"message,omitempty"`
+	LastUpdateTime       metav1.Time        `json:"lastUpdateTime,omitempty"`
+	CurrentIngestionSpec string             `json:"currentIngestionSpec.json"`
+}
+
+type AuthType string
+
+const (
+	BasicAuth AuthType = "basic-auth"
+)
+
+type Auth struct {
+	// +required
+	Type AuthType `json:"type"`
+	// +required
+	SecretRef v1.SecretReference `json:"secretRef"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.ingestionSpec.type"
+// Ingestion is the Schema for the Ingestion API
+type DruidIngestion struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   DruidIngestionSpec   `json:"spec"`
+	Status DruidIngestionStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// IngestionList contains a list of Ingestion
+type DruidIngestionList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []DruidIngestion `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&DruidIngestion{}, &DruidIngestionList{})
+}

chart/templates/crds/druid.apache.org_druidingestions.yaml

@@ -0,0 +1,110 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.11.2
+  creationTimestamp: null
+  name: druidingestions.druid.apache.org
+spec:
+  group: druid.apache.org
+  names:
+    kind: DruidIngestion
+    listKind: DruidIngestionList
+    plural: druidingestions
+    singular: druidingestion
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - jsonPath: .spec.ingestionSpec.type
+      name: Type
+      type: string
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Ingestion is the Schema for the Ingestion API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              auth:
+                properties:
+                  secretRef:
+                    description: SecretReference represents a Secret Reference. It
+                      has enough information to retrieve secret in any namespace
+                    properties:
+                      name:
+                        description: name is unique within a namespace to reference
+                          a secret resource.
+                        type: string
+                      namespace:
+                        description: namespace defines the space within which the
+                          secret name must be unique.
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
+                  type:
+                    type: string
+                required:
+                - secretRef
+                - type
+                type: object
+              druidCluster:
+                type: string
+              ingestion:
+                properties:
+                  spec:
+                    type: string
+                  type:
+                    type: string
+                required:
+                - type
+                type: object
+              suspend:
+                type: boolean
+            required:
+            - druidCluster
+            - ingestion
+            type: object
+          status:
+            properties:
+              currentIngestionSpec.json:
+                type: string
+              lastUpdateTime:
+                format: date-time
+                type: string
+              message:
+                type: string
+              reason:
+                type: string
+              status:
+                type: string
+              taskId:
+                type: string
+              type:
+                type: string
+            required:
+            - currentIngestionSpec.json
+            - taskId
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}