Merge branch 'master' into HandleResourceGroupMissing

.devcontainer/docker-compose.yml

@@ -5,8 +5,6 @@ services:
       context: ../
       dockerfile: .devcontainer/Dockerfile
     network_mode: "host"
-    ports: 
-    - 5002:5001
     volumes:
       # Update this to wherever you want VS Code to mount the folder of your project
       - ..:/workspace

Makefile

@@ -16,12 +16,12 @@ all: manager
 
 # Run tests
 test: generate fmt vet manifests
-	TEST_USE_EXISTING_CLUSTER=false go test -v -coverprofile=coverage.txt -covermode count ./api/... ./controllers/... ./pkg/resourcemanager/eventhubs/...  ./pkg/resourcemanager/resourcegroups/... 2>&1 | tee testlogs.txt
+	TEST_USE_EXISTING_CLUSTER=false go test -v -coverprofile=coverage.txt -covermode count ./api/... ./controllers/... ./pkg/resourcemanager/eventhubs/...  ./pkg/resourcemanager/resourcegroups/...  ./pkg/resourcemanager/storages/... 2>&1 | tee testlogs.txt
 	go-junit-report < testlogs.txt  > report.xml
 	go tool cover -html=coverage.txt -o cover.html
 # Run tests with existing cluster
 test-existing: generate fmt vet manifests
-	TEST_USE_EXISTING_CLUSTER=true go test -v -coverprofile=coverage-existing.txt -covermode count ./api/... ./controllers/... ./pkg/resourcemanager/eventhubs/...  ./pkg/resourcemanager/resourcegroups/... 2>&1 | tee testlogs-existing.txt
+	TEST_USE_EXISTING_CLUSTER=true go test -v -coverprofile=coverage-existing.txt -covermode count ./api/... ./controllers/... ./pkg/resourcemanager/eventhubs/...  ./pkg/resourcemanager/resourcegroups/...  ./pkg/resourcemanager/storages/... 2>&1 | tee testlogs-existing.txt
 	go-junit-report < testlogs-existing.txt  > report-existing.xml
 	go tool cover -html=coverage-existing.txt -o cover-existing.html
 

PROJECT

@@ -2,13 +2,13 @@ version: "2"
 domain: microsoft.com
 repo: github.com/Azure/azure-service-operator
 resources:
-- group: service
+- group: azure
   version: v1
   kind: Storage
-- group: service
+- group: azure
   version: v1
   kind: CosmosDB
-- group: service
+- group: azure
   version: v1
   kind: RedisCache
 - group: azure

api/v1/consumergroup_types.go

@@ -27,9 +27,9 @@ import (
 type ConsumerGroupSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
-	ResourceGroupName string `json:"resourcegroup,omitempty"`
+	ResourceGroupName string `json:"resourceGroup,omitempty"`
 	NamespaceName     string `json:"namespace,omitempty"`
-	EventhubName      string `json:"eventhub,omitempty"`
+	EventhubName      string `json:"eventHub,omitempty"`
 }
 
 // ConsumerGroupStatus defines the observed state of ConsumerGroup

api/v1/eventhub_types.go

@@ -29,7 +29,7 @@ type EventhubNamespaceResource struct {
 	Location          string                      `json:"location"`
 	Sku               EventhubNamespaceSku        `json:"sku,omitempty"`
 	Properties        EventhubNamespaceProperties `json:"properties,omitempty"`
-	ResourceGroupName string                      `json:"resourcegroup,omitempty"`
+	ResourceGroupName string                      `json:"resourceGroup,omitempty"`
 }
 
 // EventhubSpec defines the desired state of Eventhub
@@ -39,10 +39,10 @@ type EventhubSpec struct {
 	Location          string                    `json:"location"`
 	Namespace         string                    `json:"namespace,omitempty"`
 	Properties        EventhubProperties        `json:"properties,omitempty"`
-	ResourceGroup     string                    `json:"resourcegroup,omitempty"`
-	AuthorizationRule EventhubAuthorizationRule `json:"authorizationrule,omitempty"`
+	ResourceGroup     string                    `json:"resourceGroup,omitempty"`
+	AuthorizationRule EventhubAuthorizationRule `json:"authorizationRule,omitempty"`
 	// SecretName - Used to specify the name of the secret. Defaults to Event Hub name if omitted.
-	SecretName string `json:"secretname,omitempty"`
+	SecretName string `json:"secretName,omitempty"`
 }
 
 // EventhubStatus defines the observed state of Eventhub
@@ -61,16 +61,58 @@ type EventhubAuthorizationRule struct {
 	Rights []string `json:"rights,omitempty"`
 }
 
+type StorageAccount struct {
+	// ResourceGroup - Name of the storage account resource group
+	// +kubebuilder:validation:Pattern=^[-\w\._\(\)]+$
+	ResourceGroup string `json:"resourceGroup,omitempty"`
+	// AccountName - Name of the storage account
+	// +kubebuilder:validation:MaxLength=24
+	// +kubebuilder:validation:MinLength=3
+	// +kubebuilder:validation:Pattern=^[a-z0-9]+$
+	AccountName string `json:"accountName,omitempty"`
+}
+
+//Destination for capture (blob storage etc)
+type Destination struct {
+	// ArchiveNameFormat - Blob naming convention for archive, e.g. {Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}. Here all the parameters (Namespace,EventHub .. etc) are mandatory irrespective of order
+	ArchiveNameFormat string `json:"archiveNameFormat,omitempty"`
+	// BlobContainer - Blob container Name
+	BlobContainer string `json:"blobContainer,omitempty"`
+	// Name - Name for capture destination
+	// +kubebuilder:validation:Enum=EventHubArchive.AzureBlockBlob;EventHubArchive.AzureDataLake
+	Name string `json:"name,omitempty"`
+	// StorageAccount - Details of the storage account
+	StorageAccount StorageAccount `json:"storageAccount,omitempty"`
+}
+
+//CaptureDescription defines the properties required for eventhub capture
+type CaptureDescription struct {
+	// Destination - Resource id of the storage account to be used to create the blobs
+	Destination Destination `json:"destination,omitempty"`
+	// Enabled - indicates whether capture is enabled
+	Enabled bool `json:"enabled"`
+	// SizeLimitInBytes - The size window defines the amount of data built up in your Event Hub before an capture operation
+	// +kubebuilder:validation:Maximum=524288000
+	// +kubebuilder:validation:Minimum=10485760
+	SizeLimitInBytes int32 `json:"sizeLimitInBytes,omitempty"`
+	// IntervalInSeconds - The time window allows you to set the frequency with which the capture to Azure Blobs will happen
+	// +kubebuilder:validation:Maximum=900
+	// +kubebuilder:validation:Minimum=60
+	IntervalInSeconds int32 `json:"intervalInSeconds,omitempty"`
+}
+
 //EventhubProperties defines the namespace properties
 type EventhubProperties struct {
 	// +kubebuilder:validation:Maximum=7
 	// +kubebuilder:validation:Minimum=1
 	// MessageRetentionInDays - Number of days to retain the events for this Event Hub, value should be 1 to 7 days
-	MessageRetentionInDays int32 `json:"messageretentionindays,omitempty"`
+	MessageRetentionInDays int32 `json:"messageRetentionInDays,omitempty"`
 	// +kubebuilder:validation:Maximum=32
-	// +kubebuilder:validation:Minimum=1
-	// PartitionCount - Number of partitions created for the Event Hub, allowed values are from 1 to 32 partitions.
-	PartitionCount int32 `json:"partitioncount,omitempty"`
+	// +kubebuilder:validation:Minimum=2
+	// PartitionCount - Number of partitions created for the Event Hub, allowed values are from 2 to 32 partitions.
+	PartitionCount int32 `json:"partitionCount,omitempty"`
+	// CaptureDescription - Details specifying EventHub capture to persistent storage
+	CaptureDescription CaptureDescription `json:"captureDescription,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1/eventhub_types_test.go

@@ -41,32 +41,74 @@ var _ = Describe("Eventhub", func() {
 		// Add any teardown steps that needs to be executed after each test
 	})
 
+	createEventHub := func(captureDescription CaptureDescription) *Eventhub {
+		created := &Eventhub{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "foo",
+				Namespace: "default",
+			},
+			Spec: EventhubSpec{
+				Location:      resourcegroupLocation,
+				Namespace:     "foo-eventhub-ns-name",
+				ResourceGroup: "foo-resource-group",
+				Properties: EventhubProperties{
+					MessageRetentionInDays: 7,
+					PartitionCount:         2,
+					CaptureDescription:     captureDescription,
+				},
+			},
+		}
+		return created
+	}
+
 	// Add Tests for OpenAPI validation (or additonal CRD features) specified in
 	// your API definition.
 	// Avoid adding tests for vanilla CRUD operations because they would
 	// test Kubernetes API server, which isn't the goal here.
 	Context("Create API", func() {
 
-		It("should create an object successfully", func() {
+		It("should create an object successfully (without Capture)", func() {
+			key = types.NamespacedName{
+				Name:      "foo",
+				Namespace: "default",
+			}
+
+			created = createEventHub(CaptureDescription{})
+
+			By("creating an API obj")
+			Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
+
+			fetched = &Eventhub{}
+			Expect(k8sClient.Get(context.TODO(), key, fetched)).To(Succeed())
+			Expect(fetched).To(Equal(created))
 
+			By("deleting the created object")
+			Expect(k8sClient.Delete(context.TODO(), created)).To(Succeed())
+			Expect(k8sClient.Get(context.TODO(), key, created)).ToNot(Succeed())
+		})
+
+		It("should create an object successfully (with Capture)", func() {
 			key = types.NamespacedName{
 				Name:      "foo",
 				Namespace: "default",
 			}
-			created = &Eventhub{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "foo",
-					Namespace: "default",
-				},
-				Spec: EventhubSpec{
-					Location:      "westus",
-					Namespace:     "fooeventhubNamespaceName",
-					ResourceGroup: "fooresourceGroupName",
-					Properties: EventhubProperties{
-						MessageRetentionInDays: 7,
-						PartitionCount:         1,
+
+			capture := CaptureDescription{
+				Destination: Destination{
+					ArchiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}",
+					BlobContainer:     "foo-blob-container",
+					Name:              "EventHubArchive.AzureBlockBlob",
+					StorageAccount: StorageAccount{
+						ResourceGroup: "foo-resource-group",
+						AccountName:   "fooaccountname",
 					},
-				}}
+				},
+				Enabled:           true,
+				SizeLimitInBytes:  524288000,
+				IntervalInSeconds: 90,
+			}
+
+			created = createEventHub(capture)
 
 			By("creating an API obj")
 			Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())

api/v1/eventhubnamespace_types.go

@@ -30,7 +30,7 @@ type EventhubNamespaceSpec struct {
 	Location      string                      `json:"location"`
 	Sku           EventhubNamespaceSku        `json:"sku,omitempty"`
 	Properties    EventhubNamespaceProperties `json:"properties,omitempty"`
-	ResourceGroup string                      `json:"resourcegroup,omitempty"`
+	ResourceGroup string                      `json:"resourceGroup,omitempty"`
 }
 
 // EventhubNamespaceStatus defines the observed state of EventhubNamespace
@@ -70,9 +70,9 @@ type EventhubNamespaceSku struct {
 
 //EventhubNamespaceProperties defines the namespace properties
 type EventhubNamespaceProperties struct {
-	IsAutoInflateEnabled   bool  `json:"isautoinflateenabled,omitempty"`
-	MaximumThroughputUnits int32 `json:"maximumthroughputunits,omitempty"`
-	KafkaEnabled           bool  `json:"kafkaenabled,omitempty"`
+	IsAutoInflateEnabled   bool  `json:"isAutoInflateEnabled,omitempty"`
+	MaximumThroughputUnits int32 `json:"maximumThroughputUnits,omitempty"`
+	KafkaEnabled           bool  `json:"kafkaEnabled,omitempty"`
 }
 
 func init() {

api/v1/eventhubnamespace_types_test.go

@@ -59,7 +59,7 @@ var _ = Describe("EventhubNamespace", func() {
 					Namespace: "default",
 				},
 				Spec: EventhubNamespaceSpec{
-					Location:      "westus",
+					Location:      resourcegroupLocation,
 					ResourceGroup: "bar",
 				}}
 

api/v1/keyvault_types.go

@@ -60,3 +60,7 @@ func init() {
 func (keyVault *KeyVault) IsSubmitted() bool {
 	return keyVault.Status.Provisioning || keyVault.Status.Provisioned
 }
+
+func (keyVault *KeyVault) IsProvisioned() bool {
+	return keyVault.Status.Provisioned
+}

api/v1/resourcegroup_types_test.go

@@ -59,7 +59,7 @@ var _ = Describe("ResourceGroup", func() {
 					Namespace: "default",
 				},
 				Spec: ResourceGroupSpec{
-					Location: "westus",
+					Location: resourcegroupLocation,
 				}}
 
 			By("creating an API obj")

api/v1/suite_test.go

@@ -22,6 +22,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	resourcemanagerconfig "github.com/Azure/azure-service-operator/pkg/resourcemanager/config"
+
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -36,6 +38,7 @@ import (
 var cfg *rest.Config
 var k8sClient client.Client
 var testEnv *envtest.Environment
+var resourcegroupLocation string
 
 func TestAPIs(t *testing.T) {
 	t.Parallel()
@@ -54,6 +57,9 @@ var _ = BeforeSuite(func(done Done) {
 		CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")},
 	}
 
+	resourcemanagerconfig.ParseEnvironment()
+	resourcegroupLocation = resourcemanagerconfig.DefaultLocation()
+
 	err := SchemeBuilder.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 

api/v1/zz_generated.deepcopy.go

@@ -23,6 +23,22 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CaptureDescription) DeepCopyInto(out *CaptureDescription) {
+	*out = *in
+	out.Destination = in.Destination
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CaptureDescription.
+func (in *CaptureDescription) DeepCopy() *CaptureDescription {
+	if in == nil {
+		return nil
+	}
+	out := new(CaptureDescription)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ConsumerGroup) DeepCopyInto(out *ConsumerGroup) {
 	*out = *in
@@ -254,6 +270,22 @@ func (in *CosmosDBStatus) DeepCopy() *CosmosDBStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Destination) DeepCopyInto(out *Destination) {
+	*out = *in
+	out.StorageAccount = in.StorageAccount
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Destination.
+func (in *Destination) DeepCopy() *Destination {
+	if in == nil {
+		return nil
+	}
+	out := new(Destination)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Eventhub) DeepCopyInto(out *Eventhub) {
 	*out = *in
@@ -474,6 +506,7 @@ func (in *EventhubNamespaceStatus) DeepCopy() *EventhubNamespaceStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *EventhubProperties) DeepCopyInto(out *EventhubProperties) {
 	*out = *in
+	out.CaptureDescription = in.CaptureDescription
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventhubProperties.
@@ -883,6 +916,21 @@ func (in *Storage) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StorageAccount) DeepCopyInto(out *StorageAccount) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageAccount.
+func (in *StorageAccount) DeepCopy() *StorageAccount {
+	if in == nil {
+		return nil
+	}
+	out := new(StorageAccount)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StorageAdditionalResources) DeepCopyInto(out *StorageAdditionalResources) {
 	*out = *in

config/crd/bases/azure.microsoft.com_consumergroups.yaml

@@ -399,11 +399,11 @@ spec:
         spec:
           description: ConsumerGroupSpec defines the desired state of ConsumerGroup
           properties:
-            eventhub:
+            eventHub:
               type: string
             namespace:
               type: string
-            resourcegroup:
+            resourceGroup:
               description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
                 Important: Run "make" to regenerate code after modifying this file'
               type: string