diff --git a/api/v1alpha1/project_quota_types.go b/api/v1alpha1/project_quota_types.go index fecac57cc..092155d3e 100644 --- a/api/v1alpha1/project_quota_types.go +++ b/api/v1alpha1/project_quota_types.go @@ -7,34 +7,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// ResourceQuota holds the quota for a single resource with per-AZ breakdown. -// Maps to liquid.ResourceQuotaRequest from the LIQUID API. -// See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ResourceQuotaRequest -type ResourceQuota struct { - // Quota is the total quota across all AZs (for compatibility). - // Corresponds to liquid.ResourceQuotaRequest.Quota. - // +kubebuilder:validation:Required - Quota int64 `json:"quota"` - - // PerAZ holds the per-availability-zone quota breakdown. - // Key: availability zone name, Value: quota for that AZ. - // Only populated for AZSeparatedTopology resources. - // Corresponds to liquid.ResourceQuotaRequest.PerAZ[az].Quota. - // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#AZResourceQuotaRequest - // +kubebuilder:validation:Optional - PerAZ map[string]int64 `json:"perAZ,omitempty"` -} - -// ResourceQuotaUsage holds per-AZ PAYG usage for a single resource. -type ResourceQuotaUsage struct { - // PerAZ holds per-availability-zone PAYG usage values. - // Key: availability zone name, Value: PAYG usage in that AZ. - // +kubebuilder:validation:Optional - PerAZ map[string]int64 `json:"perAZ,omitempty"` -} - // ProjectQuotaSpec defines the desired state of ProjectQuota. // Populated from PUT /v1/projects/:uuid/quota payloads (liquid.ServiceQuotaRequest). +// Each ProjectQuota CRD represents quota for ONE project in ONE availability zone. // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ServiceQuotaRequest type ProjectQuotaSpec struct { // ProjectID of the OpenStack project this quota belongs to. @@ -57,12 +32,18 @@ type ProjectQuotaSpec struct { // +kubebuilder:validation:Optional DomainName string `json:"domainName,omitempty"` - // Quota maps LIQUID resource names to their per-AZ quota. + // AvailabilityZone is the AZ this quota CRD covers (e.g. "qa-de-1a"). + // In a multi-cluster setup, this determines which cluster the CRD is routed to. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + AvailabilityZone string `json:"availabilityZone"` + + // Quota maps LIQUID resource names to their quota value for THIS availability zone. // Key: liquid.ResourceName (e.g. "hw_version_hana_v2_ram") - // Mirrors liquid.ServiceQuotaRequest.Resources with AZSeparatedTopology. - // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ServiceQuotaRequest + // Value: per-AZ quota from liquid.AZResourceQuotaRequest.Quota + // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#AZResourceQuotaRequest // +kubebuilder:validation:Optional - Quota map[string]ResourceQuota `json:"quota,omitempty"` + Quota map[string]int64 `json:"quota,omitempty"` } // ProjectQuotaStatus defines the observed state of ProjectQuota. @@ -75,17 +56,17 @@ type ProjectQuotaStatus struct { // +kubebuilder:validation:Optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // TotalUsage tracks per-resource per-AZ total resource consumption (all VMs in this project). + // TotalUsage tracks per-resource total resource consumption in this AZ (all VMs in this project+AZ). // Persisted by the quota controller; updated by full reconcile and HV instance diffs. // Key: liquid.ResourceName // +kubebuilder:validation:Optional - TotalUsage map[string]ResourceQuotaUsage `json:"totalUsage,omitempty"` + TotalUsage map[string]int64 `json:"totalUsage,omitempty"` - // PaygUsage tracks per-resource per-AZ pay-as-you-go usage. + // PaygUsage tracks per-resource pay-as-you-go usage in this AZ. // Derived as TotalUsage - CRUsage (clamped >= 0). // Key: liquid.ResourceName // +kubebuilder:validation:Optional - PaygUsage map[string]ResourceQuotaUsage `json:"paygUsage,omitempty"` + PaygUsage map[string]int64 `json:"paygUsage,omitempty"` // LastReconcileAt is when the controller last reconciled this project's quota (any path). // +kubebuilder:validation:Optional @@ -106,6 +87,7 @@ type ProjectQuotaStatus struct { // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster // +kubebuilder:printcolumn:name="Project",type="string",JSONPath=".spec.projectID" +// +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" // +kubebuilder:printcolumn:name="Domain",type="string",JSONPath=".spec.domainID" // +kubebuilder:printcolumn:name="LastReconcile",type="date",JSONPath=".status.lastReconcileAt" // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" @@ -113,6 +95,8 @@ type ProjectQuotaStatus struct { // ProjectQuota is the Schema for the projectquotas API. // It persists quota values pushed by Limes via the LIQUID quota endpoint // (PUT /v1/projects/:uuid/quota → liquid.ServiceQuotaRequest). +// Each CRD stores quota for one project in one availability zone. +// In a multi-cluster setup, it is routed to the cluster serving that AZ. // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ServiceQuotaRequest type ProjectQuota struct { metav1.TypeMeta `json:",inline"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 081244ef8..a98f0bfff 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1611,9 +1611,9 @@ func (in *ProjectQuotaSpec) DeepCopyInto(out *ProjectQuotaSpec) { *out = *in if in.Quota != nil { in, out := &in.Quota, &out.Quota - *out = make(map[string]ResourceQuota, len(*in)) + *out = make(map[string]int64, len(*in)) for key, val := range *in { - (*out)[key] = *val.DeepCopy() + (*out)[key] = val } } } @@ -1633,16 +1633,16 @@ func (in *ProjectQuotaStatus) DeepCopyInto(out *ProjectQuotaStatus) { *out = *in if in.TotalUsage != nil { in, out := &in.TotalUsage, &out.TotalUsage - *out = make(map[string]ResourceQuotaUsage, len(*in)) + *out = make(map[string]int64, len(*in)) for key, val := range *in { - (*out)[key] = *val.DeepCopy() + (*out)[key] = val } } if in.PaygUsage != nil { in, out := &in.PaygUsage, &out.PaygUsage - *out = make(map[string]ResourceQuotaUsage, len(*in)) + *out = make(map[string]int64, len(*in)) for key, val := range *in { - (*out)[key] = *val.DeepCopy() + (*out)[key] = val } } if in.LastReconcileAt != nil { @@ -1822,50 +1822,6 @@ func (in *ReservationStatus) DeepCopy() *ReservationStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResourceQuota) DeepCopyInto(out *ResourceQuota) { - *out = *in - if in.PerAZ != nil { - in, out := &in.PerAZ, &out.PerAZ - *out = make(map[string]int64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceQuota. -func (in *ResourceQuota) DeepCopy() *ResourceQuota { - if in == nil { - return nil - } - out := new(ResourceQuota) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResourceQuotaUsage) DeepCopyInto(out *ResourceQuotaUsage) { - *out = *in - if in.PerAZ != nil { - in, out := &in.PerAZ, &out.PerAZ - *out = make(map[string]int64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceQuotaUsage. -func (in *ResourceQuotaUsage) DeepCopy() *ResourceQuotaUsage { - if in == nil { - return nil - } - out := new(ResourceQuotaUsage) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchedulingHistoryEntry) DeepCopyInto(out *SchedulingHistoryEntry) { *out = *in diff --git a/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml b/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml index 7c8ee1311..57354cb44 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml @@ -18,6 +18,9 @@ spec: - jsonPath: .spec.projectID name: Project type: string + - jsonPath: .spec.availabilityZone + name: AZ + type: string - jsonPath: .spec.domainID name: Domain type: string @@ -34,6 +37,8 @@ spec: ProjectQuota is the Schema for the projectquotas API. It persists quota values pushed by Limes via the LIQUID quota endpoint (PUT /v1/projects/:uuid/quota → liquid.ServiceQuotaRequest). + Each CRD stores quota for one project in one availability zone. + In a multi-cluster setup, it is routed to the cluster serving that AZ. See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ServiceQuotaRequest properties: apiVersion: @@ -57,8 +62,15 @@ spec: description: |- ProjectQuotaSpec defines the desired state of ProjectQuota. Populated from PUT /v1/projects/:uuid/quota payloads (liquid.ServiceQuotaRequest). + Each ProjectQuota CRD represents quota for ONE project in ONE availability zone. See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ServiceQuotaRequest properties: + availabilityZone: + description: |- + AvailabilityZone is the AZ this quota CRD covers (e.g. "qa-de-1a"). + In a multi-cluster setup, this determines which cluster the CRD is routed to. + minLength: 1 + type: string domainID: description: |- DomainID of the OpenStack domain this project belongs to. @@ -81,38 +93,16 @@ spec: type: string quota: additionalProperties: - description: |- - ResourceQuota holds the quota for a single resource with per-AZ breakdown. - Maps to liquid.ResourceQuotaRequest from the LIQUID API. - See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ResourceQuotaRequest - properties: - perAZ: - additionalProperties: - format: int64 - type: integer - description: |- - PerAZ holds the per-availability-zone quota breakdown. - Key: availability zone name, Value: quota for that AZ. - Only populated for AZSeparatedTopology resources. - Corresponds to liquid.ResourceQuotaRequest.PerAZ[az].Quota. - See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#AZResourceQuotaRequest - type: object - quota: - description: |- - Quota is the total quota across all AZs (for compatibility). - Corresponds to liquid.ResourceQuotaRequest.Quota. - format: int64 - type: integer - required: - - quota - type: object + format: int64 + type: integer description: |- - Quota maps LIQUID resource names to their per-AZ quota. + Quota maps LIQUID resource names to their quota value for THIS availability zone. Key: liquid.ResourceName (e.g. "hw_version_hana_v2_ram") - Mirrors liquid.ServiceQuotaRequest.Resources with AZSeparatedTopology. - See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#ServiceQuotaRequest + Value: per-AZ quota from liquid.AZResourceQuotaRequest.Quota + See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid#AZResourceQuotaRequest type: object required: + - availabilityZone - domainID - projectID type: object @@ -200,39 +190,19 @@ spec: type: integer paygUsage: additionalProperties: - description: ResourceQuotaUsage holds per-AZ PAYG usage for a single - resource. - properties: - perAZ: - additionalProperties: - format: int64 - type: integer - description: |- - PerAZ holds per-availability-zone PAYG usage values. - Key: availability zone name, Value: PAYG usage in that AZ. - type: object - type: object + format: int64 + type: integer description: |- - PaygUsage tracks per-resource per-AZ pay-as-you-go usage. + PaygUsage tracks per-resource pay-as-you-go usage in this AZ. Derived as TotalUsage - CRUsage (clamped >= 0). Key: liquid.ResourceName type: object totalUsage: additionalProperties: - description: ResourceQuotaUsage holds per-AZ PAYG usage for a single - resource. - properties: - perAZ: - additionalProperties: - format: int64 - type: integer - description: |- - PerAZ holds per-availability-zone PAYG usage values. - Key: availability zone name, Value: PAYG usage in that AZ. - type: object - type: object + format: int64 + type: integer description: |- - TotalUsage tracks per-resource per-AZ total resource consumption (all VMs in this project). + TotalUsage tracks per-resource total resource consumption in this AZ (all VMs in this project+AZ). Persisted by the quota controller; updated by full reconcile and HV instance diffs. Key: liquid.ResourceName type: object diff --git a/internal/scheduling/reservations/commitments/api/quota.go b/internal/scheduling/reservations/commitments/api/quota.go index 9c34e879c..4d6109a7b 100644 --- a/internal/scheduling/reservations/commitments/api/quota.go +++ b/internal/scheduling/reservations/commitments/api/quota.go @@ -16,21 +16,26 @@ import ( "github.com/sapcc/go-api-declarations/liquid" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) -// projectQuotaCRDName returns the CRD object name for a given project UUID. -// Convention: "quota-" -func projectQuotaCRDName(projectID string) string { - return "quota-" + projectID +// idxProjectQuotaByProjectID is the field index key used to look up ProjectQuota CRDs by project ID. +// Must match the index registered in field_index.go. +const idxProjectQuotaByProjectID = "spec.projectID" + +// projectQuotaCRDName returns the CRD object name for a given project UUID and AZ. +// Convention: "quota--" +func projectQuotaCRDName(projectID, az string) string { + return "quota-" + projectID + "-" + az } // HandleQuota implements PUT /commitments/v1/projects/:project_id/quota from Limes LIQUID API. // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid // // This endpoint receives quota requests from Limes and persists them as ProjectQuota CRDs. -// One CRD per project, named "quota-". +// One CRD per project per availability zone, named "quota--". func (api *HTTPAPI) HandleQuota(w http.ResponseWriter, r *http.Request) { startTime := time.Now() @@ -89,88 +94,108 @@ func (api *HTTPAPI) HandleQuota(w http.ResponseWriter, r *http.Request) { return } - // Build the spec quota map from the liquid request. + // Build per-AZ quota maps from the liquid request. // liquid API uses uint64; our CRD uses int64 (K8s convention). // Guard against overflow: uint64 values > MaxInt64 would wrap to negative. - specQuota := make(map[string]v1alpha1.ResourceQuota, len(req.Resources)) + // quotaByAZ[az][resourceName] = quota value for that AZ + quotaByAZ := make(map[string]map[string]int64) for resourceName, resQuota := range req.Resources { - if resQuota.Quota > math.MaxInt64 { - api.quotaError(w, http.StatusBadRequest, fmt.Sprintf("Quota value for resource %q exceeds int64 max", resourceName), startTime) - return - } - rq := v1alpha1.ResourceQuota{ - Quota: int64(resQuota.Quota), - } - if len(resQuota.PerAZ) > 0 { - rq.PerAZ = make(map[string]int64, len(resQuota.PerAZ)) - for az, azQuota := range resQuota.PerAZ { - if azQuota.Quota > math.MaxInt64 { - api.quotaError(w, http.StatusBadRequest, fmt.Sprintf("Quota value for resource %q in AZ %q exceeds int64 max", resourceName, az), startTime) - return - } - rq.PerAZ[string(az)] = int64(azQuota.Quota) + for az, azQuota := range resQuota.PerAZ { + if azQuota.Quota > math.MaxInt64 { + api.quotaError(w, http.StatusBadRequest, fmt.Sprintf("Quota value for resource %q in AZ %q exceeds int64 max", resourceName, az), startTime) + return + } + azStr := string(az) + if quotaByAZ[azStr] == nil { + quotaByAZ[azStr] = make(map[string]int64) } + quotaByAZ[azStr][string(resourceName)] = int64(azQuota.Quota) } - specQuota[string(resourceName)] = rq } - // Create or update ProjectQuota CRD with retry-on-conflict to handle - // concurrent status updates from the quota controller. - crdName := projectQuotaCRDName(projectID) ctx := r.Context() - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - var existing v1alpha1.ProjectQuota - getErr := api.client.Get(ctx, client.ObjectKey{Name: crdName}, &existing) - if getErr != nil { - if !apierrors.IsNotFound(getErr) { - return getErr - } - // Not found -- create new - pq := &v1alpha1.ProjectQuota{ - ObjectMeta: metav1.ObjectMeta{ - Name: crdName, - }, - Spec: v1alpha1.ProjectQuotaSpec{ - ProjectID: projectID, - ProjectName: projectName, - DomainID: domainID, - DomainName: domainName, - Quota: specQuota, - }, - } - if createErr := api.client.Create(ctx, pq); createErr != nil { - // If another request just created it, retry will fetch and update - if apierrors.IsAlreadyExists(createErr) { + // Create or update one ProjectQuota CRD per AZ with retry-on-conflict to handle + // concurrent status updates from the quota controller. + activeAZs := make(map[string]bool, len(quotaByAZ)) + for az, azQuota := range quotaByAZ { + activeAZs[az] = true + crdName := projectQuotaCRDName(projectID, az) + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + var existing v1alpha1.ProjectQuota + getErr := api.client.Get(ctx, client.ObjectKey{Name: crdName}, &existing) + if getErr != nil { + if !apierrors.IsNotFound(getErr) { + return getErr + } + // Not found -- create new + pq := &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdName, + }, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: projectID, + ProjectName: projectName, + DomainID: domainID, + DomainName: domainName, + AvailabilityZone: az, + Quota: azQuota, + }, + } + if createErr := api.client.Create(ctx, pq); createErr != nil { + // If another request just created it, surface as a conflict so + // RetryOnConflict re-runs the closure and falls into the update branch. + if apierrors.IsAlreadyExists(createErr) { + return apierrors.NewConflict( + schema.GroupResource{Group: "cortex.cloud", Resource: "projectquotas"}, + crdName, createErr, + ) + } return createErr } - return createErr + log.V(1).Info("created ProjectQuota", "name", crdName, "projectID", projectID, "az", az, "resources", len(azQuota)) + return nil + } + + // Update existing (re-fetched on each retry to get fresh resourceVersion) + existing.Spec.Quota = azQuota + if projectName != "" { + existing.Spec.ProjectName = projectName + } + if domainID != "" { + existing.Spec.DomainID = domainID + } + if domainName != "" { + existing.Spec.DomainName = domainName + } + if updateErr := api.client.Update(ctx, &existing); updateErr != nil { + return updateErr } - log.V(1).Info("created ProjectQuota", "name", crdName, "projectID", projectID, "resources", len(specQuota)) + log.V(1).Info("updated ProjectQuota", "name", crdName, "projectID", projectID, "az", az, "resources", len(azQuota)) return nil + }) + if err != nil { + log.Error(err, "failed to create/update ProjectQuota", "name", crdName, "az", az) + api.quotaError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to persist quota for AZ %s: %v", az, err), startTime) + return } + } - // Update existing (re-fetched on each retry to get fresh resourceVersion) - existing.Spec.Quota = specQuota - if projectName != "" { - existing.Spec.ProjectName = projectName - } - if domainID != "" { - existing.Spec.DomainID = domainID - } - if domainName != "" { - existing.Spec.DomainName = domainName - } - if updateErr := api.client.Update(ctx, &existing); updateErr != nil { - return updateErr + // Delete orphan CRDs for AZs no longer present in the quota push. + var pqList v1alpha1.ProjectQuotaList + if err := api.client.List(ctx, &pqList, client.MatchingFields{idxProjectQuotaByProjectID: projectID}); err == nil { + for i := range pqList.Items { + pq := &pqList.Items[i] + if !activeAZs[pq.Spec.AvailabilityZone] { + if delErr := api.client.Delete(ctx, pq); delErr != nil && !apierrors.IsNotFound(delErr) { + log.Error(delErr, "failed to delete orphan ProjectQuota", "name", pq.Name, "az", pq.Spec.AvailabilityZone) + // Non-fatal: orphan will be cleaned up on next push + } else { + log.V(1).Info("deleted orphan ProjectQuota", "name", pq.Name, "az", pq.Spec.AvailabilityZone) + } + } } - log.V(1).Info("updated ProjectQuota", "name", crdName, "projectID", projectID, "resources", len(specQuota)) - return nil - }) - if err != nil { - log.Error(err, "failed to create/update ProjectQuota", "name", crdName) - api.quotaError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to persist quota: %v", err), startTime) - return } // Return 204 No Content as expected by the LIQUID API diff --git a/internal/scheduling/reservations/commitments/api/quota_test.go b/internal/scheduling/reservations/commitments/api/quota_test.go index d22dcdc37..11ec744a4 100644 --- a/internal/scheduling/reservations/commitments/api/quota_test.go +++ b/internal/scheduling/reservations/commitments/api/quota_test.go @@ -15,6 +15,7 @@ import ( commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/sapcc/go-api-declarations/liquid" "go.xyrillian.de/gg/option" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -148,15 +149,14 @@ func TestHandleQuota_ErrorCases(t *testing.T) { func TestHandleQuota_CreateAndUpdate(t *testing.T) { tests := []struct { name string - // existing is a pre-existing CRD to seed (nil = create, non-nil = update) - existing *v1alpha1.ProjectQuota + // existing is a set of pre-existing per-AZ CRDs to seed (nil = create, non-nil = update) + existing []*v1alpha1.ProjectQuota projectID string resources map[liquid.ResourceName]liquid.ResourceQuotaRequest metadata *liquid.ProjectMetadata - expectQuota map[string]int64 // resource name → expected total quota - expectPerAZ map[string]map[string]int64 // resource name → az → expected quota + expectPerAZ map[string]map[string]int64 // az → resource name → expected quota expectName string - expectDomain string + expectDom string expectDomName string }{ { @@ -164,7 +164,6 @@ func TestHandleQuota_CreateAndUpdate(t *testing.T) { projectID: "project-abc-123", resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{ "hw_version_hana_1_ram": { - Quota: 100, PerAZ: map[liquid.AvailabilityZone]liquid.AZResourceQuotaRequest{ "az-a": {Quota: 60}, "az-b": {Quota: 40}, @@ -175,28 +174,21 @@ func TestHandleQuota_CreateAndUpdate(t *testing.T) { UUID: "project-abc-123", Domain: liquid.DomainMetadata{UUID: "domain-1"}, }, - expectQuota: map[string]int64{"hw_version_hana_1_ram": 100}, expectPerAZ: map[string]map[string]int64{ - "hw_version_hana_1_ram": {"az-a": 60, "az-b": 40}, + "az-a": {"hw_version_hana_1_ram": 60}, + "az-b": {"hw_version_hana_1_ram": 40}, }, - expectDomain: "domain-1", - }, - { - name: "Create_EmptyResources", - projectID: "project-empty", - resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{}, - metadata: &liquid.ProjectMetadata{ - UUID: "project-empty", - Domain: liquid.DomainMetadata{UUID: "domain-1"}, - }, - expectQuota: map[string]int64{}, - expectDomain: "domain-1", + expectDom: "domain-1", }, { name: "Create_WithMetadata", projectID: "project-meta-test", resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{ - "hw_version_hana_1_ram": {Quota: 50}, + "hw_version_hana_1_ram": { + PerAZ: map[liquid.AvailabilityZone]liquid.AZResourceQuotaRequest{ + "az-a": {Quota: 50}, + }, + }, }, metadata: &liquid.ProjectMetadata{ UUID: "project-meta-test", @@ -206,80 +198,123 @@ func TestHandleQuota_CreateAndUpdate(t *testing.T) { Name: "my-domain-name", }, }, - expectQuota: map[string]int64{"hw_version_hana_1_ram": 50}, + expectPerAZ: map[string]map[string]int64{ + "az-a": {"hw_version_hana_1_ram": 50}, + }, expectName: "my-project-name", - expectDomain: "domain-uuid-456", + expectDom: "domain-uuid-456", expectDomName: "my-domain-name", }, { - name: "Update_QuotaValues", - existing: &v1alpha1.ProjectQuota{ - Spec: v1alpha1.ProjectQuotaSpec{ - ProjectID: "project-xyz", - DomainID: "original-domain", - DomainName: "original-domain-name", - ProjectName: "original-project-name", - Quota: map[string]v1alpha1.ResourceQuota{ - "hw_version_hana_1_ram": {Quota: 50, PerAZ: map[string]int64{"az-a": 50}}, + name: "Create_EmptyResources", + projectID: "project-empty", + resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{}, + metadata: &liquid.ProjectMetadata{ + UUID: "project-empty", + Domain: liquid.DomainMetadata{UUID: "domain-1"}, + }, + // No AZs in request means no per-AZ CRDs are created. + // expectPerAZ is empty — we just verify no error and 204 response. + expectPerAZ: map[string]map[string]int64{}, + expectDom: "domain-1", + }, + { + name: "Update_WithNewMetadata", + existing: []*v1alpha1.ProjectQuota{ + { + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-update-meta-az-a"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-update-meta", + DomainID: "old-domain", + DomainName: "old-domain-name", + ProjectName: "old-project-name", + AvailabilityZone: "az-a", + Quota: map[string]int64{"hw_version_hana_1_ram": 10}, + }, + }, + }, + projectID: "project-update-meta", + resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{ + "hw_version_hana_1_ram": { + PerAZ: map[liquid.AvailabilityZone]liquid.AZResourceQuotaRequest{ + "az-a": {Quota: 99}, }, }, }, - projectID: "project-xyz", metadata: &liquid.ProjectMetadata{ - UUID: "project-xyz", - Name: "original-project-name", + UUID: "project-update-meta", + Name: "new-project-name", Domain: liquid.DomainMetadata{ - UUID: "original-domain", - Name: "original-domain-name", + UUID: "new-domain", + Name: "new-domain-name", }, }, + expectPerAZ: map[string]map[string]int64{ + "az-a": {"hw_version_hana_1_ram": 99}, + }, + expectName: "new-project-name", + expectDom: "new-domain", + expectDomName: "new-domain-name", + }, + { + name: "Create_PartialAZ_OnlyOneAZ", + projectID: "project-partial", resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{ "hw_version_hana_1_ram": { - Quota: 200, PerAZ: map[liquid.AvailabilityZone]liquid.AZResourceQuotaRequest{ - "az-a": {Quota: 120}, - "az-b": {Quota: 80}, + "az-a": {Quota: 100}, + // az-b intentionally missing }, }, }, - expectQuota: map[string]int64{"hw_version_hana_1_ram": 200}, + metadata: &liquid.ProjectMetadata{ + UUID: "project-partial", + Domain: liquid.DomainMetadata{UUID: "domain-1"}, + }, + // Only az-a should get a CRD expectPerAZ: map[string]map[string]int64{ - "hw_version_hana_1_ram": {"az-a": 120, "az-b": 80}, + "az-a": {"hw_version_hana_1_ram": 100}, }, - // Metadata should be preserved when not provided in update - expectDomain: "original-domain", - expectDomName: "original-domain-name", - expectName: "original-project-name", + expectDom: "domain-1", }, { - name: "Update_WithNewMetadata", - existing: &v1alpha1.ProjectQuota{ - Spec: v1alpha1.ProjectQuotaSpec{ - ProjectID: "project-update-meta", - DomainID: "old-domain", - DomainName: "old-domain-name", - ProjectName: "old-project-name", - Quota: map[string]v1alpha1.ResourceQuota{ - "hw_version_hana_1_ram": {Quota: 10}, + name: "Update_QuotaValues", + existing: []*v1alpha1.ProjectQuota{ + { + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-xyz-az-a"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-xyz", + DomainID: "original-domain", + DomainName: "original-domain-name", + ProjectName: "original-project-name", + AvailabilityZone: "az-a", + Quota: map[string]int64{"hw_version_hana_1_ram": 50}, }, }, }, - projectID: "project-update-meta", - resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{ - "hw_version_hana_1_ram": {Quota: 99}, - }, + projectID: "project-xyz", metadata: &liquid.ProjectMetadata{ - UUID: "project-update-meta", - Name: "new-project-name", + UUID: "project-xyz", + Name: "original-project-name", Domain: liquid.DomainMetadata{ - UUID: "new-domain", - Name: "new-domain-name", + UUID: "original-domain", + Name: "original-domain-name", }, }, - expectQuota: map[string]int64{"hw_version_hana_1_ram": 99}, - expectName: "new-project-name", - expectDomain: "new-domain", - expectDomName: "new-domain-name", + resources: map[liquid.ResourceName]liquid.ResourceQuotaRequest{ + "hw_version_hana_1_ram": { + PerAZ: map[liquid.AvailabilityZone]liquid.AZResourceQuotaRequest{ + "az-a": {Quota: 120}, + "az-b": {Quota: 80}, + }, + }, + }, + expectPerAZ: map[string]map[string]int64{ + "az-a": {"hw_version_hana_1_ram": 120}, + "az-b": {"hw_version_hana_1_ram": 80}, + }, + expectDom: "original-domain", + expectName: "original-project-name", }, } @@ -289,8 +324,11 @@ func TestHandleQuota_CreateAndUpdate(t *testing.T) { builder := fake.NewClientBuilder().WithScheme(scheme) if tc.existing != nil { - tc.existing.Name = projectQuotaCRDName(tc.projectID) - builder = builder.WithObjects(tc.existing) + objs := make([]client.Object, len(tc.existing)) + for i := range tc.existing { + objs[i] = tc.existing[i] + } + builder = builder.WithObjects(objs...) } k8sClient := builder.Build() httpAPI := NewAPI(k8sClient) @@ -316,52 +354,43 @@ func TestHandleQuota_CreateAndUpdate(t *testing.T) { t.Fatalf("expected status %d (No Content), got %d", http.StatusNoContent, resp.StatusCode) } - // Verify the ProjectQuota CRD - var pq v1alpha1.ProjectQuota - crdName := projectQuotaCRDName(tc.projectID) - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: crdName}, &pq); err != nil { - t.Fatalf("failed to get ProjectQuota CRD %q: %v", crdName, err) - } - - if pq.Spec.ProjectID != tc.projectID { - t.Errorf("expected ProjectID %q, got %q", tc.projectID, pq.Spec.ProjectID) - } + // Verify per-AZ ProjectQuota CRDs were created/updated + for az, expectedQuota := range tc.expectPerAZ { + crdName := projectQuotaCRDName(tc.projectID, az) + var pq v1alpha1.ProjectQuota + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: crdName}, &pq); err != nil { + t.Fatalf("failed to get ProjectQuota CRD %q: %v", crdName, err) + } - // Verify quota totals - for resName, expectedTotal := range tc.expectQuota { - actual, ok := pq.Spec.Quota[resName] - if !ok { - t.Errorf("expected resource %q in quota spec", resName) - continue + if pq.Spec.ProjectID != tc.projectID { + t.Errorf("CRD %q: expected ProjectID %q, got %q", crdName, tc.projectID, pq.Spec.ProjectID) } - if actual.Quota != expectedTotal { - t.Errorf("resource %q: expected quota %d, got %d", resName, expectedTotal, actual.Quota) + if pq.Spec.AvailabilityZone != az { + t.Errorf("CRD %q: expected AZ %q, got %q", crdName, az, pq.Spec.AvailabilityZone) } - } - // Verify per-AZ quotas - for resName, azMap := range tc.expectPerAZ { - actual, ok := pq.Spec.Quota[resName] - if !ok { - t.Errorf("expected resource %q in quota spec for per-AZ check", resName) - continue - } - for az, expectedAZ := range azMap { - if actual.PerAZ[az] != expectedAZ { - t.Errorf("resource %q AZ %q: expected %d, got %d", resName, az, expectedAZ, actual.PerAZ[az]) + // Verify quota values + for resName, expectedVal := range expectedQuota { + actual, ok := pq.Spec.Quota[resName] + if !ok { + t.Errorf("CRD %q: expected resource %q in quota spec", crdName, resName) + continue + } + if actual != expectedVal { + t.Errorf("CRD %q resource %q: expected %d, got %d", crdName, resName, expectedVal, actual) } } - } - // Verify metadata - if tc.expectName != "" && pq.Spec.ProjectName != tc.expectName { - t.Errorf("expected ProjectName %q, got %q", tc.expectName, pq.Spec.ProjectName) - } - if tc.expectDomain != "" && pq.Spec.DomainID != tc.expectDomain { - t.Errorf("expected DomainID %q, got %q", tc.expectDomain, pq.Spec.DomainID) - } - if tc.expectDomName != "" && pq.Spec.DomainName != tc.expectDomName { - t.Errorf("expected DomainName %q, got %q", tc.expectDomName, pq.Spec.DomainName) + // Verify metadata + if tc.expectName != "" && pq.Spec.ProjectName != tc.expectName { + t.Errorf("CRD %q: expected ProjectName %q, got %q", crdName, tc.expectName, pq.Spec.ProjectName) + } + if tc.expectDom != "" && pq.Spec.DomainID != tc.expectDom { + t.Errorf("CRD %q: expected DomainID %q, got %q", crdName, tc.expectDom, pq.Spec.DomainID) + } + if tc.expectDomName != "" && pq.Spec.DomainName != tc.expectDomName { + t.Errorf("CRD %q: expected DomainName %q, got %q", crdName, tc.expectDomName, pq.Spec.DomainName) + } } }) } diff --git a/internal/scheduling/reservations/commitments/api/report_usage_test.go b/internal/scheduling/reservations/commitments/api/report_usage_test.go index 66a1c9b9f..eb21f7b61 100644 --- a/internal/scheduling/reservations/commitments/api/report_usage_test.go +++ b/internal/scheduling/reservations/commitments/api/report_usage_test.go @@ -401,15 +401,13 @@ func TestReportUsageIntegration(t *testing.T) { {CommitmentID: "commit-1", Flavor: m1Small, ProjectID: "project-quota", AZ: "az-a", Count: 4}, }, ProjectQuota: &v1alpha1.ProjectQuota{ - ObjectMeta: metav1.ObjectMeta{Name: "quota-project-quota"}, + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-quota-az-a"}, Spec: v1alpha1.ProjectQuotaSpec{ - ProjectID: "project-quota", - DomainID: "test-domain", - Quota: map[string]v1alpha1.ResourceQuota{ - "hw_version_hana_1_ram": { - Quota: 16, - PerAZ: map[string]int64{"az-a": 16}, - }, + ProjectID: "project-quota", + DomainID: "test-domain", + AvailabilityZone: "az-a", + Quota: map[string]int64{ + "hw_version_hana_1_ram": 16, }, }, }, diff --git a/internal/scheduling/reservations/commitments/usage.go b/internal/scheduling/reservations/commitments/usage.go index 1032de5c4..9931cda9d 100644 --- a/internal/scheduling/reservations/commitments/usage.go +++ b/internal/scheduling/reservations/commitments/usage.go @@ -146,12 +146,13 @@ func (c *UsageCalculator) CalculateUsage( return liquid.ServiceUsageReport{}, fmt.Errorf("failed to read VM assignments from CRD status: %w", err) } - // Fetch the ProjectQuota CRD for this project to read per-AZ quota values. - // May not exist if Limes has not pushed quota yet — in that case quota defaults to infinite. - var projectQuota *v1alpha1.ProjectQuota + // Fetch all per-AZ ProjectQuota CRDs for this project to read quota values. + // May be empty if Limes has not pushed quota yet — in that case quota defaults to infinite. + // Each CRD holds quota for one AZ; we build a combined map[resourceName][az] = quota. var pqList v1alpha1.ProjectQuotaList + var quotaByResourceAZ map[string]map[string]int64 if err := c.client.List(ctx, &pqList, client.MatchingFields{idxProjectQuotaByProjectID: projectID}); err == nil && len(pqList.Items) > 0 { - projectQuota = &pqList.Items[0] + quotaByResourceAZ = buildCombinedQuotaMap(pqList.Items) } vms, err := getProjectVMs(ctx, c.usageDB, log, projectID, flavorGroups, allAZs) @@ -159,7 +160,7 @@ func (c *UsageCalculator) CalculateUsage( return liquid.ServiceUsageReport{}, fmt.Errorf("failed to get project VMs: %w", err) } - report := c.buildUsageResponse(vms, vmAssignments, flavorGroups, allAZs, infoVersion, projectQuota, c.config) + report := c.buildUsageResponse(vms, vmAssignments, flavorGroups, allAZs, infoVersion, quotaByResourceAZ, c.config) assignedToCommitments := 0 for _, vm := range vms { @@ -426,17 +427,34 @@ type azUsageData struct { subresources []liquid.Subresource // VM details for subresource reporting } +// buildCombinedQuotaMap aggregates per-AZ ProjectQuota CRDs into a combined lookup map. +// Returns quotaByResourceAZ[resourceName][az] = quota value. +func buildCombinedQuotaMap(pqs []v1alpha1.ProjectQuota) map[string]map[string]int64 { + result := make(map[string]map[string]int64) + for _, pq := range pqs { + az := pq.Spec.AvailabilityZone + for resourceName, quota := range pq.Spec.Quota { + if result[resourceName] == nil { + result[resourceName] = make(map[string]int64) + } + result[resourceName][az] = quota + } + } + return result +} + // buildUsageResponse constructs the Liquid API ServiceUsageReport. // All flavor groups are included in the report; commitment assignment only applies // to groups with fixed RAM/core ratio (those that accept commitments). // For each flavor group, three resources are reported: _ram, _cores, _instances. +// quotaByResourceAZ is a combined map[resourceName][az] = quota from all per-AZ ProjectQuota CRDs. func (c *UsageCalculator) buildUsageResponse( vms []VMUsageInfo, vmAssignments map[string]string, flavorGroups map[string]compute.FlavorGroupFeature, allAZs []liquid.AvailabilityZone, infoVersion int64, - projectQuota *v1alpha1.ProjectQuota, + quotaByResourceAZ map[string]map[string]int64, config APIConfig, ) liquid.ServiceUsageReport { // Initialize resources map for all flavor groups @@ -500,9 +518,9 @@ func (c *UsageCalculator) buildUsageResponse( } if ramHasAZQuota { quota := int64(-1) // default: infinite - if projectQuota != nil { - if rq, ok := projectQuota.Spec.Quota[string(ramResourceName)]; ok { - if q, ok := rq.PerAZ[string(az)]; ok { + if quotaByResourceAZ != nil { + if azMap, ok := quotaByResourceAZ[string(ramResourceName)]; ok { + if q, ok := azMap[string(az)]; ok { quota = q } } diff --git a/internal/scheduling/reservations/quota/controller.go b/internal/scheduling/reservations/quota/controller.go index 5a4d49e74..f374080bc 100644 --- a/internal/scheduling/reservations/quota/controller.go +++ b/internal/scheduling/reservations/quota/controller.go @@ -183,7 +183,7 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // Determine if this is a spec change (new CRD or quota update) vs. a CR UsedAmount change specChanged := pq.Generation > pq.Status.ObservedGeneration - var totalUsage map[string]v1alpha1.ResourceQuotaUsage + var totalUsage map[string]map[string]int64 if specChanged { // Spec changed (new CRD or quota update) — recompute TotalUsage from Postgres logger.Info("spec changed, recomputing TotalUsage from Postgres", @@ -195,9 +195,12 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, err } } else { - // CR UsedAmount changed — read persisted TotalUsage, only recompute PaygUsage - totalUsage = pq.Status.TotalUsage - if totalUsage == nil { + // CR UsedAmount changed — read persisted TotalUsage, only recompute PaygUsage. + // Status stores flat map[string]int64 (for this AZ only), but internal functions + // operate on map[string]map[string]int64. Reconstruct the multi-AZ view. + if pq.Status.TotalUsage != nil { + totalUsage = expandAZSlice(pq.Status.TotalUsage, pq.Spec.AvailabilityZone) + } else { // Safety fallback: TotalUsage should always be set after first spec reconcile logger.Info("no TotalUsage persisted, computing as fallback") var err error @@ -247,7 +250,7 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // computeTotalUsageForProject computes TotalUsage for a single project by reading // all VMs from Postgres and filtering to the target project. Used as bootstrap when // a ProjectQuota is first created and has no persisted TotalUsage yet. -func (c *QuotaController) computeTotalUsageForProject(ctx context.Context, projectID string) (map[string]v1alpha1.ResourceQuotaUsage, error) { +func (c *QuotaController) computeTotalUsageForProject(ctx context.Context, projectID string) (map[string]map[string]int64, error) { // Fetch flavor groups from Knowledge CRD flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) @@ -452,7 +455,7 @@ func (c *QuotaController) isVMNewSinceLastReconcile(ctx context.Context, vm *fai } // Look up the ProjectQuota for this VM's project - crdName := "quota-" + vm.ProjectID + crdName := "quota-" + vm.ProjectID + "-" + vm.AvailabilityZone var pq v1alpha1.ProjectQuota if err := c.Get(ctx, client.ObjectKey{Name: crdName}, &pq); err != nil { // If we can't find the ProjectQuota, skip (full reconcile will handle it) @@ -535,8 +538,8 @@ func (c *QuotaController) accumulateRemovedVM( delta.addDecrement(commitments.ResourceNameCores(groupName), info.AvailabilityZone, coresAmount) } -// applyDeltaAndUpdateStatus fetches the ProjectQuota, applies the batched delta to TotalUsage, -// recomputes PaygUsage, and persists with conflict retry. +// applyDeltaAndUpdateStatus applies batched deltas to ALL per-AZ ProjectQuota CRDs for a project. +// It lists all per-AZ CRDs, applies relevant deltas to each, recomputes PaygUsage, and persists. func (c *QuotaController) applyDeltaAndUpdateStatus( ctx context.Context, projectID string, @@ -545,51 +548,80 @@ func (c *QuotaController) applyDeltaAndUpdateStatus( flavorGroups map[string]compute.FlavorGroupFeature, ) error { - crdName := "quota-" + projectID - - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Re-fetch fresh copy on each retry - var pq v1alpha1.ProjectQuota - if err := c.Get(ctx, client.ObjectKey{Name: crdName}, &pq); err != nil { - if client.IgnoreNotFound(err) == nil { - return nil // PQ deleted, nothing to do - } - return err + // Collect all AZs affected by this delta + affectedAZs := make(map[string]bool) + for _, azAmounts := range delta.increments { + for az := range azAmounts { + affectedAZs[az] = true } - - if pq.Status.TotalUsage == nil { - pq.Status.TotalUsage = make(map[string]v1alpha1.ResourceQuotaUsage) + } + for _, azAmounts := range delta.decrements { + for az := range azAmounts { + affectedAZs[az] = true } + } + + crUsage := c.computeCRUsage(projectCRs, flavorGroups) - // Apply increments - for resourceName, azAmounts := range delta.increments { - for az, amount := range azAmounts { - incrementUsage(pq.Status.TotalUsage, resourceName, az, amount) + for az := range affectedAZs { + crdName := "quota-" + projectID + "-" + az + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var pq v1alpha1.ProjectQuota + if err := c.Get(ctx, client.ObjectKey{Name: crdName}, &pq); err != nil { + if client.IgnoreNotFound(err) == nil { + return nil // PQ for this AZ doesn't exist, skip + } + return err } - } - // Apply decrements - for resourceName, azAmounts := range delta.decrements { - for az, amount := range azAmounts { - decrementUsage(pq.Status.TotalUsage, resourceName, az, amount) + if pq.Status.TotalUsage == nil { + pq.Status.TotalUsage = make(map[string]int64) } - } - // Recompute PaygUsage - crUsage := c.computeCRUsage(projectCRs, flavorGroups) - paygUsage := derivePaygUsage(pq.Status.TotalUsage, crUsage) + // Apply increments for this AZ + for resourceName, azAmounts := range delta.increments { + if amount, ok := azAmounts[az]; ok { + pq.Status.TotalUsage[resourceName] += amount + } + } - pq.Status.PaygUsage = paygUsage - now := metav1.Now() - pq.Status.LastReconcileAt = &now + // Apply decrements for this AZ + for resourceName, azAmounts := range delta.decrements { + if amount, ok := azAmounts[az]; ok { + pq.Status.TotalUsage[resourceName] -= amount + if pq.Status.TotalUsage[resourceName] < 0 { + pq.Status.TotalUsage[resourceName] = 0 + } + } + } - if err := c.Status().Update(ctx, &pq); err != nil { + // Derive PaygUsage for this AZ: totalUsage[resource] - crUsage[resource][az] + pq.Status.PaygUsage = make(map[string]int64) + for resourceName, totalAmount := range pq.Status.TotalUsage { + crAmount := int64(0) + if cr, ok := crUsage[resourceName]; ok { + if azAmount, ok := cr[az]; ok { + crAmount = azAmount + } + } + paygAmount := totalAmount - crAmount + if paygAmount < 0 { + paygAmount = 0 + } + pq.Status.PaygUsage[resourceName] = paygAmount + } + + now := metav1.Now() + pq.Status.LastReconcileAt = &now + return c.Status().Update(ctx, &pq) + }) + if err != nil { return err } + } - c.recordUsageMetrics(projectID, pq.Status.TotalUsage, paygUsage, crUsage) - return nil - }) + return nil } // ============================================================================ @@ -679,9 +711,9 @@ func (c *QuotaController) computeTotalUsage( vms []failover.VM, flavorToGroup map[string]string, flavorGroups map[string]compute.FlavorGroupFeature, -) map[string]map[string]v1alpha1.ResourceQuotaUsage { +) map[string]map[string]map[string]int64 { // result[projectID][resourceName] = ResourceQuotaUsage{PerAZ: {az: amount}} - result := make(map[string]map[string]v1alpha1.ResourceQuotaUsage) + result := make(map[string]map[string]map[string]int64) for _, vm := range vms { groupName, ok := flavorToGroup[vm.FlavorName] @@ -698,23 +730,23 @@ func (c *QuotaController) computeTotalUsage( ramUnits, coresAmount := vmResourceUnits(vm.Resources) if _, ok := result[vm.ProjectID]; !ok { - result[vm.ProjectID] = make(map[string]v1alpha1.ResourceQuotaUsage) + result[vm.ProjectID] = make(map[string]map[string]int64) } // Accumulate RAM usage for this project + AZ ramUsage := result[vm.ProjectID][ramResourceName] - if ramUsage.PerAZ == nil { - ramUsage.PerAZ = make(map[string]int64) + if ramUsage == nil { + ramUsage = make(map[string]int64) } - ramUsage.PerAZ[vm.AvailabilityZone] += ramUnits + ramUsage[vm.AvailabilityZone] += ramUnits result[vm.ProjectID][ramResourceName] = ramUsage // Accumulate cores usage for this project + AZ coresUsage := result[vm.ProjectID][coresResourceName] - if coresUsage.PerAZ == nil { - coresUsage.PerAZ = make(map[string]int64) + if coresUsage == nil { + coresUsage = make(map[string]int64) } - coresUsage.PerAZ[vm.AvailabilityZone] += coresAmount + coresUsage[vm.AvailabilityZone] += coresAmount result[vm.ProjectID][coresResourceName] = coresUsage } @@ -733,8 +765,8 @@ func groupCRsByProject(crs []v1alpha1.CommittedResource) map[string][]v1alpha1.C // computeCRUsage computes the committed resource usage from a pre-filtered slice of CRs for one project. // It reads UsedResources from each CR's status and converts to commitment units (multiples for RAM, raw for cores). -func (c *QuotaController) computeCRUsage(crs []v1alpha1.CommittedResource, flavorGroups map[string]compute.FlavorGroupFeature) map[string]v1alpha1.ResourceQuotaUsage { - result := make(map[string]v1alpha1.ResourceQuotaUsage) +func (c *QuotaController) computeCRUsage(crs []v1alpha1.CommittedResource, flavorGroups map[string]compute.FlavorGroupFeature) map[string]map[string]int64 { + result := make(map[string]map[string]int64) for i := range crs { cr := &crs[i] @@ -789,10 +821,10 @@ func (c *QuotaController) computeCRUsage(crs []v1alpha1.CommittedResource, flavo // Accumulate per AZ usage := result[resourceName] - if usage.PerAZ == nil { - usage.PerAZ = make(map[string]int64) + if usage == nil { + usage = make(map[string]int64) } - usage.PerAZ[spec.AvailabilityZone] += usedAmount + usage[spec.AvailabilityZone] += usedAmount result[resourceName] = usage } @@ -811,20 +843,18 @@ func (c *QuotaController) isCRStateIncluded(state v1alpha1.CommitmentStatus) boo // derivePaygUsage computes PaygUsage = TotalUsage - CRUsage (clamped >= 0). func derivePaygUsage( - totalUsage map[string]v1alpha1.ResourceQuotaUsage, - crUsage map[string]v1alpha1.ResourceQuotaUsage, -) map[string]v1alpha1.ResourceQuotaUsage { + totalUsage map[string]map[string]int64, + crUsage map[string]map[string]int64, +) map[string]map[string]int64 { - result := make(map[string]v1alpha1.ResourceQuotaUsage) + result := make(map[string]map[string]int64) for resourceName, total := range totalUsage { - payg := v1alpha1.ResourceQuotaUsage{ - PerAZ: make(map[string]int64), - } - for az, totalAmount := range total.PerAZ { + payg := make(map[string]int64) + for az, totalAmount := range total { crAmount := int64(0) if cr, ok := crUsage[resourceName]; ok { - if azAmount, ok := cr.PerAZ[az]; ok { + if azAmount, ok := cr[az]; ok { crAmount = azAmount } } @@ -832,7 +862,7 @@ func derivePaygUsage( if paygAmount < 0 { paygAmount = 0 // Clamp >= 0 } - payg.PerAZ[az] = paygAmount + payg[az] = paygAmount } result[resourceName] = payg } @@ -840,14 +870,38 @@ func derivePaygUsage( return result } +// extractAZSlice extracts the data for a single AZ from a multi-AZ usage map. +// Returns map[resourceName] = value for that AZ only. +func extractAZSlice(usage map[string]map[string]int64, az string) map[string]int64 { + result := make(map[string]int64) + for resourceName, azMap := range usage { + if val, ok := azMap[az]; ok { + result[resourceName] = val + } + } + return result +} + +// expandAZSlice reconstructs a multi-AZ map from a flat per-AZ map. +// Used when reading persisted status (flat) back into the controller's internal format. +func expandAZSlice(flat map[string]int64, az string) map[string]map[string]int64 { + result := make(map[string]map[string]int64) + for resourceName, val := range flat { + result[resourceName] = map[string]int64{az: val} + } + return result +} + // updateProjectQuotaStatusWithRetry writes TotalUsage + PaygUsage + LastReconcileAt // with retry-on-conflict to handle concurrent updates. +// totalUsage and paygUsage are multi-AZ maps; this function extracts the relevant AZ +// slice based on the CRD's Spec.AvailabilityZone. // If fullReconcile is true, also updates LastFullReconcileAt and ObservedGeneration. func (c *QuotaController) updateProjectQuotaStatusWithRetry( ctx context.Context, pqName string, - totalUsage map[string]v1alpha1.ResourceQuotaUsage, - paygUsage map[string]v1alpha1.ResourceQuotaUsage, + totalUsage map[string]map[string]int64, + paygUsage map[string]map[string]int64, fullReconcile bool, ) error { @@ -858,8 +912,10 @@ func (c *QuotaController) updateProjectQuotaStatusWithRetry( return err } - pq.Status.TotalUsage = totalUsage - pq.Status.PaygUsage = paygUsage + // Extract only this AZ's data from the multi-AZ maps + az := pq.Spec.AvailabilityZone + pq.Status.TotalUsage = extractAZSlice(totalUsage, az) + pq.Status.PaygUsage = extractAZSlice(paygUsage, az) pq.Status.ObservedGeneration = pq.Generation now := metav1.Now() pq.Status.LastReconcileAt = &now @@ -892,24 +948,24 @@ func buildFlavorToGroupMap(flavorGroups map[string]compute.FlavorGroupFeature) m } // incrementUsage increments a usage value in the map. -func incrementUsage(usage map[string]v1alpha1.ResourceQuotaUsage, resourceName, az string, amount int64) { +func incrementUsage(usage map[string]map[string]int64, resourceName, az string, amount int64) { u := usage[resourceName] - if u.PerAZ == nil { - u.PerAZ = make(map[string]int64) + if u == nil { + u = make(map[string]int64) } - u.PerAZ[az] += amount + u[az] += amount usage[resourceName] = u } // decrementUsage decrements a usage value in the map (clamp >= 0). -func decrementUsage(usage map[string]v1alpha1.ResourceQuotaUsage, resourceName, az string, amount int64) { +func decrementUsage(usage map[string]map[string]int64, resourceName, az string, amount int64) { u := usage[resourceName] - if u.PerAZ == nil { + if u == nil { return } - u.PerAZ[az] -= amount - if u.PerAZ[az] < 0 { - u.PerAZ[az] = 0 + u[az] -= amount + if u[az] < 0 { + u[az] = 0 } usage[resourceName] = u } @@ -917,20 +973,20 @@ func decrementUsage(usage map[string]v1alpha1.ResourceQuotaUsage, resourceName, // recordUsageMetrics emits Prometheus metrics for all resources in a project. func (c *QuotaController) recordUsageMetrics( projectID string, - totalUsage map[string]v1alpha1.ResourceQuotaUsage, - paygUsage map[string]v1alpha1.ResourceQuotaUsage, - crUsage map[string]v1alpha1.ResourceQuotaUsage, + totalUsage map[string]map[string]int64, + paygUsage map[string]map[string]int64, + crUsage map[string]map[string]int64, ) { for resourceName, total := range totalUsage { - for az, totalAmount := range total.PerAZ { + for az, totalAmount := range total { paygAmount := int64(0) if payg, ok := paygUsage[resourceName]; ok { - paygAmount = payg.PerAZ[az] + paygAmount = payg[az] } crAmount := int64(0) if cr, ok := crUsage[resourceName]; ok { - crAmount = cr.PerAZ[az] + crAmount = cr[az] } c.Metrics.RecordUsage(projectID, az, resourceName, totalAmount, paygAmount, crAmount) } @@ -947,17 +1003,17 @@ func (c *QuotaController) mapCRToProjectQuota(_ context.Context, obj client.Obje if !ok { return nil } - // Map to the ProjectQuota for this project - crdName := "quota-" + cr.Spec.ProjectID + // Map to the per-AZ ProjectQuota for this project + AZ + crdName := "quota-" + cr.Spec.ProjectID + "-" + cr.Spec.AvailabilityZone return []reconcile.Request{ {NamespacedName: client.ObjectKey{Name: crdName}}, } } -// crUsedResourcesChangePredicate triggers only when Status.UsedResources changes on a CommittedResource. +// crUsedResourcesChangePredicate triggers on create, delete, and UsedResources changes of a CommittedResource. func crUsedAmountChangePredicate() predicate.Predicate { return predicate.Funcs{ - CreateFunc: func(_ event.CreateEvent) bool { return false }, + CreateFunc: func(_ event.CreateEvent) bool { return true }, UpdateFunc: func(e event.UpdateEvent) bool { oldCR, ok1 := e.ObjectOld.(*v1alpha1.CommittedResource) newCR, ok2 := e.ObjectNew.(*v1alpha1.CommittedResource) diff --git a/internal/scheduling/reservations/quota/controller_test.go b/internal/scheduling/reservations/quota/controller_test.go index df995876e..b5b724647 100644 --- a/internal/scheduling/reservations/quota/controller_test.go +++ b/internal/scheduling/reservations/quota/controller_test.go @@ -101,19 +101,19 @@ func TestComputeTotalUsage(t *testing.T) { } ramUsage := projectA["hw_version_hana_v2_ram"] - if ramUsage.PerAZ["az-1"] != 96 { - t.Errorf("expected project-a az-1 hana_v2_ram = 96, got %d", ramUsage.PerAZ["az-1"]) + if ramUsage["az-1"] != 96 { + t.Errorf("expected project-a az-1 hana_v2_ram = 96, got %d", ramUsage["az-1"]) } - if ramUsage.PerAZ["az-2"] != 32 { - t.Errorf("expected project-a az-2 hana_v2_ram = 32, got %d", ramUsage.PerAZ["az-2"]) + if ramUsage["az-2"] != 32 { + t.Errorf("expected project-a az-2 hana_v2_ram = 32, got %d", ramUsage["az-2"]) } coresUsage := projectA["hw_version_hana_v2_cores"] - if coresUsage.PerAZ["az-1"] != 24 { - t.Errorf("expected project-a az-1 hana_v2_cores = 24, got %d", coresUsage.PerAZ["az-1"]) + if coresUsage["az-1"] != 24 { + t.Errorf("expected project-a az-1 hana_v2_cores = 24, got %d", coresUsage["az-1"]) } - if coresUsage.PerAZ["az-2"] != 8 { - t.Errorf("expected project-a az-2 hana_v2_cores = 8, got %d", coresUsage.PerAZ["az-2"]) + if coresUsage["az-2"] != 8 { + t.Errorf("expected project-a az-2 hana_v2_cores = 8, got %d", coresUsage["az-2"]) } // project-b: general in az-1: 4096/1024 = 4 GiB RAM, 2 cores @@ -121,11 +121,11 @@ func TestComputeTotalUsage(t *testing.T) { if projectB == nil { t.Fatal("expected project-b in results") } - if projectB["hw_version_general_ram"].PerAZ["az-1"] != 4 { - t.Errorf("expected project-b az-1 general_ram = 4, got %d", projectB["hw_version_general_ram"].PerAZ["az-1"]) + if projectB["hw_version_general_ram"]["az-1"] != 4 { + t.Errorf("expected project-b az-1 general_ram = 4, got %d", projectB["hw_version_general_ram"]["az-1"]) } - if projectB["hw_version_general_cores"].PerAZ["az-1"] != 2 { - t.Errorf("expected project-b az-1 general_cores = 2, got %d", projectB["hw_version_general_cores"].PerAZ["az-1"]) + if projectB["hw_version_general_cores"]["az-1"] != 2 { + t.Errorf("expected project-b az-1 general_cores = 2, got %d", projectB["hw_version_general_cores"]["az-1"]) } // project-c: unknown flavor → not in results @@ -216,35 +216,35 @@ func TestComputeCRUsage(t *testing.T) { // Should include confirmed + guaranteed for project-a only ramUsage := result["hw_version_hana_v2_ram"] - if ramUsage.PerAZ["az-1"] != 8 { // 5 + 3 - t.Errorf("expected cr ram usage az-1 = 8, got %d", ramUsage.PerAZ["az-1"]) + if ramUsage["az-1"] != 8 { // 5 + 3 + t.Errorf("expected cr ram usage az-1 = 8, got %d", ramUsage["az-1"]) } coresUsage := result["hw_version_hana_v2_cores"] - if coresUsage.PerAZ["az-1"] != 2 { - t.Errorf("expected cr cores usage az-1 = 2, got %d", coresUsage.PerAZ["az-1"]) + if coresUsage["az-1"] != 2 { + t.Errorf("expected cr cores usage az-1 = 2, got %d", coresUsage["az-1"]) } - // az-2 should NOT be included (pending state) - if ramUsage.PerAZ["az-2"] != 0 { - t.Errorf("expected cr ram usage az-2 = 0 (pending excluded), got %d", ramUsage.PerAZ["az-2"]) + // az-2 should NOT be included (pending state) — assert key absence, not zero value + if got, exists := ramUsage["az-2"]; exists { + t.Errorf("expected cr ram usage az-2 to be absent (pending excluded), got %d", got) } } func TestDerivePaygUsage(t *testing.T) { tests := []struct { name string - totalUsage map[string]v1alpha1.ResourceQuotaUsage - crUsage map[string]v1alpha1.ResourceQuotaUsage + totalUsage map[string]map[string]int64 + crUsage map[string]map[string]int64 expected map[string]map[string]int64 // resourceName -> az -> amount }{ { name: "basic subtraction", - totalUsage: map[string]v1alpha1.ResourceQuotaUsage{ - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 10, "az-2": 5}}, + totalUsage: map[string]map[string]int64{ + "hw_version_hana_v2_ram": {"az-1": 10, "az-2": 5}, }, - crUsage: map[string]v1alpha1.ResourceQuotaUsage{ - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 3}}, + crUsage: map[string]map[string]int64{ + "hw_version_hana_v2_ram": {"az-1": 3}, }, expected: map[string]map[string]int64{ "hw_version_hana_v2_ram": {"az-1": 7, "az-2": 5}, @@ -252,11 +252,11 @@ func TestDerivePaygUsage(t *testing.T) { }, { name: "clamp to zero", - totalUsage: map[string]v1alpha1.ResourceQuotaUsage{ - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 2}}, + totalUsage: map[string]map[string]int64{ + "hw_version_hana_v2_ram": {"az-1": 2}, }, - crUsage: map[string]v1alpha1.ResourceQuotaUsage{ - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 10}}, + crUsage: map[string]map[string]int64{ + "hw_version_hana_v2_ram": {"az-1": 10}, }, expected: map[string]map[string]int64{ "hw_version_hana_v2_ram": {"az-1": 0}, @@ -264,19 +264,19 @@ func TestDerivePaygUsage(t *testing.T) { }, { name: "no CR usage", - totalUsage: map[string]v1alpha1.ResourceQuotaUsage{ - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 5}}, + totalUsage: map[string]map[string]int64{ + "hw_version_hana_v2_ram": {"az-1": 5}, }, - crUsage: map[string]v1alpha1.ResourceQuotaUsage{}, + crUsage: map[string]map[string]int64{}, expected: map[string]map[string]int64{ "hw_version_hana_v2_ram": {"az-1": 5}, }, }, { name: "empty total usage", - totalUsage: map[string]v1alpha1.ResourceQuotaUsage{}, - crUsage: map[string]v1alpha1.ResourceQuotaUsage{ - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 5}}, + totalUsage: map[string]map[string]int64{}, + crUsage: map[string]map[string]int64{ + "hw_version_hana_v2_ram": {"az-1": 5}, }, expected: map[string]map[string]int64{}, }, @@ -293,9 +293,9 @@ func TestDerivePaygUsage(t *testing.T) { continue } for az, expectedAmount := range expectedAZ { - if resUsage.PerAZ[az] != expectedAmount { + if resUsage[az] != expectedAmount { t.Errorf("resource=%s az=%s: expected %d, got %d", - resourceName, az, expectedAmount, resUsage.PerAZ[az]) + resourceName, az, expectedAmount, resUsage[az]) } } } @@ -342,37 +342,37 @@ func TestBuildFlavorToGroupMap(t *testing.T) { } func TestIncrementDecrementUsage(t *testing.T) { - usage := make(map[string]v1alpha1.ResourceQuotaUsage) + usage := make(map[string]map[string]int64) // Increment from empty incrementUsage(usage, "res1", "az-1", 5) - if usage["res1"].PerAZ["az-1"] != 5 { - t.Errorf("expected 5 after increment, got %d", usage["res1"].PerAZ["az-1"]) + if usage["res1"]["az-1"] != 5 { + t.Errorf("expected 5 after increment, got %d", usage["res1"]["az-1"]) } // Increment again incrementUsage(usage, "res1", "az-1", 3) - if usage["res1"].PerAZ["az-1"] != 8 { - t.Errorf("expected 8 after second increment, got %d", usage["res1"].PerAZ["az-1"]) + if usage["res1"]["az-1"] != 8 { + t.Errorf("expected 8 after second increment, got %d", usage["res1"]["az-1"]) } // Decrement decrementUsage(usage, "res1", "az-1", 2) - if usage["res1"].PerAZ["az-1"] != 6 { - t.Errorf("expected 6 after decrement, got %d", usage["res1"].PerAZ["az-1"]) + if usage["res1"]["az-1"] != 6 { + t.Errorf("expected 6 after decrement, got %d", usage["res1"]["az-1"]) } // Decrement below zero → clamp to 0 decrementUsage(usage, "res1", "az-1", 100) - if usage["res1"].PerAZ["az-1"] != 0 { - t.Errorf("expected 0 after over-decrement, got %d", usage["res1"].PerAZ["az-1"]) + if usage["res1"]["az-1"] != 0 { + t.Errorf("expected 0 after over-decrement, got %d", usage["res1"]["az-1"]) } // Decrement non-existent resource (no-op) decrementUsage(usage, "res2", "az-1", 5) // Should not panic, and res2 should not exist if _, exists := usage["res2"]; exists { - if usage["res2"].PerAZ != nil { + if usage["res2"] != nil { t.Error("expected res2 to not have PerAZ after decrement on non-existent") } } @@ -502,8 +502,8 @@ func TestAccumulateAddedVM_KnownFlavor(t *testing.T) { } pq := &v1alpha1.ProjectQuota{ - ObjectMeta: metav1.ObjectMeta{Name: "quota-project-a"}, - Spec: v1alpha1.ProjectQuotaSpec{ProjectID: "project-a"}, + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-a-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ProjectID: "project-a", AvailabilityZone: "az-1"}, Status: v1alpha1.ProjectQuotaStatus{ LastReconcileAt: &lastReconcile, LastFullReconcileAt: &lastReconcile, diff --git a/internal/scheduling/reservations/quota/integration_test.go b/internal/scheduling/reservations/quota/integration_test.go index 3ec320ca3..dbe174f69 100644 --- a/internal/scheduling/reservations/quota/integration_test.go +++ b/internal/scheduling/reservations/quota/integration_test.go @@ -34,8 +34,9 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), - makePQ("project-b", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), + makePQPerAZ("project-b", "az-1", nil), }, Actions: []TestAction{ { @@ -44,29 +45,29 @@ func TestIntegration(t *testing.T) { // project-a: hana_v2 az-2: 32768/1024 = 32 GiB, 8 cores // project-a: general az-1: 4096/1024 = 4 GiB, 2 cores // project-b: general az-1: 4096/1024 = 4 GiB, 2 cores - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, // No CRs -> PaygUsage == TotalUsage - ExpectedPaygUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedPaygUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -77,7 +78,8 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, CommittedResources: []*v1alpha1.CommittedResource{ // 2 units of hana_v2 RAM committed in az-1 for project-a @@ -90,24 +92,24 @@ func TestIntegration(t *testing.T) { Actions: []TestAction{ { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, // PaygUsage = TotalUsage - CRUsage // hana_v2 RAM: 96-2=94 in az-1, 32-0=32 in az-2 // hana_v2 Cores: 24-10=14 in az-1, 8-0=8 in az-2 // general: no CRs so PaygUsage == TotalUsage - ExpectedPaygUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedPaygUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 94, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 14, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 94, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 14, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -118,18 +120,19 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, Actions: []TestAction{ // Step 1: full reconcile to establish baseline { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -158,12 +161,12 @@ func TestIntegration(t *testing.T) { ), // vm-new is created AFTER last reconcile, so it gets incremented // +32 GiB RAM (32768/1024), +8 cores - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 128, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 32, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 128, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 32, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -174,18 +177,19 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, Actions: []TestAction{ // Step 1: full reconcile { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -197,12 +201,12 @@ func TestIntegration(t *testing.T) { activeInstance("vm-1"), // migrated here, created before reconcile }), // Should NOT increment -- vm-1 CreatedAt is 2025-12-01 which is before reconcile time - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -226,18 +230,19 @@ func TestIntegration(t *testing.T) { "vm-del": false, // not active (truly deleted) }, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, Actions: []TestAction{ // Step 1: full reconcile (vm-del not in VMs so not counted) { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -256,12 +261,12 @@ func TestIntegration(t *testing.T) { }), // vm-del: IsServerActive=false, deleted info found // Decrement: -32 GiB RAM, -8 cores in az-1 - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 64, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 16, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 64, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 16, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -275,17 +280,18 @@ func TestIntegration(t *testing.T) { "vm-1": true, // still active (migrated to another HV) }, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, Actions: []TestAction{ { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -301,12 +307,12 @@ func TestIntegration(t *testing.T) { // vm-1 gone from this HV }), // vm-1: IsServerActive=true, so NOT decremented - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -317,7 +323,8 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, CommittedResources: []*v1alpha1.CommittedResource{ makeCR("cr-ram-1", "project-a", "hana_v2", "az-1", @@ -327,12 +334,12 @@ func TestIntegration(t *testing.T) { // Step 1: full reconcile with initial CR (UsedAmount=1) { Type: "full_reconcile", - ExpectedPaygUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedPaygUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 95, "az-2": 32}}, // 96-1=95 - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 95, "az-2": 32}, // 96-1=95 + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -341,12 +348,12 @@ func TestIntegration(t *testing.T) { Type: "cr_update", CRName: "cr-ram-1", UsedAmount: 3, - ExpectedPaygUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedPaygUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 93, "az-2": 32}}, // 96-3=93 - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 93, "az-2": 32}, // 96-3=93 + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -366,16 +373,16 @@ func TestIntegration(t *testing.T) { }, }, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-x", nil), + makePQPerAZ("project-x", "az-1", nil), }, Actions: []TestAction{ { Type: "full_reconcile", // No usage for project-x (unknown flavor skipped) - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-x": {}, }, - ExpectedPaygUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedPaygUsage: map[string]map[string]map[string]int64{ "project-x": {}, }, }, @@ -386,38 +393,39 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), - makePQ("project-b", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), + makePQPerAZ("project-b", "az-1", nil), }, Actions: []TestAction{ { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, // Second full reconcile - same result { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -428,7 +436,8 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, CommittedResources: []*v1alpha1.CommittedResource{ // Pending CR should NOT reduce PaygUsage @@ -439,12 +448,12 @@ func TestIntegration(t *testing.T) { { Type: "full_reconcile", // PaygUsage == TotalUsage because pending CRs are excluded - ExpectedPaygUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedPaygUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -455,18 +464,19 @@ func TestIntegration(t *testing.T) { FlavorGroups: testFlavorGroups, VMs: testVMs, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), }, Actions: []TestAction{ // Step 1: full reconcile establishes correct baseline { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -496,12 +506,12 @@ func TestIntegration(t *testing.T) { }, ), // TotalUsage now has phantom's contribution (drift) - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 128, "az-2": 32}}, // 96+32 drift - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 32, "az-2": 8}}, // 24+8 drift - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 128, "az-2": 32}, // 96+32 drift + "hw_version_hana_v2_cores": {"az-1": 32, "az-2": 8}, // 24+8 drift + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -510,12 +520,12 @@ func TestIntegration(t *testing.T) { { Type: "full_reconcile", OverrideVMs: baseVMsPtr(), - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, // corrected - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, // corrected - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, // corrected + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, // corrected + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -539,8 +549,9 @@ func TestIntegration(t *testing.T) { "vm-1": true, // still active (for migration scenario) }, ProjectQuotas: []*v1alpha1.ProjectQuota{ - makePQ("project-a", nil), - makePQ("project-b", nil), + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), + makePQPerAZ("project-b", "az-1", nil), }, Actions: []TestAction{ // Step 1: full reconcile establishes baseline for both projects @@ -548,16 +559,16 @@ func TestIntegration(t *testing.T) { // project-b general: az-1=4 GiB / 2 cores { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -585,12 +596,12 @@ func TestIntegration(t *testing.T) { }, }, ), - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 128, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 32, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 128, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 32, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -616,10 +627,10 @@ func TestIntegration(t *testing.T) { }, }, ), - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 8}}, // 4+4 drift - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 4}}, // 2+2 drift + "hw_version_general_ram": {"az-1": 8}, // 4+4 drift + "hw_version_general_cores": {"az-1": 4}, // 2+2 drift }, }, }, @@ -650,12 +661,12 @@ func TestIntegration(t *testing.T) { }, }, ), - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 96, "az-2": 32}}, // 128-32=96 - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 24, "az-2": 8}}, // 32-8=24 - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, // 128-32=96 + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, // 32-8=24 + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -679,16 +690,16 @@ func TestIntegration(t *testing.T) { }, }, }, - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 128, "az-2": 32}}, // corrected up - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 32, "az-2": 8}}, // corrected up - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 128, "az-2": 32}, // corrected up + "hw_version_hana_v2_cores": {"az-1": 32, "az-2": 8}, // corrected up + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, // corrected down - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, // corrected down + "hw_version_general_ram": {"az-1": 4}, // corrected down + "hw_version_general_cores": {"az-1": 2}, // corrected down }, }, }, @@ -717,12 +728,12 @@ func TestIntegration(t *testing.T) { }, ), // vm-1 migrated, NOT decremented -- totals unchanged - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 128, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 32, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 128, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 32, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -730,16 +741,100 @@ func TestIntegration(t *testing.T) { // This is the "reconcile that matches the deltas" -- nothing to fix. { Type: "full_reconcile", - ExpectedTotalUsage: map[string]map[string]v1alpha1.ResourceQuotaUsage{ + ExpectedTotalUsage: map[string]map[string]map[string]int64{ "project-a": { - "hw_version_hana_v2_ram": {PerAZ: map[string]int64{"az-1": 128, "az-2": 32}}, - "hw_version_hana_v2_cores": {PerAZ: map[string]int64{"az-1": 32, "az-2": 8}}, - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_hana_v2_ram": {"az-1": 128, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 32, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, "project-b": { - "hw_version_general_ram": {PerAZ: map[string]int64{"az-1": 4}}, - "hw_version_general_cores": {PerAZ: map[string]int64{"az-1": 2}}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, + }, + }, + }, + }, + }, + { + Name: "partial AZ coverage - only az-1 has CRD, az-2 VMs are ignored", + FlavorGroups: testFlavorGroups, + VMs: testVMs, // project-a has VMs in az-1 AND az-2 + ProjectQuotas: []*v1alpha1.ProjectQuota{ + // Only az-1 CRD exists — az-2 has VMs but no CRD + makePQPerAZ("project-a", "az-1", nil), + }, + Actions: []TestAction{ + { + Type: "full_reconcile", + // Only az-1 data should be written (az-2 CRD doesn't exist) + ExpectedTotalUsage: map[string]map[string]map[string]int64{ + "project-a": { + "hw_version_hana_v2_ram": {"az-1": 96}, + "hw_version_hana_v2_cores": {"az-1": 24}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, + }, + }, + ExpectedPaygUsage: map[string]map[string]map[string]int64{ + "project-a": { + "hw_version_hana_v2_ram": {"az-1": 96}, + "hw_version_hana_v2_cores": {"az-1": 24}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, + }, + }, + }, + }, + }, + { + Name: "total calculation - multi-resource multi-AZ verified", + FlavorGroups: testFlavorGroups, + VMs: testVMs, + ProjectQuotas: []*v1alpha1.ProjectQuota{ + makePQPerAZ("project-a", "az-1", nil), + makePQPerAZ("project-a", "az-2", nil), + makePQPerAZ("project-b", "az-1", nil), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + // 5 GiB hana_v2 RAM committed in az-1, 3 GiB in az-2 + makeCR("cr-ram-az1", "project-a", "hana_v2", "az-1", + v1alpha1.CommittedResourceTypeMemory, v1alpha1.CommitmentStatusConfirmed, int64Ptr(5)), + makeCR("cr-ram-az2", "project-a", "hana_v2", "az-2", + v1alpha1.CommittedResourceTypeMemory, v1alpha1.CommitmentStatusConfirmed, int64Ptr(3)), + // 4 cores committed in az-1 + makeCR("cr-cores-az1", "project-a", "hana_v2", "az-1", + v1alpha1.CommittedResourceTypeCores, v1alpha1.CommitmentStatusConfirmed, int64Ptr(4)), + }, + Actions: []TestAction{ + { + Type: "full_reconcile", + // Verify TotalUsage is correctly computed from VMs + ExpectedTotalUsage: map[string]map[string]map[string]int64{ + "project-a": { + "hw_version_hana_v2_ram": {"az-1": 96, "az-2": 32}, + "hw_version_hana_v2_cores": {"az-1": 24, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, + }, + "project-b": { + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, + }, + }, + // Verify PaygUsage = TotalUsage - CRUsage per AZ + // az-1: hana_v2_ram: 96-5=91, hana_v2_cores: 24-4=20 + // az-2: hana_v2_ram: 32-3=29, hana_v2_cores: 8-0=8 + ExpectedPaygUsage: map[string]map[string]map[string]int64{ + "project-a": { + "hw_version_hana_v2_ram": {"az-1": 91, "az-2": 29}, + "hw_version_hana_v2_cores": {"az-1": 20, "az-2": 8}, + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, + }, + "project-b": { + "hw_version_general_ram": {"az-1": 4}, + "hw_version_general_cores": {"az-1": 2}, }, }, }, @@ -869,8 +964,8 @@ type TestAction struct { // Optional: verify state AFTER this action completes. // Keys are project IDs. If nil, no verification for this step. - ExpectedTotalUsage map[string]map[string]v1alpha1.ResourceQuotaUsage - ExpectedPaygUsage map[string]map[string]v1alpha1.ResourceQuotaUsage + ExpectedTotalUsage map[string]map[string]map[string]int64 + ExpectedPaygUsage map[string]map[string]map[string]int64 } // IntegrationTestCase defines a complete integration test scenario. @@ -982,69 +1077,123 @@ func newIntegrationTestEnv(t *testing.T, tc IntegrationTestCase) *integrationTes } } -func (env *integrationTestEnv) verifyTotalUsage(projectID string, expected map[string]v1alpha1.ResourceQuotaUsage) { +func (env *integrationTestEnv) verifyTotalUsage(projectID string, expected map[string]map[string]int64) { env.t.Helper() - crdName := "quota-" + projectID - var pq v1alpha1.ProjectQuota - if err := env.client.Get(context.Background(), client.ObjectKey{Name: crdName}, &pq); err != nil { - env.t.Fatalf("failed to get ProjectQuota %s: %v", crdName, err) + + if expected == nil { + return } - if expected == nil && pq.Status.TotalUsage == nil { - return // both nil, ok + // Collect expected data per AZ: az → resourceName → value + perAZ := make(map[string]map[string]int64) + for resourceName, azMap := range expected { + for az, val := range azMap { + if perAZ[az] == nil { + perAZ[az] = make(map[string]int64) + } + perAZ[az][resourceName] = val + } } - for resourceName, expectedUsage := range expected { - actual, ok := pq.Status.TotalUsage[resourceName] - if !ok { - env.t.Errorf("project %s: expected TotalUsage resource %q not found", projectID, resourceName) - continue + for az, expectedResources := range perAZ { + crdName := "quota-" + projectID + "-" + az + var pq v1alpha1.ProjectQuota + if err := env.client.Get(context.Background(), client.ObjectKey{Name: crdName}, &pq); err != nil { + env.t.Fatalf("failed to get ProjectQuota %s: %v", crdName, err) } - for az, expectedAmount := range expectedUsage.PerAZ { - if actual.PerAZ[az] != expectedAmount { + + for resourceName, expectedAmount := range expectedResources { + actual, ok := pq.Status.TotalUsage[resourceName] + if !ok { + env.t.Errorf("project %s AZ %s: expected TotalUsage resource %q not found", projectID, az, resourceName) + continue + } + if actual != expectedAmount { env.t.Errorf("project %s: TotalUsage[%s][%s] = %d, want %d", - projectID, resourceName, az, actual.PerAZ[az], expectedAmount) + projectID, resourceName, az, actual, expectedAmount) + } + } + + // Check no unexpected resources + for resourceName := range pq.Status.TotalUsage { + if _, ok := expectedResources[resourceName]; !ok { + env.t.Errorf("project %s AZ %s: unexpected TotalUsage resource %q", projectID, az, resourceName) } } } - // Check no unexpected resources - for resourceName := range pq.Status.TotalUsage { - if _, ok := expected[resourceName]; !ok { - env.t.Errorf("project %s: unexpected TotalUsage resource %q", projectID, resourceName) + // Ensure no unexpected AZ CRDs carry TotalUsage for this project. + var allPQ v1alpha1.ProjectQuotaList + if err := env.client.List(context.Background(), &allPQ); err != nil { + env.t.Fatalf("failed to list ProjectQuota objects: %v", err) + } + for _, pq := range allPQ.Items { + if pq.Spec.ProjectID != projectID { + continue + } + az := pq.Spec.AvailabilityZone + if _, ok := perAZ[az]; !ok && len(pq.Status.TotalUsage) > 0 { + env.t.Errorf("project %s AZ %s: unexpected TotalUsage in non-expected AZ CRD", projectID, az) } } } -func (env *integrationTestEnv) verifyPaygUsage(projectID string, expected map[string]v1alpha1.ResourceQuotaUsage) { +func (env *integrationTestEnv) verifyPaygUsage(projectID string, expected map[string]map[string]int64) { env.t.Helper() - crdName := "quota-" + projectID - var pq v1alpha1.ProjectQuota - if err := env.client.Get(context.Background(), client.ObjectKey{Name: crdName}, &pq); err != nil { - env.t.Fatalf("failed to get ProjectQuota %s: %v", crdName, err) - } - if expected == nil && pq.Status.PaygUsage == nil { + if expected == nil { return } - for resourceName, expectedUsage := range expected { - actual, ok := pq.Status.PaygUsage[resourceName] - if !ok { - env.t.Errorf("project %s: expected PaygUsage resource %q not found", projectID, resourceName) - continue + // Collect expected data per AZ: az → resourceName → value + perAZ := make(map[string]map[string]int64) + for resourceName, azMap := range expected { + for az, val := range azMap { + if perAZ[az] == nil { + perAZ[az] = make(map[string]int64) + } + perAZ[az][resourceName] = val + } + } + + for az, expectedResources := range perAZ { + crdName := "quota-" + projectID + "-" + az + var pq v1alpha1.ProjectQuota + if err := env.client.Get(context.Background(), client.ObjectKey{Name: crdName}, &pq); err != nil { + env.t.Fatalf("failed to get ProjectQuota %s: %v", crdName, err) } - for az, expectedAmount := range expectedUsage.PerAZ { - if actual.PerAZ[az] != expectedAmount { + + for resourceName, expectedAmount := range expectedResources { + actual, ok := pq.Status.PaygUsage[resourceName] + if !ok { + env.t.Errorf("project %s AZ %s: expected PaygUsage resource %q not found", projectID, az, resourceName) + continue + } + if actual != expectedAmount { env.t.Errorf("project %s: PaygUsage[%s][%s] = %d, want %d", - projectID, resourceName, az, actual.PerAZ[az], expectedAmount) + projectID, resourceName, az, actual, expectedAmount) + } + } + + for resourceName := range pq.Status.PaygUsage { + if _, ok := expectedResources[resourceName]; !ok { + env.t.Errorf("project %s AZ %s: unexpected PaygUsage resource %q", projectID, az, resourceName) } } } - for resourceName := range pq.Status.PaygUsage { - if _, ok := expected[resourceName]; !ok { - env.t.Errorf("project %s: unexpected PaygUsage resource %q", projectID, resourceName) + // Ensure no unexpected AZ CRDs carry PaygUsage for this project. + var allPQ v1alpha1.ProjectQuotaList + if err := env.client.List(context.Background(), &allPQ); err != nil { + env.t.Fatalf("failed to list ProjectQuota objects: %v", err) + } + for _, pq := range allPQ.Items { + if pq.Spec.ProjectID != projectID { + continue + } + az := pq.Spec.AvailabilityZone + if _, ok := perAZ[az]; !ok && len(pq.Status.PaygUsage) > 0 { + env.t.Errorf("project %s AZ %s: unexpected PaygUsage in non-expected AZ CRD", projectID, az) } } } @@ -1091,8 +1240,8 @@ func (env *integrationTestEnv) executeAction(action TestAction) { env.t.Fatalf("failed to update CR %s status: %v", action.CRName, err) } - // Simulate watch trigger: call Reconcile for the affected project - pqName := "quota-" + cr.Spec.ProjectID + // Simulate watch trigger: call Reconcile for the affected per-AZ CRD + pqName := "quota-" + cr.Spec.ProjectID + "-" + cr.Spec.AvailabilityZone _, err := env.controller.Reconcile(ctx, reconcileRequest(pqName)) if err != nil { env.t.Fatalf("Reconcile failed after CR update: %v", err) @@ -1171,7 +1320,7 @@ func reconcileRequest(name string) ctrl.Request { return ctrl.Request{NamespacedName: client.ObjectKey{Name: name}} } -func makePQ(projectID string, lastReconcileAt *metav1.Time) *v1alpha1.ProjectQuota { //nolint:unparam +func makePQ(projectID string, lastReconcileAt *metav1.Time) *v1alpha1.ProjectQuota { //nolint:unused return &v1alpha1.ProjectQuota{ ObjectMeta: metav1.ObjectMeta{Name: "quota-" + projectID}, Spec: v1alpha1.ProjectQuotaSpec{ProjectID: projectID, DomainID: "domain-1"}, @@ -1181,6 +1330,21 @@ func makePQ(projectID string, lastReconcileAt *metav1.Time) *v1alpha1.ProjectQuo } } +// makePQPerAZ creates a per-AZ ProjectQuota CRD for integration tests. +func makePQPerAZ(projectID, az string, lastReconcileAt *metav1.Time) *v1alpha1.ProjectQuota { //nolint:unparam + return &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-" + projectID + "-" + az}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: projectID, + DomainID: "domain-1", + AvailabilityZone: az, + }, + Status: v1alpha1.ProjectQuotaStatus{ + LastReconcileAt: lastReconcileAt, + }, + } +} + func makeCR(name, projectID, flavorGroup, az string, resourceType v1alpha1.CommittedResourceType, state v1alpha1.CommitmentStatus, usedAmount *int64) *v1alpha1.CommittedResource { //nolint:unparam cr := &v1alpha1.CommittedResource{ ObjectMeta: metav1.ObjectMeta{Name: name}, diff --git a/pkg/multicluster/routers.go b/pkg/multicluster/routers.go index f2c5ef6a8..fdbd251dc 100644 --- a/pkg/multicluster/routers.go +++ b/pkg/multicluster/routers.go @@ -20,6 +20,7 @@ var DefaultResourceRouters = map[schema.GroupVersionKind]ResourceRouter{ {Group: "cortex.cloud", Version: "v1alpha1", Kind: "Reservation"}: ReservationsResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "History"}: HistoryResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "CommittedResource"}: CommittedResourceRouter{}, + {Group: "cortex.cloud", Version: "v1alpha1", Kind: "ProjectQuota"}: ProjectQuotaResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "FlavorGroupCapacity"}: FlavorGroupCapacityResourceRouter{}, } @@ -165,3 +166,30 @@ func (h HistoryResourceRouter) Match(obj any, labels map[string]string) (bool, e } return *hist.Spec.AvailabilityZone == availabilityZone, nil } + +// ProjectQuotaResourceRouter routes project quotas to clusters based on availability zone. +type ProjectQuotaResourceRouter struct{} + +func (p ProjectQuotaResourceRouter) Match(obj any, labels map[string]string) (bool, error) { + var pq v1alpha1.ProjectQuota + + switch v := obj.(type) { + case *v1alpha1.ProjectQuota: + if v == nil { + return false, errors.New("object is nil") + } + pq = *v + case v1alpha1.ProjectQuota: + pq = v + default: + return false, errors.New("object is not a ProjectQuota") + } + availabilityZone, ok := labels["availabilityZone"] + if !ok { + return false, errors.New("cluster does not have availabilityZone label") + } + if pq.Spec.AvailabilityZone == "" { + return false, errors.New("project quota does not have availability zone in spec") + } + return pq.Spec.AvailabilityZone == availabilityZone, nil +}