Neessing/vz 8729 1

.verrazzano-development-version

@@ -1,4 +1,4 @@
 # Copyright (C) 2020, 2021, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
-verrazzano-development-version=1.5.0
+verrazzano-development-version=1.5.1

Makefile

@@ -1,4 +1,4 @@
-# Copyright (C) 2020, 2022, Oracle and/or its affiliates.
+# Copyright (C) 2020, 2023, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 OPERATOR_NAME:=verrazzano-monitoring-operator
 ESWAIT_NAME:=verrazzano-monitoring-instance-eswait
@@ -42,8 +42,6 @@ endif
 
 DOCKER_NAMESPACE ?= verrazzano
 DOCKER_REPO ?= ghcr.io
-DIST_DIR:=dist
-BIN_DIR:=${DIST_DIR}/bin
 BIN_NAME:=${OPERATOR_NAME}
 K8S_EXTERNAL_IP:=localhost
 K8S_NAMESPACE:=verrazzano-system
@@ -64,8 +62,8 @@ CRD_FILE:=./k8s/crds/verrazzano.io_verrazzanomonitoringinstances.yaml
 .PHONY: all
 all: build
 
-BUILDVERSION=`git describe --tags`
-BUILDDATE=`date +%FT%T%z`
+BUILDVERSION=$(shell grep verrazzano-development-version .verrazzano-development-version | cut -d= -f 2)
+BUILDDATE=$(shell date +"%Y-%m-%dT%H:%M:%SZ")
 
 .PHONY: manifests
 manifests: controller-gen
@@ -112,31 +110,22 @@ go-vendor:
 # Docker-related tasks and functions
 #
 
-.PHONY: docker-clean
-docker-clean:
-	rm -rf ${DIST_DIR}
-
-.PHONY: k8s-dist
-k8s-dist: docker-clean
-	echo ${DOCKER_IMAGE_TAG} ${JENKINS_URL} ${CI_COMMIT_TAG} ${CI_COMMIT_SHA}
-	echo ${DOCKER_IMAGE_NAME_OPERATOR}
-	mkdir -p ${DIST_DIR}
-	cp -r docker-images/verrazzano-monitoring-operator/* ${DIST_DIR}
-	cp -r k8s/manifests/verrazzano-monitoring-operator.yaml $(DIST_DIR)/verrazzano-monitoring-operator.yaml
-
-	# Fill in Docker image and tag that's being tested
-	sed -i.bak "s|${DOCKER_REPO}/${DOCKER_NAMESPACE}/verrazzano-monitoring-operator:latest|${DOCKER_REPO}/${DOCKER_NAMESPACE}/${DOCKER_IMAGE_NAME_OPERATOR}:$(DOCKER_IMAGE_TAG)|g" $(DIST_DIR)/verrazzano-monitoring-operator.yaml
-	sed -i.bak "s/latest/$(DOCKER_IMAGE_TAG)/g" $(DIST_DIR)/verrazzano-monitoring-operator.yaml
-	sed -i.bak "s/default/${K8S_NAMESPACE}/g" $(DIST_DIR)/verrazzano-monitoring-operator.yaml
-
-	rm -rf $(DIST_DIR)/verrazzano-monitoring-operator*.bak
-	mkdir -p ${BIN_DIR}
-
 .PHONY: build
-build: k8s-dist
+build:
+	docker build --pull --no-cache \
+		--build-arg BUILDVERSION=${BUILDVERSION} \
+		--build-arg BUILDDATE=${BUILDDATE} \
+		--build-arg EXTLDFLAGS="-s -w" \
+		-t ${DOCKER_IMAGE_NAME_OPERATOR}:${DOCKER_IMAGE_TAG} \
+		-f ${DOCKERFILE_OPERATOR} \
+		.
+
+.PHONY: build-debug
+build-debug:
 	docker build --pull --no-cache \
 		--build-arg BUILDVERSION=${BUILDVERSION} \
 		--build-arg BUILDDATE=${BUILDDATE} \
+		--build-arg EXTLDFLAGS="" \
 		-t ${DOCKER_IMAGE_NAME_OPERATOR}:${DOCKER_IMAGE_TAG} \
 		-f ${DOCKERFILE_OPERATOR} \
 		.
@@ -149,9 +138,14 @@ buildhook:
            -ldflags "-X main.buildVersion=${BUILDVERSION} -X main.buildDate=${BUILDDATE}" \
            -o /usr/bin/verrazzano-backup-hook ./verrazzano-backup-hook
 
+.PHONY: push-debug
+push-debug: build-debug push-common
 
 .PHONY: push
-push: build
+push: build push-common
+
+.PHONY: push-common
+push-common:
 	docker tag ${DOCKER_IMAGE_NAME_OPERATOR}:${DOCKER_IMAGE_TAG} ${DOCKER_IMAGE_FULLNAME_OPERATOR}:${DOCKER_IMAGE_TAG}
 	docker push ${DOCKER_IMAGE_FULLNAME_OPERATOR}:${DOCKER_IMAGE_TAG}
 

docker-images/verrazzano-monitoring-operator/Dockerfile

@@ -13,28 +13,28 @@ RUN yum update -y \
 
 ARG BUILDVERSION
 ARG BUILDDATE
+ARG EXTLDFLAGS
 
 # Need to use specific WORKDIR to match verrazzano-monitoring-operator's source packages
 WORKDIR /root/go/src/github.com/verrazzano/verrazzano-monitoring-operator
 ENV GOPATH /root/go
 ENV CGO_ENABLED 0
 COPY . .
 RUN go build \
-    -ldflags '-extldflags "-static"' \
-    -ldflags "-X main.buildVersion=${BUILDVERSION} -X main.buildDate=${BUILDDATE}" \
-    -o /usr/bin/verrazzano-monitoring-operator ./cmd/verrazzano-monitoring-ctrl
+    -ldflags "$EXTLDFLAGS -extldflags -static -X 'main.buildVersion=$BUILDVERSION' -X 'main.buildDate=$BUILDDATE'" \
+    -o /usr/bin/verrazzano-monitoring-operator ./cmd/verrazzano-monitoring-ctrl \
+    && chmod 500 /usr/bin/verrazzano-monitoring-operator
 
 
-FROM ghcr.io/oracle/oraclelinux:7-slim AS final
+FROM ghcr.io/oracle/oraclelinux:8-slim AS final
 
-RUN yum update -y \
-    && yum install -y openssl \
-    && yum clean all \
-    && rm -rf /var/cache/yum
-COPY --from=build_base /usr/bin/verrazzano-monitoring-operator /usr/local/bin/verrazzano-monitoring-operator
-WORKDIR /usr/local/bin/
-RUN groupadd -r verrazzano-monitoring-operator && useradd --no-log-init -r -g verrazzano-monitoring-operator -u 1000 verrazzano-monitoring-operator
-RUN chown 1000:verrazzano-monitoring-operator /usr/local/bin/verrazzano-monitoring-operator && chmod 500 /usr/local/bin/verrazzano-monitoring-operator
+RUN microdnf update -y \
+    && microdnf clean all \
+    && rm -rf /var/cache/yum /var/lib/rpm/__db.* \
+    && groupadd -r verrazzano-monitoring-operator \
+    && useradd --no-log-init -r -g verrazzano-monitoring-operator -u 1000 verrazzano-monitoring-operator
+
+COPY --from=build_base --chown=verrazzano-monitoring-operator:verrazzano-monitoring-operator /usr/bin/verrazzano-monitoring-operator /usr/local/bin/verrazzano-monitoring-operator
 USER 1000
 
 ENTRYPOINT ["/usr/local/bin/verrazzano-monitoring-operator"]

pkg/opensearch/ism.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/verrazzano/verrazzano-monitoring-operator/pkg/util/logs/vzlog"
 	"net/http"
 	"strings"
 
@@ -252,11 +253,7 @@ func (o *OSClient) deletePolicy(opensearchEndpoint, policyName string) (*http.Re
 
 // updateISMPolicyFromFile creates or updates the ISM policy from the given json file.
 // If ISM policy doesn't exist, it will create new. Otherwise, it'll create one.
-func (o *OSClient) updateISMPolicyFromFile(openSearchEndpoint string, policyFileName string, policyName string) (*ISMPolicy, error) {
-	policy, err := getISMPolicyFromFile(policyFileName)
-	if err != nil {
-		return nil, err
-	}
+func (o *OSClient) updateISMPolicy(openSearchEndpoint string, policyName string, policy *ISMPolicy) (*ISMPolicy, error) {
 	existingPolicyURL := fmt.Sprintf("%s/_plugins/_ism/policies/%s", openSearchEndpoint, policyName)
 	existingPolicy, err := o.getPolicyByName(existingPolicyURL)
 	if err != nil {
@@ -266,14 +263,27 @@ func (o *OSClient) updateISMPolicyFromFile(openSearchEndpoint string, policyFile
 }
 
 // createOrUpdateDefaultISMPolicy creates the default ISM policies if not exist, else the policies will be updated.
-func (o *OSClient) createOrUpdateDefaultISMPolicy(openSearchEndpoint string) ([]*ISMPolicy, error) {
+func (o *OSClient) createOrUpdateDefaultISMPolicy(log vzlog.VerrazzanoLogger, openSearchEndpoint string) ([]*ISMPolicy, error) {
 	var defaultPolicies []*ISMPolicy
+	allPolicyList, err := o.getAllPolicies(openSearchEndpoint)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf("os system has %v policies", len(allPolicyList.Policies))
 	for policyName, policyFile := range defaultISMPoliciesMap {
-		createdPolicy, err := o.updateISMPolicyFromFile(openSearchEndpoint, policyFile, policyName)
+		policy, err := getISMPolicyFromFile(policyFile)
 		if err != nil {
-			return defaultPolicies, err
+			return nil, err
+		}
+		log.Debugf("checking if custom policy exists for %s from file %s", policyName, policyFile)
+		if !o.isCustomPolicyExists(log, policy, policyName, allPolicyList.Policies) {
+			log.Debugf("creating default policy for policy %s", policyName)
+			createdPolicy, err := o.updateISMPolicy(openSearchEndpoint, policyName, policy)
+			if err != nil {
+				return defaultPolicies, err
+			}
+			defaultPolicies = append(defaultPolicies, createdPolicy)
 		}
-		defaultPolicies = append(defaultPolicies, createdPolicy)
 	}
 	return defaultPolicies, nil
 }
@@ -377,3 +387,26 @@ func getISMPolicyFromFile(policyFileName string) (*ISMPolicy, error) {
 	}
 	return &policy, nil
 }
+
+func (o *OSClient) isCustomPolicyExists(log vzlog.VerrazzanoLogger, searchPolicy *ISMPolicy, searchPolicyName string, policyList []ISMPolicy) bool {
+	for _, policy := range policyList {
+		if *policy.ID != searchPolicyName && policy.Policy.ISMTemplate[0].Priority == searchPolicy.Policy.ISMTemplate[0].Priority && isItemAlreadyExists(log, policy.Policy.ISMTemplate[0].IndexPatterns, searchPolicy.Policy.ISMTemplate[0].IndexPatterns) {
+			log.Debugf("custom policy exists for policy %s", searchPolicyName)
+			return true
+		}
+	}
+	return false
+}
+func isItemAlreadyExists(log vzlog.VerrazzanoLogger, allListPolicyPatterns []string, subListPolicyPattern []string) bool {
+	matched := false
+	log.Debugf("searching for index pattern %s in all ISM policies %s", subListPolicyPattern, allListPolicyPatterns)
+	for _, al := range allListPolicyPatterns {
+		for _, sl := range subListPolicyPattern {
+			if al == sl {
+				matched = true
+				break
+			}
+		}
+	}
+	return matched
+}

pkg/opensearch/ism_test.go

@@ -590,7 +590,11 @@ func TestUpdateISMPolicyFromFile(t *testing.T) {
 				DoHTTP:            tt.fields.DoHTTP,
 				statefulSetLister: tt.fields.statefulSetLister,
 			}
-			got, err := o.updateISMPolicyFromFile(tt.args.openSearchEndpoint, tt.args.policyFileName, tt.args.policyName)
+			policyObject, err := getISMPolicyFromFile(tt.args.policyFileName)
+			if err != nil {
+				return
+			}
+			got, err := o.updateISMPolicy(tt.args.openSearchEndpoint, tt.args.policyFileName, policyObject)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("updateISMPolicyFromFile() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/opensearch/opensearch.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/verrazzano/verrazzano-monitoring-operator/pkg/util/logs/vzlog"
 	"net/http"
 
 	"go.uber.org/zap"
@@ -128,7 +129,7 @@ func (o *OSClient) DeleteDefaultISMPolicy(vmi *vmcontrollerv1.VerrazzanoMonitori
 
 // SyncDefaultISMPolicy set up the default ISM Policies.
 // The returned channel should be read for exactly one response, which tells whether default ISM policies are synced.
-func (o *OSClient) SyncDefaultISMPolicy(vmi *vmcontrollerv1.VerrazzanoMonitoringInstance) chan error {
+func (o *OSClient) SyncDefaultISMPolicy(log vzlog.VerrazzanoLogger, vmi *vmcontrollerv1.VerrazzanoMonitoringInstance) chan error {
 	ch := make(chan error)
 	go func() {
 		if !vmi.Spec.Opensearch.Enabled || vmi.Spec.Opensearch.DisableDefaultPolicy {
@@ -141,7 +142,8 @@ func (o *OSClient) SyncDefaultISMPolicy(vmi *vmcontrollerv1.VerrazzanoMonitoring
 			return
 		}
 		openSearchEndpoint := resources.GetOpenSearchHTTPEndpoint(vmi)
-		_, err := o.createOrUpdateDefaultISMPolicy(openSearchEndpoint)
+		log.Debugf("calling createOrUpdateDefaultISMPolicy")
+		_, err := o.createOrUpdateDefaultISMPolicy(log, openSearchEndpoint)
 		ch <- err
 	}()
 

pkg/vmo/controller.go

@@ -478,7 +478,7 @@ func (c *Controller) syncHandlerStandardMode(vmo *vmcontrollerv1.VerrazzanoMonit
 	/*********************
 	 * Synchronise Default ISM Policies
 	 **********************/
-	defaultISMChannel := c.osClient.SyncDefaultISMPolicy(vmo)
+	defaultISMChannel := c.osClient.SyncDefaultISMPolicy(c.log, vmo)
 
 	/********************************************
 	 * Migrate old indices if any to data streams