From bcb1545500694733634675a4fd3f765b37e99032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 8 Sep 2025 22:38:38 +0200 Subject: [PATCH 01/17] Dump diagnostic information fixes --- .../e2e/dump_diagnostic_information.sh | 342 ++++++++++-------- ...gnostic_information_from_all_namespaces.sh | 7 +- scripts/evergreen/e2e/e2e.sh | 2 +- 3 files changed, 205 insertions(+), 146 deletions(-) diff --git a/scripts/evergreen/e2e/dump_diagnostic_information.sh b/scripts/evergreen/e2e/dump_diagnostic_information.sh index 5b889621a..021843b04 100755 --- a/scripts/evergreen/e2e/dump_diagnostic_information.sh +++ b/scripts/evergreen/e2e/dump_diagnostic_information.sh @@ -8,12 +8,10 @@ set +e source scripts/funcs/printing -dump_all_non_default_namespaces() { - echo "Gathering logs from all non-default namespaces" - +_dump_all_non_default_namespaces() { + local context="${1}" local original_context original_context="$(kubectl config current-context)" - kubectl config use-context "${1:-${original_context}}" &> /dev/null prefix="${1:-${original_context}}_" # shellcheck disable=SC2154 if [[ "${KUBE_ENVIRONMENT_NAME:-}" != "multi" ]]; then @@ -21,36 +19,40 @@ dump_all_non_default_namespaces() { fi mkdir -p logs - namespaces=$(kubectl get namespace --output=jsonpath="{.items[*].metadata.name}" | tr ' ' '\n' | \ + namespaces=$(kubectl --context="${context}" get namespace --output=jsonpath="{.items[*].metadata.name}" | tr ' ' '\n' | \ grep -v "default" | \ grep -v "kube-node-lease" | \ grep -v "kube-node-lease" | \ grep -v "kube-public" | \ grep -v "kube-system" | \ grep -v "local-path-storage" | \ + grep -v "gmp-" | \ + grep -v "gke-managed" | \ + grep -v "local-path-storage" | \ + grep -v "local-path-storage" | \ grep -v "metallb-system" ) for ns in ${namespaces}; do - if kubectl get namespace "${ns}" -o jsonpath='{.metadata.annotations}'; then + if kubectl --context="${context}" get namespace "${ns}" -o jsonpath='{.metadata.annotations}'; then echo "Dumping all diagnostic information for namespace ${ns}" - dump_namespace "${ns}" "${prefix}" + dump_namespace "${context}" "${ns}" "${prefix}" fi done } -dump_all() { +dump_all_non_default_namespaces() { + local context="${1}" + _dump_all_non_default_namespaces "$@" 2>&1 | prepend "${context}" +} + +_dump_all() { [[ "${MODE-}" = "dev" ]] && return mkdir -p logs - # TODO: provide a cleaner way of handling this. For now we run the same command with kubectl configured - # with a different context. - local original_context - original_context="$(kubectl config current-context)" - kubectl config use-context "${1:-${original_context}}" &> /dev/null - prefix="${1:-${original_context}}_" - # shellcheck disable=SC2154 + local context="${1}" + prefix="${context}_" if [[ "${KUBE_ENVIRONMENT_NAME:-}" != "multi" ]]; then prefix="" fi @@ -59,194 +61,242 @@ dump_all() { # but in some exceptional cases (e.g. clusterwide operator) there can be more than 1 namespace to print diagnostics # In this case the python test app may create the test namespace and add necessary labels and annotations so they # would be dumped for diagnostics as well - for ns in $(kubectl get namespace -l "evg=task" --output=jsonpath={.items..metadata.name}); do - if kubectl get namespace "${ns}" -o jsonpath='{.metadata.annotations}' | grep -q "${task_id:-'not-specified'}"; then + for ns in $(kubectl --context="${context}" get namespace -l "evg=task" --output=jsonpath={.items..metadata.name}); do + if kubectl --context="${context}" get namespace "${ns}" -o jsonpath='{.metadata.annotations}' | grep -q "${task_id:-'not-specified'}"; then echo "Dumping all diagnostic information for namespace ${ns}" - dump_namespace "${ns}" "${prefix}" + dump_namespace "${context}" "${ns}" "${prefix}" fi done - if kubectl get namespace "olm" &>/dev/null; then + if kubectl --context="${context}" get namespace "olm" &>/dev/null; then echo "Dumping olm namespace" - dump_namespace "olm" "olm" + dump_namespace "${context}" "olm" "olm" fi - kubectl config use-context "${original_context}" &> /dev/null + kubectl --context="${context}" -n "kube-system" get configmap coredns -o yaml > "logs/${prefix}coredns.yaml" - kubectl -n "kube-system" get configmap coredns -o yaml > "logs/${prefix}coredns.yaml" + kubectl --context="${context}" events --all-namespaces > "logs/${prefix}kube_events.json" +} - kubectl events --all-namespaces > "logs/${prefix}kube_events.json" +dump_all() { + local context="${1}" + _dump_all "$@" 2>&1 | prepend "${context}" } dump_objects() { - local object=$1 - local msg=$2 - local namespace=${3} - local action=${4:-get -o yaml} + local context=$1 + local object=$2 + local msg=$3 + local namespace=${4} + local action=${5:-get -o yaml} + local out_file=${6:-""} + + # First check if the resource type exists + if ! kubectl --context="${context}" get "${object}" --no-headers -o name -n "${namespace}" >/dev/null 2>&1; then + # Resource type doesn't exist, skip silently + return + fi - if [ "$(kubectl get "${object}" --no-headers -o name -n "${namespace}" | wc -l)" = "0" ]; then - # if no objects of this type, return + # Check if there are any objects of this type + local resource_count + resource_count=$(kubectl --context="${context}" get "${object}" --no-headers -o name -n "${namespace}" 2>/dev/null | wc -l) + if [ "${resource_count}" -eq 0 ]; then + # Resource type exists but no objects of this type, return return fi - header "${msg}" + # Capture output first to check if it contains actual resources + local temp_output # shellcheck disable=SC2086 - kubectl -n "${namespace}" ${action} "${object}" 2>&1 + temp_output=$(kubectl --context="${context}" -n "${namespace}" ${action} "${object}" 2>&1) + + # Check if output contains actual resources (not just empty list) + # Skip if it's an empty YAML list (contains "items: []") + if printf '%s\n' "${temp_output}" | grep -Fq "items: []"; then + # Empty list, don't create file + return + fi + + if [[ -n "${out_file}" ]]; then + { + header "${msg}" + echo "${temp_output}" + } > "${out_file}" + else + header "${msg}" + # shellcheck disable=SC2086 + kubectl --context="${context}" -n "${namespace}" ${action} "${object}" 2>&1 + fi } # get_operator_managed_pods returns a list of names of the Pods that are managed # by the Operator. get_operator_managed_pods() { - local namespace=${1} - kubectl get pods --namespace "${namespace}" --selector=controller=mongodb-enterprise-operator --no-headers -o custom-columns=":metadata.name" + local context=${1} + local namespace=${2} + kubectl --context="${context}" get pods --namespace "${namespace}" --selector=controller=mongodb-enterprise-operator --no-headers -o custom-columns=":metadata.name" } get_all_pods() { - local namespace=${1} - kubectl get pods --namespace "${namespace}" --no-headers -o custom-columns=":metadata.name" + local context=${1} + local namespace=${2} + kubectl --context="${context}" get pods --namespace "${namespace}" --no-headers -o custom-columns=":metadata.name" } is_mdb_resource_pod() { - local pod="${1}" - local namespace="${2}" + local context="${1}" + local pod="${2}" + local namespace="${3}" - kubectl exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-verbose.log &>/dev/null + kubectl --context="${context}" exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-verbose.log &>/dev/null } # dump_pod_logs dumps agent and mongodb logs. dump_pod_logs() { - local pod="${1}" - local namespace="${2}" - local prefix="${3}" + local context="${1}" + local pod="${2}" + local namespace="${3}" + local prefix="${4}" - if is_mdb_resource_pod "${pod}" "${namespace}"; then + if is_mdb_resource_pod "${context}" "${pod}" "${namespace}"; then # for MDB resource Pods, we dump the log files from the file system echo "Writing agent and mongodb logs for pod ${pod} to logs" - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent-verbose.log" "logs/${prefix}${pod}-agent-verbose.log" &> /dev/null - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent.log" "logs/${prefix}${pod}-agent.log" &> /dev/null - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent-verbose.log" "logs/${prefix}${pod}-monitoring-agent-verbose.log" &> /dev/null - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent.log" "logs/${prefix}${pod}-monitoring-agent.log" &> /dev/null - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/mongodb.log" "logs/${prefix}${pod}-mongodb.log" &> /dev/null || true + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent-verbose.log" "logs/${prefix}${pod}-agent-verbose.log" &> /dev/null + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent.log" "logs/${prefix}${pod}-agent.log" &> /dev/null + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent-verbose.log" "logs/${prefix}${pod}-monitoring-agent-verbose.log" &> /dev/null + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent.log" "logs/${prefix}${pod}-monitoring-agent.log" &> /dev/null + kubectl --context="${context}" logs -n "${namespace}" "${pod}" -c "mongodb-agent-monitoring" > "logs/${prefix}${pod}-monitoring-agent-stdout.log" || true + kubectl --context="${context}" logs -n "${namespace}" "${pod}" -c "mongod" > "logs/${prefix}${pod}-mongod-container.log" || true + kubectl --context="${context}" logs -n "${namespace}" "${pod}" -c "mongodb-agent" > "logs/${prefix}${pod}-mongodb-agent-container.log" || true + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/mongodb.log" "logs/${prefix}${pod}-mongodb.log" &> /dev/null || true # note that this file may get empty if the logs have already grew too much - seems it's better to have it explicitly empty then just omit - kubectl logs -n "${namespace}" "${pod}" | jq -c -r 'select( .logType == "agent-launcher-script") | .contents' 2> /dev/null > "logs/${prefix}${pod}-launcher.log" + kubectl --context="${context}" logs -n "${namespace}" "${pod}" | jq -c -r 'select( .logType == "agent-launcher-script") | .contents' 2> /dev/null > "logs/${prefix}${pod}-launcher.log" fi - for container in $(kubectl get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do - echo "Writing log file for pod ${pod} - container ${container} to logs/${pod}-${container}.log" - kubectl logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}.log" + for container in $(kubectl --context="${context}" get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do + echo "Writing log file for pod ${pod} - container ${container} to logs/${prefix}${pod}-${container}.log" + kubectl --context="${context}" logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${prefix}${pod}-${container}.log" # Check if the container has restarted by examining its restart count - restartCount=$(kubectl get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}") + restartCount=$(kubectl --context="${context}" get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}") if [ "${restartCount}" -gt 0 ]; then - echo "Writing log file for restarted ${pod} - container ${container} to logs/${pod}-${container}-previous.log" - kubectl logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}-previous.log" || true + echo "Writing log file for restarted ${pod} - container ${container} to logs/${prefix}${pod}-${container}-previous.log" + kubectl --context="${context}" logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${prefix}${pod}-${container}-previous.log" || true fi done - - if kubectl exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-stderr.log &>/dev/null; then - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent-stderr.log" "logs/${prefix}${pod}-agent-stderr.log" &> /dev/null + if kubectl --context="${context}" exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-stderr.log &>/dev/null; then + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent-stderr.log" "logs/${prefix}${pod}-agent-stderr.log" &> /dev/null fi } # dump_pod_readiness_state dumps readiness and agent-health-status files. dump_pod_readiness_state() { - local pod="${1}" - local namespace="${2}" - local prefix="${3}" + local context="${1}" + local pod="${2}" + local namespace="${3}" + local prefix="${4}" # kubectl cp won't create any files if the file doesn't exist in the container agent_health_status="logs/${prefix}${pod}-agent-health-status.json" echo "Writing agent ${agent_health_status}" - kubectl cp -c "mongodb-agent" "${namespace}/${pod}:/var/log/mongodb-mms-automation/agent-health-status.json" "${agent_health_status}" &> /dev/null + kubectl --context="${context}" cp -c "mongodb-agent" "${namespace}/${pod}:/var/log/mongodb-mms-automation/agent-health-status.json" "${agent_health_status}" &> /dev/null ([[ -f "${agent_health_status}" ]] && jq . < "${agent_health_status}" > tmpfile && mv tmpfile "${agent_health_status}") if [[ ! -f "${agent_health_status}" ]]; then echo "Agent health status not found; trying community health status: " - kubectl cp -c "mongodb-agent" "${namespace}/${pod}:/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" "${agent_health_status}" &> /dev/null + kubectl --context="${context}" cp -c "mongodb-agent" "${namespace}/${pod}:/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" "${agent_health_status}" &> /dev/null ([[ -f "${agent_health_status}" ]] && jq . < "${agent_health_status}" > tmpfile && mv tmpfile "${agent_health_status}") fi - kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/readiness.log" "logs/${prefix}${pod}-readiness.log" &> /dev/null + kubectl --context="${context}" cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/readiness.log" "logs/${prefix}${pod}-readiness.log" &> /dev/null } # dump_pod_config dumps mongod configuration and cluster-config. dump_pod_config() { - local pod="${1}" - local namespace="${2}" - local prefix="${3}" + local context="${1}" + local pod="${2}" + local namespace="${3}" + local prefix="${4}" # cluster-config.json is a mounted volume and the actual file is located in the "..data" directory pod_cluster_config="logs/${prefix}${pod}-cluster-config.json" - kubectl cp "${namespace}/${pod}:/var/lib/mongodb-automation/..data/cluster-config.json" "${pod_cluster_config}" &> /dev/null + kubectl --context="${context}" cp "${namespace}/${pod}:/var/lib/mongodb-automation/..data/cluster-config.json" "${pod_cluster_config}" &> /dev/null ([[ -f "${pod_cluster_config}" ]] && jq . < "${pod_cluster_config}" > tmpfile && mv tmpfile "${pod_cluster_config}") # Mongodb Configuration - kubectl cp "${namespace}/${pod}:data/automation-mongod.conf" "logs/${prefix}${pod}-automation-mongod.conf" &> /dev/null + kubectl --context="${context}" cp "${namespace}/${pod}:data/automation-mongod.conf" "logs/${prefix}${pod}-automation-mongod.conf" &> /dev/null } dump_configmaps() { - local namespace="${1}" - local prefix="${2}" - kubectl -n "${namespace}" get configmaps -o yaml > "logs/${prefix}z_configmaps.txt" + local context="${1}" + local namespace="${2}" + local prefix="${3}" + kubectl --context="${context}" -n "${namespace}" get configmaps -o yaml > "logs/${prefix}z_configmaps.txt" } decode_secret() { - local secret=${1} - local namespace=${2} + local context=${1} + local secret=${2} + local namespace=${3} - kubectl get secret "${secret}" -o json -n "${namespace}" | jq -r '.data | with_entries(.value |= @base64d)' 2> /dev/null + kubectl --context="${context}" get secret "${secret}" -o json -n "${namespace}" | jq -r '.data | with_entries(.value |= @base64d)' 2> /dev/null } dump_secrets() { - local namespace="${1}" - local prefix="${2}" - for secret in $(kubectl get secrets -n "${namespace}" --no-headers | grep -v service-account-token | grep -v dockerconfigjson | awk '{ print $1 }'); do - decode_secret "${secret}" "${namespace}" > "logs/${prefix}z_secret_${secret}.txt" + local context="${1}" + local namespace="${2}" + local prefix="${3}" + for secret in $(kubectl --context="${context}" get secrets -n "${namespace}" --no-headers | grep -v service-account-token | grep -v dockerconfigjson | awk '{ print $1 }'); do + decode_secret "${context}" "${secret}" "${namespace}" > "logs/${prefix}z_secret_${secret}.txt" done } dump_services() { - local namespace="${1}" - local prefix="${2}" - kubectl -n "${namespace}" get svc -o yaml > "logs/${prefix}z_services.txt" + local context="${1}" + local namespace="${2}" + local prefix="${3}" + kubectl --context="${context}" -n "${namespace}" get svc -o yaml > "logs/${prefix}z_services.txt" } dump_metrics() { - local namespace="${1}" - local operator_pod="${2}" - kubectl exec -it "${operator_pod}" -n "${namespace}" -- curl localhost:8080/metrics > "logs/metrics_${operator_pod}.txt" + local context="${1}" + local namespace="${2}" + local operator_pod="${3}" + local prefix="${4}" + kubectl --context="${context}" exec -it "${operator_pod}" -n "${namespace}" -- curl localhost:8080/metrics > "logs/${prefix}metrics_${operator_pod}.txt" } # dump_pods writes logs for each relevant Pod in the namespace: agent, mongodb # logs, etc. dump_pods() { - local namespace="${1}" - local prefix="${2}" + local context="${1}" + local namespace="${2}" + local prefix="${3}" - pods=$(get_all_pods "${namespace}") + pods=$(get_all_pods "${context}" "${namespace}") # we only have readiness and automationconfig in mdb pods for pod in ${pods}; do - dump_pod_readiness_state "${pod}" "${namespace}" "${prefix}" - dump_pod_config "${pod}" "${namespace}" "${prefix}" + dump_pod_readiness_state "${context}" "${pod}" "${namespace}" "${prefix}" + dump_pod_config "${context}" "${pod}" "${namespace}" "${prefix}" done # for all pods in the namespace we want to have logs and describe output echo "Iterating over pods to dump logs: ${pods}" for pod in ${pods}; do - kubectl describe "pod/${pod}" -n "${namespace}" > "logs/${prefix}${pod}-pod-describe.txt" - dump_pod_logs "${pod}" "${namespace}" "${prefix}" + kubectl --context="${context}" describe "pod/${pod}" -n "${namespace}" > "logs/${prefix}${pod}-pod-describe.txt" + dump_pod_logs "${context}" "${pod}" "${namespace}" "${prefix}" done - if (kubectl get pod -n "${namespace}" -l app.kubernetes.io/name=controller ) &> /dev/null ; then - operator_pod=$(kubectl get pod -n "${namespace}" -l app.kubernetes.io/component=controller --no-headers -o custom-columns=":metadata.name") + if (kubectl --context="${context}" get pod -n "${namespace}" -l app.kubernetes.io/name=controller ) &> /dev/null ; then + operator_pod=$(kubectl --context="${context}" get pod -n "${namespace}" -l app.kubernetes.io/component=controller --no-headers -o custom-columns=":metadata.name") if [ -n "${operator_pod}" ]; then - kubectl describe "pod/${operator_pod}" -n "${namespace}" > "logs/z_${operator_pod}-pod-describe.txt" - dump_metrics "${namespace}" "${operator_pod}" + kubectl --context="${context}" describe "pod/${operator_pod}" -n "${namespace}" > "logs/${prefix}z_${operator_pod}-pod-describe.txt" + dump_metrics "${context}" "${namespace}" "${operator_pod}" "${prefix}" fi fi @@ -257,53 +307,57 @@ dump_pods() { # high density of information; the main objective of this file is to direct you # to a place where to find your problem, not to tell you what the problem is. dump_diagnostics() { - local namespace="${1}" - - dump_objects mongodb "MongoDB Resources" "${namespace}" - dump_objects mongodbcommunity "MongoDBCommunity Resources" "${namespace}" - dump_objects mongodbusers "MongoDBUser Resources" "${namespace}" - dump_objects opsmanagers "MongoDBOpsManager Resources" "${namespace}" - dump_objects mongodbmulticluster "MongoDB Multi Resources" "${namespace}" - dump_objects mongodbsearch "MongoDB Search Resources" "${namespace}" + local context="${1}" + local namespace="${2}" header "All namespace resources" - kubectl get all -n "${namespace}" + kubectl --context="${context}" get all -n "${namespace}" + + dump_objects "${context}" mongodb "MongoDB Resources" "${namespace}" "get -o yaml" + dump_objects "${context}" mongodbcommunity "MongoDBCommunity Resources" "${namespace}" "get -o yaml" + dump_objects "${context}" mongodbusers "MongoDBUser Resources" "${namespace}" "get -o yaml" + dump_objects "${context}" opsmanagers "MongoDBOpsManager Resources" "${namespace}" "get -o yaml" + dump_objects "${context}" mongodbmulticluster "MongoDB Multi Resources" "${namespace}" "get -o yaml" + dump_objects "${context}" mongodbsearch "MongoDB Search Resources" "${namespace}" "get -o yaml" } download_test_results() { - local namespace="${1}" - local test_pod_name="${2:-e2e-test}" + local context="${1}" + local namespace="${2}" + local test_pod_name="${3:-e2e-test}" echo "Downloading test results from ${test_pod_name} pod" # Try to copy from shared volume using the keepalive container - if kubectl cp "${namespace}/${test_pod_name}:/tmp/results/result.suite" "logs/result.suite" -c keepalive 2>/dev/null; then + if kubectl --context="${context}" cp "${namespace}/${test_pod_name}:/tmp/results/result.suite" "logs/result.suite" -c keepalive 2>/dev/null; then echo "Successfully downloaded result.suite from test pod" else echo "Could not find result.suite via direct copy" # Get logs from the test container - kubectl logs -n "${namespace}" "${test_pod_name}" -c e2e-test > "logs/result.suite" 2>/dev/null + kubectl --context="${context}" logs -n "${namespace}" "${test_pod_name}" -c e2e-test > "logs/result.suite" 2>/dev/null fi } # dump_events gets all events from a namespace and saves them to a file dump_events() { - local namespace="${1}" - local prefix="${2}" + local context="${1}" + local namespace="${2}" + local prefix="${3}" echo "Collecting events for namespace ${namespace}" # Sort by lastTimestamp to have the most recent events at the top - kubectl get events --sort-by='.lastTimestamp' -n "${namespace}" > "logs/${prefix}events.txt" + kubectl --context="${context}" get events --sort-by='.lastTimestamp' -n "${namespace}" > "logs/${prefix}events.txt" # Also get events in yaml format for more details - kubectl get events -n "${namespace}" -o yaml > "logs/${prefix}events_detailed.yaml" + kubectl --context="${context}" get events -n "${namespace}" -o yaml > "logs/${prefix}events_detailed.yaml" } # dump_namespace dumps a namespace, diagnostics, logs and generic Kubernetes # resources. dump_namespace() { - local namespace=${1} - local prefix="${2}" + local context=${1} + local namespace=${2} + local prefix="${3}_${namespace}_" # do not fail for any reason set +e @@ -313,42 +367,42 @@ dump_namespace() { mkdir -p logs # 2. Write diagnostics file - dump_diagnostics "${namespace}" > "logs/${prefix}0_diagnostics.txt" + dump_diagnostics "${context}" "${namespace}" > "logs/${prefix}0_diagnostics.txt" # 3. Print Pod logs - dump_pods "${namespace}" "${prefix}" + dump_pods "${context}" "${namespace}" "${prefix}" # 4. Print other Kubernetes resources - dump_configmaps "${namespace}" "${prefix}" - dump_secrets "${namespace}" "${prefix}" - dump_services "${namespace}" "${prefix}" - dump_events "${namespace}" "${prefix}" + dump_configmaps "${context}" "${namespace}" "${prefix}" + dump_secrets "${context}" "${namespace}" "${prefix}" + dump_services "${context}" "${namespace}" "${prefix}" + dump_events "${context}" "${namespace}" "${prefix}" # Download test results from the test pod in community - download_test_results "${namespace}" "e2e-test" - - dump_objects pvc "Persistent Volume Claims" "${namespace}" > "logs/${prefix}z_persistent_volume_claims.txt" - dump_objects deploy "Deployments" "${namespace}" > "logs/${prefix}z_deployments.txt" - dump_objects deploy "Deployments" "${namespace}" "describe" > "logs/${prefix}z_deployments_describe.txt" - dump_objects sts "StatefulSets" "${namespace}" describe > "logs/${prefix}z_statefulsets.txt" - dump_objects sts "StatefulSets Yaml" "${namespace}" >> "logs/${prefix}z_statefulsets.txt" - dump_objects serviceaccounts "ServiceAccounts" "${namespace}" > "logs/${prefix}z_service_accounts.txt" - dump_objects clusterrolebindings "ClusterRoleBindings" "${namespace}" > "logs/${prefix}z_clusterrolebindings.txt" - dump_objects clusterroles "ClusterRoles" "${namespace}" > "logs/${prefix}z_clusterroles.txt" - dump_objects rolebindings "RoleBindings" "${namespace}" > "logs/${prefix}z_rolebindings.txt" - dump_objects roles "Roles" "${namespace}" > "logs/${prefix}z_roles.txt" - dump_objects validatingwebhookconfigurations "Validating Webhook Configurations" "${namespace}" > "logs/${prefix}z_validatingwebhookconfigurations.txt" - dump_objects certificates.cert-manager.io "Cert-manager certificates" "${namespace}" 2> /dev/null > "logs/${prefix}z_certificates_certmanager.txt" - dump_objects catalogsources "OLM CatalogSources" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_catalogsources.txt" - dump_objects operatorgroups "OLM OperatorGroups" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_operatorgroups.txt" - dump_objects subscriptions "OLM Subscriptions" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_subscriptions.txt" - dump_objects installplans "OLM InstallPlans" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_installplans.txt" - dump_objects clusterserviceversions "OLM ClusterServiceVersions" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_clusterserviceversions.txt" - dump_objects pods "Pods" "${namespace}" 2> /dev/null > "logs/${prefix}z_pods.txt" - - kubectl get crd -o name + download_test_results "${context}" "${namespace}" "e2e-test" + + dump_objects "${context}" pvc "Persistent Volume Claims" "${namespace}" "get -o yaml" "logs/${prefix}z_persistent_volume_claims.txt" + dump_objects "${context}" deploy "Deployments" "${namespace}" "get -o yaml" "logs/${prefix}z_deployments.txt" + dump_objects "${context}" deploy "Deployments" "${namespace}" "describe" "logs/${prefix}z_deployments_describe.txt" + dump_objects "${context}" sts "StatefulSets" "${namespace}" "describe" "logs/${prefix}z_statefulsets.txt" + dump_objects "${context}" sts "StatefulSets Yaml" "${namespace}" "get -o yaml" "logs/${prefix}z_statefulsets.txt" + dump_objects "${context}" serviceaccounts "ServiceAccounts" "${namespace}" "get -o yaml" "logs/${prefix}z_service_accounts.txt" + dump_objects "${context}" clusterrolebindings "ClusterRoleBindings" "${namespace}" "get -o yaml" "logs/${prefix}z_clusterrolebindings.txt" + dump_objects "${context}" clusterroles "ClusterRoles" "${namespace}" "get -o yaml" "logs/${prefix}z_clusterroles.txt" + dump_objects "${context}" rolebindings "RoleBindings" "${namespace}" "get -o yaml" "logs/${prefix}z_rolebindings.txt" + dump_objects "${context}" roles "Roles" "${namespace}" "get -o yaml" "logs/${prefix}z_roles.txt" + dump_objects "${context}" validatingwebhookconfigurations "Validating Webhook Configurations" "${namespace}" "get -o yaml" "logs/${prefix}z_validatingwebhookconfigurations.txt" + dump_objects "${context}" certificates.cert-manager.io "Cert-manager certificates" "${namespace}" "get -o yaml" "logs/${prefix}z_certificates_certmanager.txt" 2> /dev/null + dump_objects "${context}" catalogsources "OLM CatalogSources" "${namespace}" "get -o yaml" "logs/${prefix}z_olm_catalogsources.txt" 2> /dev/null + dump_objects "${context}" operatorgroups "OLM OperatorGroups" "${namespace}" "get -o yaml" "logs/${prefix}z_olm_operatorgroups.txt" 2> /dev/null + dump_objects "${context}" subscriptions "OLM Subscriptions" "${namespace}" "get -o yaml" "logs/${prefix}z_olm_subscriptions.txt" 2> /dev/null + dump_objects "${context}" installplans "OLM InstallPlans" "${namespace}" "get -o yaml" "logs/${prefix}z_olm_installplans.txt" 2> /dev/null + dump_objects "${context}" clusterserviceversions "OLM ClusterServiceVersions" "${namespace}" "get -o yaml" "logs/${prefix}z_olm_clusterserviceversions.txt" 2> /dev/null + dump_objects "${context}" pods "Pods" "${namespace}" "get -o yaml" "logs/${prefix}z_pods.txt" 2> /dev/null + + kubectl --context="${context}" get crd -o name # shellcheck disable=SC2046 - kubectl describe $(kubectl get crd -o name | grep mongodb) > "logs/${prefix}z_mongodb_crds.log" + kubectl --context="${context}" describe $(kubectl --context="${context}" get crd -o name | grep mongodb) > "logs/${prefix}z_mongodb_crds.log" - kubectl describe nodes > "logs/${prefix}z_nodes_detailed.log" || true + kubectl --context="${context}" describe nodes > "logs/${prefix}z_nodes_detailed.log" || true } diff --git a/scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh b/scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh index 72571a811..59a41f19f 100755 --- a/scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh +++ b/scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh @@ -7,4 +7,9 @@ set +e source scripts/funcs/printing source scripts/evergreen/e2e/dump_diagnostic_information.sh -dump_all_non_default_namespaces "$@" +# If no context provided, use current context +if [ $# -eq 0 ]; then + dump_all_non_default_namespaces "$(kubectl config current-context)" +else + dump_all_non_default_namespaces "$@" +fi diff --git a/scripts/evergreen/e2e/e2e.sh b/scripts/evergreen/e2e/e2e.sh index 9d6e8de83..e1e6ed556 100755 --- a/scripts/evergreen/e2e/e2e.sh +++ b/scripts/evergreen/e2e/e2e.sh @@ -42,7 +42,7 @@ dump_cluster_information() { done else # Dump all the information we can from this namespace - dump_all || true + dump_all "$(kubectl config current-context)" || true fi } From b5d2cb151146a14b327fec46dd1da423de57a060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 8 Sep 2025 22:36:19 +0200 Subject: [PATCH 02/17] CLOUDP-312564: MongoDB Search - Public Preview Co-authored-by: Anand Singh Co-authored-by: Yavor Georgiev --- .evergreen-release.yml | 13 - .evergreen-tasks.yml | 26 ++ .evergreen.yml | 6 + api/v1/search/mongodbsearch_types.go | 99 ++++- api/v1/search/zz_generated.deepcopy.go | 98 +++++ .../mongodb.com_clustermongodbroles.yaml | 2 +- config/crd/bases/mongodb.com_mongodb.yaml | 48 +-- .../mongodb.com_mongodbmulticluster.yaml | 48 +-- .../crd/bases/mongodb.com_mongodbsearch.yaml | 130 ++++--- .../crd/bases/mongodb.com_mongodbusers.yaml | 2 +- config/crd/bases/mongodb.com_opsmanagers.yaml | 112 +----- ...ommunity.mongodb.com_mongodbcommunity.yaml | 54 +-- .../operator/mongodbreplicaset_controller.go | 100 ++++- .../operator/mongodbsearch_controller.go | 80 +++- .../operator/mongodbsearch_controller_test.go | 153 ++++++-- .../operator/watch/config_change_handler.go | 12 +- .../community_search_source.go | 84 +++++ .../community_search_source_test.go | 208 +++++++++++ .../enterprise_search_source.go | 88 +++++ .../enterprise_search_source_test.go | 290 +++++++++++++++ .../external_search_source.go | 49 +++ .../mongodbsearch_reconcile_helper.go | 347 +++++++++++++++--- .../mongodbsearch_reconcile_helper_test.go | 62 +--- .../search_controller/search_construction.go | 136 +++---- .../edit_mms_configuration.go | 4 +- .../kubetester/helm.py | 3 + .../kubetester/mongodb.py | 6 +- .../common/search/movies_search_helper.py | 42 ++- .../tests/common/search/search_tester.py | 14 +- ...nity-replicaset-sample-mflix-external.yaml | 112 ++++++ .../community-replicaset-sample-mflix.yaml | 69 ++-- .../enterprise-replicaset-sample-mflix.yaml | 34 ++ .../fixtures/mongodbuser-mdb-admin.yaml | 16 + .../search/fixtures/mongodbuser-mdb-user.yaml | 16 + .../mongodbuser-search-sync-source-user.yaml | 18 + .../fixtures/search-with-user-password.yaml | 5 + .../tests/search/search_community_basic.py | 42 ++- .../search_community_external_mongod_basic.py | 144 ++++++++ .../search_community_external_mongod_tls.py | 190 ++++++++++ .../tests/search/search_community_tls.py | 173 +++++++++ .../tests/search/search_enterprise_basic.py | 183 +++++++++ .../tests/search/search_enterprise_tls.py | 239 ++++++++++++ .../crds/mongodb.com_mongodbsearch.yaml | 107 +++++- .../controllers/replica_set_controller.go | 76 +++- .../pkg/mongot/mongot_config.go | 89 +++-- mongodb-community-operator/pkg/tls/tls.go | 113 ++++++ pkg/telemetry/collector.go | 52 ++- pkg/telemetry/collector_test.go | 113 ++++-- public/crds.yaml | 107 +++++- scripts/dev/contexts/e2e_mdb_community | 2 + scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa | 2 + .../contexts/e2e_static_mdb_kind_ubi_cloudqa | 2 + scripts/dev/contexts/evg-private-context | 4 +- scripts/dev/contexts/root-context | 8 +- .../dev/contexts/variables/mongodb_search_dev | 8 + scripts/dev/setup_kind_cluster.sh | 46 +-- scripts/funcs/operator_deployment | 11 +- 57 files changed, 3566 insertions(+), 731 deletions(-) create mode 100644 controllers/search_controller/community_search_source.go create mode 100644 controllers/search_controller/community_search_source_test.go create mode 100644 controllers/search_controller/enterprise_search_source.go create mode 100644 controllers/search_controller/enterprise_search_source_test.go create mode 100644 controllers/search_controller/external_search_source.go create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py create mode 100644 mongodb-community-operator/pkg/tls/tls.go create mode 100644 scripts/dev/contexts/variables/mongodb_search_dev diff --git a/.evergreen-release.yml b/.evergreen-release.yml index f7f7b9ac0..b53288912 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -151,19 +151,6 @@ buildvariants: tasks: - name: run_conditionally_prepare_and_upload_openshift_bundles - - name: prerelease_gke_code_snippets - display_name: prerelease_gke_code_snippets - tags: [ "release" ] - allowed_requesters: ["patch", "github_tag"] - depends_on: - - variant: release_images - name: '*' - patch_optional: true - run_on: - - ubuntu2404-small - tasks: - - name: gke_code_snippets_task_group - - name: init_smoke_tests display_name: init_smoke_tests tags: [ "e2e_smoke_release_test_suite" ] diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml index 9b6cfd893..6ec158a41 100644 --- a/.evergreen-tasks.yml +++ b/.evergreen-tasks.yml @@ -1288,3 +1288,29 @@ tasks: tags: ["patch-run"] commands: - func: "e2e_test" + + - name: e2e_search_community_tls + tags: ["patch-run"] + commands: + - func: "e2e_test" + + - name: e2e_search_external_basic + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_search_external_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_search_enterprise_basic + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_search_enterprise_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + diff --git a/.evergreen.yml b/.evergreen.yml index d77ff676e..cde2a1674 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -601,6 +601,9 @@ task_groups: tasks: - e2e_community_replicaset_scale - e2e_search_community_basic + - e2e_search_community_tls + - e2e_search_external_basic + - e2e_search_external_tls # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant - name: e2e_mdb_kind_cloudqa_task_group @@ -725,6 +728,9 @@ task_groups: - e2e_replica_set_oidc_workforce - e2e_sharded_cluster_oidc_m2m_group - e2e_sharded_cluster_oidc_m2m_user + # MongoDBSearch test group + - e2e_search_enterprise_basic + - e2e_search_enterprise_tls <<: *teardown_group # this task group contains just a one task, which is smoke testing whether the operator diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index 43e9be1be..5a8037b51 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -1,6 +1,8 @@ package search import ( + "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -14,8 +16,10 @@ import ( ) const ( - MongotDefaultPort = 27027 - MongotDefaultMetricsPort = 9946 + MongotDefaultPort = 27027 + MongotDefaultMetricsPort = 9946 + MongotDefautHealthCheckPort = 8080 + MongotDefaultSyncSourceUsername = "search-sync-source" ) func init() { @@ -23,21 +27,66 @@ func init() { } type MongoDBSearchSpec struct { + // Optional version of MongoDB Search component (mongot). If not set, then the operator will set the most appropriate version of MongoDB Search. // +optional Version string `json:"version"` + // MongoDB database connection details from which MongoDB Search will synchronize data to build indexes. // +optional Source *MongoDBSource `json:"source"` + // StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + // which aren't exposed as fields in the MongoDBSearch.spec. // +optional StatefulSetConfiguration *common.StatefulSetConfiguration `json:"statefulSet,omitempty"` + // Configure MongoDB Search's persistent volume. If not defined, the operator will request 10GB of storage. // +optional Persistence *common.Persistence `json:"persistence,omitempty"` + // Configure resource requests and limits for the MongoDB Search pods. // +optional ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"` + // Configure security settings of the MongoDB Search server that MongoDB database is connecting to when performing search queries. + // +optional + Security Security `json:"security"` } type MongoDBSource struct { // +optional MongoDBResourceRef *userv1.MongoDBResourceRef `json:"mongodbResourceRef,omitempty"` + // +optional + ExternalMongoDBSource *ExternalMongoDBSource `json:"external,omitempty"` + // +optional + PasswordSecretRef *userv1.SecretKeyRef `json:"passwordSecretRef,omitempty"` + // +optional + Username *string `json:"username,omitempty"` +} + +type ExternalMongoDBSource struct { + HostAndPorts []string `json:"hostAndPorts,omitempty"` + // mongod keyfile used to connect to the external MongoDB deployment + KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyFileSecretRef,omitempty"` + // +optional + TLS *ExternalMongodTLS `json:"tls,omitempty"` // TLS configuration for the external MongoDB deployment +} + +type ExternalMongodTLS struct { + Enabled bool `json:"enabled"` + // +optional + CA *corev1.LocalObjectReference `json:"ca,omitempty"` +} + +type Security struct { + // +optional + TLS TLS `json:"tls"` +} + +type TLS struct { + Enabled bool `json:"enabled"` + // CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + // The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + // This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + // Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + // If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + // +optional + CertificateKeySecret corev1.LocalObjectReference `json:"certificateKeySecretRef"` } type MongoDBSearchStatus struct { @@ -105,6 +154,25 @@ func (s *MongoDBSearch) MongotConfigConfigMapNamespacedName() types.NamespacedNa return types.NamespacedName{Name: s.Name + "-search-config", Namespace: s.Namespace} } +func (s *MongoDBSearch) SourceUserPasswordSecretRef() *userv1.SecretKeyRef { + if s.Spec.Source != nil && s.Spec.Source.PasswordSecretRef != nil { + return s.Spec.Source.PasswordSecretRef + } + + return &userv1.SecretKeyRef{ + Name: fmt.Sprintf("%s-%s-password", s.Name, MongotDefaultSyncSourceUsername), + Key: "password", + } +} + +func (s *MongoDBSearch) SourceUsername() string { + if s.Spec.Source != nil && s.Spec.Source.Username != nil { + return *s.Spec.Source.Username + } + + return MongotDefaultSyncSourceUsername +} + func (s *MongoDBSearch) StatefulSetNamespacedName() types.NamespacedName { return types.NamespacedName{Name: s.Name + "-search", Namespace: s.Namespace} } @@ -118,13 +186,17 @@ func (s *MongoDBSearch) GetOwnerReferences() []metav1.OwnerReference { return []metav1.OwnerReference{ownerReference} } -func (s *MongoDBSearch) GetMongoDBResourceRef() userv1.MongoDBResourceRef { +func (s *MongoDBSearch) GetMongoDBResourceRef() *userv1.MongoDBResourceRef { + if s.IsExternalMongoDBSource() { + return nil + } + mdbResourceRef := userv1.MongoDBResourceRef{Namespace: s.Namespace, Name: s.Name} if s.Spec.Source != nil && s.Spec.Source.MongoDBResourceRef != nil && s.Spec.Source.MongoDBResourceRef.Name != "" { mdbResourceRef.Name = s.Spec.Source.MongoDBResourceRef.Name } - return mdbResourceRef + return &mdbResourceRef } func (s *MongoDBSearch) GetMongotPort() int32 { @@ -134,3 +206,22 @@ func (s *MongoDBSearch) GetMongotPort() int32 { func (s *MongoDBSearch) GetMongotMetricsPort() int32 { return MongotDefaultMetricsPort } + +// TLSSecretNamespacedName will get the namespaced name of the Secret containing the server certificate and key +func (s *MongoDBSearch) TLSSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: s.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: s.Namespace} +} + +// TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator +// containing the combined certificate and key. +func (s *MongoDBSearch) TLSOperatorSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: s.Name + "-search-certificate-key", Namespace: s.Namespace} +} + +func (s *MongoDBSearch) GetMongotHealthCheckPort() int32 { + return MongotDefautHealthCheckPort +} + +func (s *MongoDBSearch) IsExternalMongoDBSource() bool { + return s.Spec.Source != nil && s.Spec.Source.ExternalMongoDBSource != nil +} diff --git a/api/v1/search/zz_generated.deepcopy.go b/api/v1/search/zz_generated.deepcopy.go index e9384e3de..c66322146 100644 --- a/api/v1/search/zz_generated.deepcopy.go +++ b/api/v1/search/zz_generated.deepcopy.go @@ -28,6 +28,56 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalMongoDBSource) DeepCopyInto(out *ExternalMongoDBSource) { + *out = *in + if in.HostAndPorts != nil { + in, out := &in.HostAndPorts, &out.HostAndPorts + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.KeyFileSecretKeyRef != nil { + in, out := &in.KeyFileSecretKeyRef, &out.KeyFileSecretKeyRef + *out = new(user.SecretKeyRef) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(ExternalMongodTLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalMongoDBSource. +func (in *ExternalMongoDBSource) DeepCopy() *ExternalMongoDBSource { + if in == nil { + return nil + } + out := new(ExternalMongoDBSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalMongodTLS) DeepCopyInto(out *ExternalMongodTLS) { + *out = *in + if in.CA != nil { + in, out := &in.CA, &out.CA + *out = new(v1.LocalObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalMongodTLS. +func (in *ExternalMongodTLS) DeepCopy() *ExternalMongodTLS { + if in == nil { + return nil + } + out := new(ExternalMongodTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MongoDBSearch) DeepCopyInto(out *MongoDBSearch) { *out = *in @@ -109,6 +159,7 @@ func (in *MongoDBSearchSpec) DeepCopyInto(out *MongoDBSearchSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + out.Security = in.Security } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBSearchSpec. @@ -150,6 +201,21 @@ func (in *MongoDBSource) DeepCopyInto(out *MongoDBSource) { *out = new(user.MongoDBResourceRef) **out = **in } + if in.ExternalMongoDBSource != nil { + in, out := &in.ExternalMongoDBSource, &out.ExternalMongoDBSource + *out = new(ExternalMongoDBSource) + (*in).DeepCopyInto(*out) + } + if in.PasswordSecretRef != nil { + in, out := &in.PasswordSecretRef, &out.PasswordSecretRef + *out = new(user.SecretKeyRef) + **out = **in + } + if in.Username != nil { + in, out := &in.Username, &out.Username + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBSource. @@ -161,3 +227,35 @@ func (in *MongoDBSource) DeepCopy() *MongoDBSource { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Security) DeepCopyInto(out *Security) { + *out = *in + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Security. +func (in *Security) DeepCopy() *Security { + if in == nil { + return nil + } + out := new(Security) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLS) DeepCopyInto(out *TLS) { + *out = *in + out.CertificateKeySecret = in.CertificateKeySecret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLS. +func (in *TLS) DeepCopy() *TLS { + if in == nil { + return nil + } + out := new(TLS) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/mongodb.com_clustermongodbroles.yaml b/config/crd/bases/mongodb.com_clustermongodbroles.yaml index 3d583bcfd..9241b7dad 100644 --- a/config/crd/bases/mongodb.com_clustermongodbroles.yaml +++ b/config/crd/bases/mongodb.com_clustermongodbroles.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: clustermongodbroles.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_mongodb.yaml b/config/crd/bases/mongodb.com_mongodb.yaml index d421d8837..2a7076877 100644 --- a/config/crd/bases/mongodb.com_mongodb.yaml +++ b/config/crd/bases/mongodb.com_mongodb.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodb.mongodb.com spec: group: mongodb.com @@ -1410,29 +1410,7 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' automationUserName: type: string clientCertificateSecretRef: @@ -1467,29 +1445,9 @@ spec: bindQueryUser: type: string caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml index 3f1fa05c9..01fe3f2e6 100644 --- a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml +++ b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodbmulticluster.mongodb.com spec: group: mongodb.com @@ -670,29 +670,7 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' automationUserName: type: string clientCertificateSecretRef: @@ -727,29 +705,9 @@ spec: bindQueryUser: type: string caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbsearch.yaml b/config/crd/bases/mongodb.com_mongodbsearch.yaml index 72ea0e50e..7c53c195c 100644 --- a/config/crd/bases/mongodb.com_mongodbsearch.yaml +++ b/config/crd/bases/mongodb.com_mongodbsearch.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodbsearch.mongodb.com spec: group: mongodb.com @@ -49,6 +49,8 @@ spec: spec: properties: persistence: + description: Configure MongoDB Search's persistent volume. If not + defined, the operator will request 10GB of storage. properties: multiple: properties: @@ -95,66 +97,60 @@ spec: type: object type: object resourceRequirements: - description: ResourceRequirements describes the compute resource requirements. + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ResourceRequirements' + description: Configure resource requests and limits for the MongoDB + Search pods. + security: + description: Configure security settings of the MongoDB Search server + that MongoDB database is connecting to when performing search queries. properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + tls: + properties: + certificateKeySecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + enabled: + type: boolean + required: + - enabled type: object type: object source: + description: MongoDB database connection details from which MongoDB + Search will synchronize data to build indexes. properties: + external: + properties: + hostAndPorts: + items: + type: string + type: array + keyFileSecretRef: + description: mongod keyfile used to connect to the external + MongoDB deployment + properties: + key: + type: string + name: + type: string + required: + - name + type: object + tls: + properties: + ca: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' + enabled: + type: boolean + required: + - enabled + type: object + type: object mongodbResourceRef: properties: name: @@ -164,11 +160,26 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. + StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + which aren't exposed as fields in the MongoDBSearch.spec. properties: metadata: description: StatefulSetMetadataWrapper is a wrapper around Labels @@ -190,6 +201,9 @@ spec: - spec type: object version: + description: Optional version of MongoDB Search component (mongot). + If not set, then the operator will set the most appropriate version + of MongoDB Search. type: string type: object status: diff --git a/config/crd/bases/mongodb.com_mongodbusers.yaml b/config/crd/bases/mongodb.com_mongodbusers.yaml index 89713ce7f..a81f0d449 100644 --- a/config/crd/bases/mongodb.com_mongodbusers.yaml +++ b/config/crd/bases/mongodb.com_mongodbusers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodbusers.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_opsmanagers.yaml b/config/crd/bases/mongodb.com_opsmanagers.yaml index 3ace001da..c830b9a24 100644 --- a/config/crd/bases/mongodb.com_opsmanagers.yaml +++ b/config/crd/bases/mongodb.com_opsmanagers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: opsmanagers.mongodb.com spec: group: mongodb.com @@ -730,30 +730,7 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - description: SecretKeySelector selects a key of a - Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' automationUserName: type: string clientCertificateSecretRef: @@ -789,29 +766,9 @@ spec: bindQueryUser: type: string caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic servers: items: type: string @@ -1236,29 +1193,7 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' type: array irsaEnabled: description: |- @@ -1328,29 +1263,7 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' type: array irsaEnabled: description: |- @@ -1495,8 +1408,6 @@ spec: required: - spec type: object - required: - - members type: object clusterDomain: description: Cluster domain to override the default *.svc.cluster.local @@ -1536,13 +1447,13 @@ spec: Service when creating a ClusterIP type Service type: string externalTrafficPolicy: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local - type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. @@ -1553,12 +1464,12 @@ spec: format: int32 type: integer type: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP - type: string required: - type type: object @@ -1600,7 +1511,6 @@ spec: - spec type: object required: - - clusterName - members type: object type: array @@ -1626,13 +1536,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local - type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1642,12 +1552,12 @@ spec: format: int32 type: integer type: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP - type: string required: - type type: object @@ -1667,13 +1577,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local - type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1683,12 +1593,12 @@ spec: format: int32 type: integer type: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP - type: string required: - type type: object diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 36d5c892d..a0004e22e 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 service.binding: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret service.binding/connectionString: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv service.binding/password: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password @@ -330,24 +330,13 @@ spec: authentication: properties: agentCertificateSecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- AgentCertificateSecret is a reference to a Secret containing the certificate and the key for the automation agent The secret needs to have available: - certificate under key: "tls.crt" - private key under key: "tls.key" If additionally, tls.pem is present, then it needs to be equal to the concatenation of tls.crt and tls.key - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic agentMode: description: AgentMode contains the authentication mode used by the automation agent. @@ -466,57 +455,24 @@ spec: communication properties: caCertificateSecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" This field is ignored when CaCertificateSecretRef is configured - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic certificateKeySecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic enabled: type: boolean optional: diff --git a/controllers/operator/mongodbreplicaset_controller.go b/controllers/operator/mongodbreplicaset_controller.go index bf6b2d2b7..fad1ec6b4 100644 --- a/controllers/operator/mongodbreplicaset_controller.go +++ b/controllers/operator/mongodbreplicaset_controller.go @@ -4,12 +4,16 @@ import ( "context" "fmt" + "github.com/blang/semver" "go.uber.org/zap" "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -22,6 +26,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" rolev1 "github.com/mongodb/mongodb-kubernetes/api/v1/role" + searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" mdbstatus "github.com/mongodb/mongodb-kubernetes/api/v1/status" "github.com/mongodb/mongodb-kubernetes/controllers/om" "github.com/mongodb/mongodb-kubernetes/controllers/om/backup" @@ -39,6 +44,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/controllers/operator/recovery" "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" + "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" mcoConstruct "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/annotations" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/configmap" @@ -52,6 +58,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/pkg/util/architectures" "github.com/mongodb/mongodb-kubernetes/pkg/util/env" util_int "github.com/mongodb/mongodb-kubernetes/pkg/util/int" + "github.com/mongodb/mongodb-kubernetes/pkg/util/maputil" "github.com/mongodb/mongodb-kubernetes/pkg/vault" "github.com/mongodb/mongodb-kubernetes/pkg/vault/vaultwatcher" ) @@ -222,6 +229,8 @@ func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reco return r.updateStatus(ctx, rs, workflow.Failed(xerrors.Errorf("Failed to reconcileHostnameOverrideConfigMap: %w", err)), log) } + shouldMirrorKeyfile := r.applySearchOverrides(ctx, rs, log) + sts := construct.DatabaseStatefulSet(*rs, rsConfig, log) if status := r.ensureRoles(ctx, rs.Spec.DbCommonSpec, r.enableClusterMongoDBRoles, conn, kube.ObjectKeyFromApiObject(rs), log); !status.IsOK() { return r.updateStatus(ctx, rs, status, log) @@ -251,7 +260,7 @@ func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reco // See CLOUDP-189433 and CLOUDP-229222 for more details. if recovery.ShouldTriggerRecovery(rs.Status.Phase != mdbstatus.PhaseRunning, rs.Status.LastTransition) { log.Warnf("Triggering Automatic Recovery. The MongoDB resource %s/%s is in %s state since %s", rs.Namespace, rs.Name, rs.Status.Phase, rs.Status.LastTransition) - automationConfigStatus := r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, true).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + automationConfigStatus := r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, true, shouldMirrorKeyfile).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") deploymentError := create.DatabaseInKubernetes(ctx, r.client, *rs, sts, rsConfig, log) if deploymentError != nil { log.Errorf("Recovery failed because of deployment errors, %w", deploymentError) @@ -267,7 +276,7 @@ func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reco } status = workflow.RunInGivenOrder(publishAutomationConfigFirst(ctx, r.client, *rs, lastSpec, rsConfig, log), func() workflow.Status { - return r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, false).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + return r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, false, shouldMirrorKeyfile).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") }, func() workflow.Status { workflowStatus := create.HandlePVCResize(ctx, r.client, &sts, log) @@ -421,6 +430,19 @@ func AddReplicaSetController(ctx context.Context, mgr manager.Manager, imageUrls zap.S().Errorf("Failed to watch for vault secret changes: %w", err) } } + + err = c.Watch(source.Kind(mgr.GetCache(), &searchv1.MongoDBSearch{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, search *searchv1.MongoDBSearch) []reconcile.Request { + source := search.GetMongoDBResourceRef() + if source == nil { + return []reconcile.Request{} + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: source.Namespace, Name: source.Name}}} + }))) + if err != nil { + return err + } + zap.S().Infof("Registered controller %s", util.MongoDbReplicaSetController) return nil @@ -428,7 +450,7 @@ func AddReplicaSetController(ctx context.Context, mgr manager.Manager, imageUrls // updateOmDeploymentRs performs OM registration operation for the replicaset. So the changes will be finally propagated // to automation agents in containers -func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, conn om.Connection, membersNumberBefore int, rs *mdbv1.MongoDB, set appsv1.StatefulSet, log *zap.SugaredLogger, caFilePath, tlsCertPath, internalClusterCertPath string, agentCertSecretSelector corev1.SecretKeySelector, prometheusCertHash string, isRecovering bool) workflow.Status { +func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, conn om.Connection, membersNumberBefore int, rs *mdbv1.MongoDB, set appsv1.StatefulSet, log *zap.SugaredLogger, caFilePath, tlsCertPath, internalClusterCertPath string, agentCertSecretSelector corev1.SecretKeySelector, prometheusCertHash string, isRecovering bool, shouldMirrorKeyfileForMongot bool) workflow.Status { log.Debug("Entering UpdateOMDeployments") // Only "concrete" RS members should be observed // - if scaling down, let's observe only members that will remain after scale-down operation @@ -477,6 +499,11 @@ func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, c err = conn.ReadUpdateDeployment( func(d om.Deployment) error { + if shouldMirrorKeyfileForMongot { + if err := r.mirrorKeyfileIntoSecretForMongot(ctx, d, rs, log); err != nil { + return err + } + } return ReconcileReplicaSetAC(ctx, d, rs.Spec.DbCommonSpec, lastRsConfig.ToMap(), rs.Name, replicaSet, caFilePath, internalClusterCertPath, &p, log) }, log, @@ -617,3 +644,70 @@ func getAllHostsRs(set appsv1.StatefulSet, clusterName string, membersCount int, hostnames, _ := dns.GetDnsForStatefulSetReplicasSpecified(set, clusterName, membersCount, externalDomain) return hostnames } + +func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, rs *mdbv1.MongoDB, log *zap.SugaredLogger) bool { + search := r.lookupCorrespondingSearchResource(ctx, rs, log) + if search == nil { + log.Debugf("No MongoDBSearch resource found, skipping search overrides") + return false + } + + log.Infof("Applying search overrides from MongoDBSearch %s", search.NamespacedName()) + + if rs.Spec.AdditionalMongodConfig == nil { + rs.Spec.AdditionalMongodConfig = mdbv1.NewEmptyAdditionalMongodConfig() + } + searchMongodConfig := search_controller.GetMongodConfigParameters(search) + rs.Spec.AdditionalMongodConfig.AddOption("setParameter", searchMongodConfig["setParameter"]) + + mdbVersion, err := semver.ParseTolerant(rs.Spec.Version) + if err != nil { + log.Warnf("Failed to parse MongoDB version %q: %w. Proceeding without the automatic creation of the searchCoordinator role that's necessary for MongoDB <8.2", rs.Spec.Version, err) + } else if semver.MustParse("8.2.0").GT(mdbVersion) { + log.Infof("Polyfilling the searchCoordinator role for MongoDB %s", rs.Spec.Version) + + if rs.Spec.Security == nil { + rs.Spec.Security = &mdbv1.Security{} + } + rs.Spec.Security.Roles = append(rs.Spec.Security.Roles, search_controller.SearchCoordinatorRole()) + } + + return true +} + +func (r *ReconcileMongoDbReplicaSet) mirrorKeyfileIntoSecretForMongot(ctx context.Context, d om.Deployment, rs *mdbv1.MongoDB, log *zap.SugaredLogger) error { + keyfileContents := maputil.ReadMapValueAsString(d, "auth", "key") + keyfileSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-keyfile", rs.Name), Namespace: rs.Namespace}} + + log.Infof("Mirroring the replicaset %s's keyfile into the secret %s", rs.ObjectKey(), kube.ObjectKeyFromApiObject(keyfileSecret)) + + _, err := controllerutil.CreateOrUpdate(ctx, r.client, keyfileSecret, func() error { + keyfileSecret.StringData = map[string]string{"keyfile": keyfileContents} + return controllerutil.SetOwnerReference(rs, keyfileSecret, r.client.Scheme()) + }) + if err != nil { + return xerrors.Errorf("Failed to mirror the replicaset's keyfile into a secret: %w", err) + } else { + return nil + } +} + +func (r *ReconcileMongoDbReplicaSet) lookupCorrespondingSearchResource(ctx context.Context, rs *mdbv1.MongoDB, log *zap.SugaredLogger) *searchv1.MongoDBSearch { + var search *searchv1.MongoDBSearch + searchList := &searchv1.MongoDBSearchList{} + if err := r.client.List(ctx, searchList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(search_controller.MongoDBSearchIndexFieldName, rs.GetNamespace()+"/"+rs.GetName()), + }); err != nil { + log.Debugf("Failed to list MongoDBSearch resources: %v", err) + } + // this validates that there is exactly one MongoDBSearch pointing to this resource, + // and that this resource passes search validations. If either fails, proceed without a search target + // for the mongod automation config. + if len(searchList.Items) == 1 { + searchSource := search_controller.NewEnterpriseResourceSearchSource(rs) + if searchSource.Validate() == nil { + search = &searchList.Items[0] + } + } + return search +} diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index 63ed00a5f..a25efed3c 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -13,12 +13,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" - "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/watch" kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" "github.com/mongodb/mongodb-kubernetes/pkg/kube/commoncontroller" "github.com/mongodb/mongodb-kubernetes/pkg/util" @@ -27,15 +30,14 @@ import ( type MongoDBSearchReconciler struct { kubeClient kubernetesClient.Client - mdbcWatcher *watch.ResourceWatcher + watch *watch.ResourceWatcher operatorSearchConfig search_controller.OperatorSearchConfig } func newMongoDBSearchReconciler(client client.Client, operatorSearchConfig search_controller.OperatorSearchConfig) *MongoDBSearchReconciler { - mdbcWatcher := watch.New() return &MongoDBSearchReconciler{ kubeClient: kubernetesClient.NewClient(client), - mdbcWatcher: &mdbcWatcher, + watch: watch.NewResourceWatcher(), operatorSearchConfig: operatorSearchConfig, } } @@ -50,31 +52,77 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci return result, err } - sourceResource, err := getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch) + searchSource, err := r.getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch, log) if err != nil { return reconcile.Result{RequeueAfter: time.Second * util.RetryTimeSec}, err } - r.mdbcWatcher.Watch(ctx, sourceResource.NamespacedName(), request.NamespacedName) + r.watch.AddWatchedResourceIfNotAdded(searchSource.KeyfileSecretName(), mdbSearch.Namespace, watch.Secret, mdbSearch.NamespacedName()) - reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, sourceResource, r.operatorSearchConfig) + // Watch for changes in database source CA certificate secrets or configmaps + tlsSourceConfig := searchSource.TLSConfig() + if tlsSourceConfig != nil { + for wType, resources := range tlsSourceConfig.ResourcesToWatch { + for _, resource := range resources { + r.watch.AddWatchedResourceIfNotAdded(resource.Name, resource.Namespace, wType, mdbSearch.NamespacedName()) + } + } + } + + // Watch our own TLS certificate secret for changes + if mdbSearch.Spec.Security.TLS.Enabled { + r.watch.AddWatchedResourceIfNotAdded(mdbSearch.Spec.Security.TLS.CertificateKeySecret.Name, mdbSearch.Namespace, watch.Secret, mdbSearch.NamespacedName()) + } + + reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, searchSource, r.operatorSearchConfig) return reconcileHelper.Reconcile(ctx, log).ReconcileResult() } -func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, error) { +func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch, log *zap.SugaredLogger) (search_controller.SearchSourceDBResource, error) { + if search.IsExternalMongoDBSource() { + return search_controller.NewExternalSearchSource(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil + } + sourceMongoDBResourceRef := search.GetMongoDBResourceRef() - mdbcName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name} + if sourceMongoDBResourceRef == nil { + return nil, xerrors.New("MongoDBSearch source MongoDB resource reference is not set") + } + + sourceName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name} + log.Infof("Looking up Search source %s", sourceName) + + mdb := &mdbv1.MongoDB{} + if err := kubeClient.Get(ctx, sourceName, mdb); err != nil { + if !apierrors.IsNotFound(err) { + return nil, xerrors.Errorf("error getting MongoDB %s: %w", sourceName, err) + } + } else { + r.watch.AddWatchedResourceIfNotAdded(sourceMongoDBResourceRef.Name, sourceMongoDBResourceRef.Namespace, watch.MongoDB, search.NamespacedName()) + return search_controller.NewEnterpriseResourceSearchSource(mdb), nil + } + mdbc := &mdbcv1.MongoDBCommunity{} - if err := kubeClient.Get(ctx, mdbcName, mdbc); err != nil { - return nil, xerrors.Errorf("error getting MongoDBCommunity %s", mdbcName) + if err := kubeClient.Get(ctx, sourceName, mdbc); err != nil { + if !apierrors.IsNotFound(err) { + return nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", sourceName, err) + } + } else { + r.watch.AddWatchedResourceIfNotAdded(sourceMongoDBResourceRef.Name, sourceMongoDBResourceRef.Namespace, "MongoDBCommunity", search.NamespacedName()) + return search_controller.NewCommunityResourceSearchSource(mdbc), nil } - return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), nil + + return nil, xerrors.Errorf("No database resource named %s found", sourceName) } func mdbcSearchIndexBuilder(rawObj client.Object) []string { mdbSearch := rawObj.(*searchv1.MongoDBSearch) - return []string{mdbSearch.GetMongoDBResourceRef().Namespace + "/" + mdbSearch.GetMongoDBResourceRef().Name} + resourceRef := mdbSearch.GetMongoDBResourceRef() + if resourceRef == nil { + return []string{} + } + + return []string{resourceRef.Namespace + "/" + resourceRef.Name} } func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operatorSearchConfig search_controller.OperatorSearchConfig) error { @@ -87,7 +135,11 @@ func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operat return ctrl.NewControllerManagedBy(mgr). WithOptions(controller.Options{MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}). // nolint:forbidigo For(&searchv1.MongoDBSearch{}). - Watches(&mdbcv1.MongoDBCommunity{}, r.mdbcWatcher). + Watches(&mdbv1.MongoDB{}, &watch.ResourcesHandler{ResourceType: watch.MongoDB, ResourceWatcher: r.watch}). + Watches(&mdbcv1.MongoDBCommunity{}, &watch.ResourcesHandler{ResourceType: "MongoDBCommunity", ResourceWatcher: r.watch}). + Watches(&corev1.Secret{}, &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: r.watch}). + Watches(&corev1.ConfigMap{}, &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: r.watch}). Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Secret{}). Complete(r) } diff --git a/controllers/operator/mongodbsearch_controller_test.go b/controllers/operator/mongodbsearch_controller_test.go index 0c7ddcce5..28f966ff8 100644 --- a/controllers/operator/mongodbsearch_controller_test.go +++ b/controllers/operator/mongodbsearch_controller_test.go @@ -8,9 +8,8 @@ import ( "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1 "k8s.io/api/apps/v1" @@ -25,7 +24,9 @@ import ( "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1/common" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/mongot" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/util/constants" ) func newMongoDBCommunity(name, namespace string) *mdbcv1.MongoDBCommunity { @@ -34,7 +35,7 @@ func newMongoDBCommunity(name, namespace string) *mdbcv1.MongoDBCommunity { Spec: mdbcv1.MongoDBCommunitySpec{ Type: mdbcv1.ReplicaSet, Members: 1, - Version: "8.0", + Version: "8.0.10", }, } } @@ -50,15 +51,25 @@ func newMongoDBSearch(name, namespace, mdbcName string) *searchv1.MongoDBSearch } } -func newSearchReconciler( +func newSearchReconcilerWithOperatorConfig( mdbc *mdbcv1.MongoDBCommunity, + operatorConfig search_controller.OperatorSearchConfig, searches ...*searchv1.MongoDBSearch, ) (*MongoDBSearchReconciler, client.Client) { builder := mock.NewEmptyFakeClientBuilder() builder.WithIndex(&searchv1.MongoDBSearch{}, search_controller.MongoDBSearchIndexFieldName, mdbcSearchIndexBuilder) if mdbc != nil { - builder.WithObjects(mdbc) + keyfileSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: mdbc.GetAgentKeyfileSecretNamespacedName().Name, + Namespace: mdbc.Namespace, + }, + StringData: map[string]string{ + constants.AgentKeyfileKey: "keyfile", + }, + } + builder.WithObjects(mdbc, keyfileSecret) } for _, search := range searches { @@ -68,25 +79,58 @@ func newSearchReconciler( } fakeClient := builder.Build() - return newMongoDBSearchReconciler(fakeClient, search_controller.OperatorSearchConfig{}), fakeClient + + return newMongoDBSearchReconciler(fakeClient, operatorConfig), fakeClient +} + +func newSearchReconciler( + mdbc *mdbcv1.MongoDBCommunity, + searches ...*searchv1.MongoDBSearch, +) (*MongoDBSearchReconciler, client.Client) { + return newSearchReconcilerWithOperatorConfig(mdbc, search_controller.OperatorSearchConfig{}, searches...) } func buildExpectedMongotConfig(search *searchv1.MongoDBSearch, mdbc *mdbcv1.MongoDBCommunity) mongot.Config { - return mongot.Config{CommunityPrivatePreview: mongot.CommunityPrivatePreview{ - MongodHostAndPort: fmt.Sprintf( - "%s.%s.svc.cluster.local:%d", - mdbc.ServiceName(), mdbc.Namespace, - mdbc.GetMongodConfiguration().GetDBPort(), - ), - QueryServerAddress: fmt.Sprintf("localhost:%d", search.GetMongotPort()), - KeyFilePath: "/mongot/keyfile/keyfile", - DataPath: "/mongot/data/config.yml", - Metrics: mongot.Metrics{ + var hostAndPorts []string + for i := range mdbc.Spec.Members { + hostAndPorts = append(hostAndPorts, fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", mdbc.Name, i, mdbc.Name+"-svc", search.Namespace, 27017)) + } + return mongot.Config{ + SyncSource: mongot.ConfigSyncSource{ + ReplicaSet: mongot.ConfigReplicaSet{ + HostAndPort: hostAndPorts, + Username: searchv1.MongotDefaultSyncSourceUsername, + PasswordFile: "/tmp/sourceUserPassword", + TLS: ptr.To(false), + ReadPreference: ptr.To("secondaryPreferred"), + AuthSource: ptr.To("admin"), + }, + }, + Storage: mongot.ConfigStorage{ + DataPath: "/mongot/data/config.yml", + }, + Server: mongot.ConfigServer{ + Wireproto: &mongot.ConfigWireproto{ + Address: "0.0.0.0:27027", + Authentication: &mongot.ConfigAuthentication{ + Mode: "keyfile", + KeyFile: "/tmp/keyfile", + }, + TLS: mongot.ConfigTLS{Mode: mongot.ConfigTLSModeDisabled}, + }, + }, + Metrics: mongot.ConfigMetrics{ Enabled: true, - Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()), + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotMetricsPort()), + }, + HealthCheck: mongot.ConfigHealthCheck{ + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotHealthCheckPort()), + }, + Logging: mongot.ConfigLogging{ + Verbosity: "TRACE", + LogPath: nil, }, - Logging: mongot.Logging{Verbosity: "DEBUG"}, - }} + } } func TestMongoDBSearchReconcile_NotFound(t *testing.T) { @@ -146,10 +190,6 @@ func TestMongoDBSearchReconcile_Success(t *testing.T) { sts := &appsv1.StatefulSet{} err = c.Get(ctx, search.StatefulSetNamespacedName(), sts) assert.NoError(t, err) - - queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request]()) - reconciler.mdbcWatcher.Create(ctx, event.CreateEvent{Object: mdbc}, queue) - assert.Equal(t, 1, queue.Len()) } func checkSearchReconcileFailed( @@ -183,16 +223,6 @@ func TestMongoDBSearchReconcile_InvalidVersion(t *testing.T) { checkSearchReconcileFailed(ctx, t, reconciler, c, search, "MongoDB version") } -func TestMongoDBSearchReconcile_TLSNotSupported(t *testing.T) { - ctx := context.Background() - search := newMongoDBSearch("search", mock.TestNamespace, "mdb") - mdbc := newMongoDBCommunity("mdb", mock.TestNamespace) - mdbc.Spec.Security.TLS.Enabled = true - reconciler, c := newSearchReconciler(mdbc, search) - - checkSearchReconcileFailed(ctx, t, reconciler, c, search, "TLS-enabled") -} - func TestMongoDBSearchReconcile_MultipleSearchResources(t *testing.T) { ctx := context.Background() search1 := newMongoDBSearch("search1", mock.TestNamespace, "mdb") @@ -202,3 +232,60 @@ func TestMongoDBSearchReconcile_MultipleSearchResources(t *testing.T) { checkSearchReconcileFailed(ctx, t, reconciler, c, search1, "multiple MongoDBSearch") } + +func TestMongoDBSearchReconcile_InvalidSearchImageVersion(t *testing.T) { + ctx := context.Background() + expectedMsg := "MongoDBSearch version 1.47.0 is not supported because of breaking changes. The operator will ignore this resource: it will not reconcile or reconfigure the workload. Existing deployments will continue to run, but cannot be managed by the operator. To regain operator management, you must delete and recreate the MongoDBSearch resource." + + tests := []struct { + name string + specVersion string + operatorVersion string + statefulSetConfig *common.StatefulSetConfiguration + }{ + { + name: "unsupported version in Spec.Version", + specVersion: "1.47.0", + }, + { + name: "unsupported version in operator config", + operatorVersion: "1.47.0", + }, + { + name: "unsupported version in StatefulSetConfiguration", + statefulSetConfig: &common.StatefulSetConfiguration{ + SpecWrapper: common.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: search_controller.MongotContainerName, + Image: "testrepo/mongot:1.47.0", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + search := newMongoDBSearch("search", mock.TestNamespace, "mdb") + mdbc := newMongoDBCommunity("mdb", mock.TestNamespace) + + search.Spec.Version = tc.specVersion + search.Spec.StatefulSetConfiguration = tc.statefulSetConfig + + operatorConfig := search_controller.OperatorSearchConfig{ + SearchVersion: tc.operatorVersion, + } + reconciler, _ := newSearchReconcilerWithOperatorConfig(mdbc, operatorConfig, search) + + checkSearchReconcileFailed(ctx, t, reconciler, reconciler.kubeClient, search, expectedMsg) + }) + } +} diff --git a/controllers/operator/watch/config_change_handler.go b/controllers/operator/watch/config_change_handler.go index 7b53ee4fe..77f725f20 100644 --- a/controllers/operator/watch/config_change_handler.go +++ b/controllers/operator/watch/config_change_handler.go @@ -14,8 +14,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" corev1 "k8s.io/api/core/v1" - - rolev1 "github.com/mongodb/mongodb-kubernetes/api/v1/role" ) // Type is an enum for all kubernetes types watched by controller for changes for configuration @@ -87,10 +85,14 @@ func (c *ResourcesHandler) doHandle(namespace, name string, q workqueue.TypedRat // Seems we don't need to react on config map/secret removal.. func (c *ResourcesHandler) Delete(ctx context.Context, e event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - switch v := e.Object.(type) { - case *rolev1.ClusterMongoDBRole: - c.doHandle(v.GetNamespace(), v.GetName(), q) + switch e.Object.(type) { + case *corev1.ConfigMap: + return + case *corev1.Secret: + return } + + c.doHandle(e.Object.GetNamespace(), e.Object.GetName(), q) } func (c *ResourcesHandler) Generic(context.Context, event.TypedGenericEvent[client.Object], workqueue.TypedRateLimitingInterface[reconcile.Request]) { diff --git a/controllers/search_controller/community_search_source.go b/controllers/search_controller/community_search_source.go new file mode 100644 index 000000000..8b10a1cbf --- /dev/null +++ b/controllers/search_controller/community_search_source.go @@ -0,0 +1,84 @@ +package search_controller + +import ( + "fmt" + "strings" + + "github.com/blang/semver" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + corev1 "k8s.io/api/core/v1" + + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" + mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" + "github.com/mongodb/mongodb-kubernetes/pkg/util" +) + +func NewCommunityResourceSearchSource(mdbc *mdbcv1.MongoDBCommunity) SearchSourceDBResource { + return &CommunitySearchSource{MongoDBCommunity: mdbc} +} + +type CommunitySearchSource struct { + *mdbcv1.MongoDBCommunity +} + +func (r *CommunitySearchSource) HostSeeds() []string { + seeds := make([]string, r.Spec.Members) + for i := range seeds { + seeds[i] = fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", r.Name, i, r.ServiceName(), r.Namespace, r.GetMongodConfiguration().GetDBPort()) + } + return seeds +} + +func (r *CommunitySearchSource) KeyfileSecretName() string { + return r.MongoDBCommunity.GetAgentKeyfileSecretNamespacedName().Name +} + +func (r *CommunitySearchSource) TLSConfig() *TLSSourceConfig { + if !r.Spec.Security.TLS.Enabled { + return nil + } + + var volume corev1.Volume + watchedResources := make(map[watch.Type][]types.NamespacedName) + + if r.Spec.Security.TLS.CaCertificateSecret != nil { + volume = statefulset.CreateVolumeFromSecret("ca", r.Spec.Security.TLS.CaCertificateSecret.Name) + watchedResources[watch.Secret] = []types.NamespacedName{r.TLSCaCertificateSecretNamespacedName()} + } else { + volume = statefulset.CreateVolumeFromConfigMap("ca", r.Spec.Security.TLS.CaConfigMap.Name) + watchedResources[watch.ConfigMap] = []types.NamespacedName{r.TLSConfigMapNamespacedName()} + } + + return &TLSSourceConfig{ + CAFileName: "ca.crt", + CAVolume: volume, + ResourcesToWatch: watchedResources, + } +} + +func (r *CommunitySearchSource) Validate() error { + version, err := semver.ParseTolerant(r.GetMongoDBVersion()) + if err != nil { + return xerrors.Errorf("error parsing MongoDB version '%s': %w", r.Spec.Version, err) + } else if version.LT(semver.MustParse("8.0.10")) { + return xerrors.New("MongoDB version must be 8.0.10 or higher") + } + + foundScram := false + for _, authMode := range r.Spec.Security.Authentication.Modes { + // Check for SCRAM, SCRAM-SHA-1, or SCRAM-SHA-256 + if strings.HasPrefix(strings.ToUpper(string(authMode)), util.SCRAM) { + foundScram = true + break + } + } + + if !foundScram && len(r.Spec.Security.Authentication.Modes) > 0 { + return xerrors.New("MongoDBSearch requires SCRAM authentication to be enabled") + } + + return nil +} diff --git a/controllers/search_controller/community_search_source_test.go b/controllers/search_controller/community_search_source_test.go new file mode 100644 index 000000000..90bf9967e --- /dev/null +++ b/controllers/search_controller/community_search_source_test.go @@ -0,0 +1,208 @@ +package search_controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" +) + +func newCommunitySearchSource(version string, authModes []mdbcv1.AuthMode) *CommunitySearchSource { + return &CommunitySearchSource{ + MongoDBCommunity: &mdbcv1.MongoDBCommunity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mongodb", + Namespace: "test-namespace", + }, + Spec: mdbcv1.MongoDBCommunitySpec{ + Version: version, + Security: mdbcv1.Security{ + Authentication: mdbcv1.Authentication{ + Modes: authModes, + }, + }, + }, + }, + } +} + +func TestCommunitySearchSource_Validate(t *testing.T) { + cases := []struct { + name string + version string + authModes []mdbcv1.AuthMode + expectError bool + expectedErrMsg string + }{ + // Version validation tests + { + name: "Invalid version", + version: "invalid.version", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "error parsing MongoDB version", + }, + { + name: "Version too old", + version: "7.0.0", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Version just below minimum", + version: "8.0.9", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid minimum version", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Version above minimum", + version: "8.1.0", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Version with build number", + version: "8.1.0-rc1", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + // Authentication mode tests - empty/nil cases + { + name: "Empty auth modes", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{}, + expectError: false, + }, + { + name: "Nil auth modes", + version: "8.0.10", + authModes: nil, + expectError: false, + }, + { + name: "X509 mode only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + { + name: "X509 and SCRAM", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"X509", "SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Multiple auth modes with SCRAM first", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-1", "X509"}, + expectError: false, + }, + { + name: "Multiple auth modes with SCRAM last", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"PLAIN", "X509", "SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Multiple non-SCRAM modes", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"PLAIN", "X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + // SCRAM variant tests + { + name: "SCRAM only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM"}, + expectError: false, + }, + { + name: "SCRAM-SHA-1 only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-1"}, + expectError: false, + }, + { + name: "SCRAM-SHA-256 only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "All SCRAM variants", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM", "SCRAM-SHA-1", "SCRAM-SHA-256"}, + expectError: false, + }, + // Case-insensitive tests (now supported with ToUpper) + { + name: "Lowercase SCRAM", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"scram-sha-256"}, + expectError: false, + }, + { + name: "Mixed case SCRAM", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"Scram-Sha-256"}, + expectError: false, + }, + // Edge case tests + { + name: "PLAIN only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"PLAIN"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + // Combined validation tests + { + name: "Invalid version with valid auth", + version: "7.0.0", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid version with invalid auth", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + { + name: "Invalid version with invalid auth", + version: "7.0.0", + authModes: []mdbcv1.AuthMode{"X509"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", // Should fail on version first + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + src := newCommunitySearchSource(c.version, c.authModes) + err := src.Validate() + + if c.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), c.expectedErrMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/controllers/search_controller/enterprise_search_source.go b/controllers/search_controller/enterprise_search_source.go new file mode 100644 index 000000000..c1256fbbb --- /dev/null +++ b/controllers/search_controller/enterprise_search_source.go @@ -0,0 +1,88 @@ +package search_controller + +import ( + "fmt" + "strings" + + "github.com/blang/semver" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" + "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" + "github.com/mongodb/mongodb-kubernetes/pkg/util" +) + +type EnterpriseResourceSearchSource struct { + *mdbv1.MongoDB +} + +func NewEnterpriseResourceSearchSource(mdb *mdbv1.MongoDB) SearchSourceDBResource { + return EnterpriseResourceSearchSource{mdb} +} + +func (r EnterpriseResourceSearchSource) HostSeeds() []string { + seeds := make([]string, r.Spec.Members) + for i := range seeds { + seeds[i] = fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", r.Name, i, r.ServiceName(), r.Namespace, r.Spec.GetAdditionalMongodConfig().GetPortOrDefault()) + } + return seeds +} + +func (r EnterpriseResourceSearchSource) TLSConfig() *TLSSourceConfig { + if !r.Spec.Security.IsTLSEnabled() { + return nil + } + + return &TLSSourceConfig{ + CAFileName: "ca-pem", + CAVolume: statefulset.CreateVolumeFromConfigMap("ca", r.Spec.Security.TLSConfig.CA), + ResourcesToWatch: map[watch.Type][]types.NamespacedName{ + watch.ConfigMap: { + {Namespace: r.Namespace, Name: r.Spec.Security.TLSConfig.CA}, + }, + }, + } +} + +func (r EnterpriseResourceSearchSource) KeyfileSecretName() string { + return fmt.Sprintf("%s-keyfile", r.Name) +} + +func (r EnterpriseResourceSearchSource) Validate() error { + version, err := semver.ParseTolerant(r.Spec.GetMongoDBVersion()) + if err != nil { + return xerrors.Errorf("error parsing MongoDB version '%s': %w", r.Spec.GetMongoDBVersion(), err) + } else if version.LT(semver.MustParse("8.0.10")) { + return xerrors.New("MongoDB version must be 8.0.10 or higher") + } + + if r.Spec.GetTopology() != mdbv1.ClusterTopologySingleCluster { + return xerrors.Errorf("MongoDBSearch is only supported for %s topology", mdbv1.ClusterTopologySingleCluster) + } + + if r.GetResourceType() != mdbv1.ReplicaSet { + return xerrors.Errorf("MongoDBSearch is only supported for %s resources", mdbv1.ReplicaSet) + } + + authModes := r.Spec.GetSecurityAuthenticationModes() + foundScram := false + for _, authMode := range authModes { + // Check for SCRAM, SCRAM-SHA-1, or SCRAM-SHA-256 + if strings.HasPrefix(strings.ToUpper(authMode), util.SCRAM) { + foundScram = true + break + } + } + + if !foundScram && len(authModes) > 0 { + return xerrors.New("MongoDBSearch requires SCRAM authentication to be enabled") + } + + if r.Spec.Security.GetInternalClusterAuthenticationMode() == util.X509 { + return xerrors.New("MongoDBSearch does not support X.509 internal cluster authentication") + } + + return nil +} diff --git a/controllers/search_controller/enterprise_search_source_test.go b/controllers/search_controller/enterprise_search_source_test.go new file mode 100644 index 000000000..1362ab605 --- /dev/null +++ b/controllers/search_controller/enterprise_search_source_test.go @@ -0,0 +1,290 @@ +package search_controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" +) + +func newEnterpriseSearchSource(version string, topology string, resourceType mdbv1.ResourceType, authModes []string, internalClusterAuth string) EnterpriseResourceSearchSource { + authModesList := make([]mdbv1.AuthMode, len(authModes)) + for i, mode := range authModes { + authModesList[i] = mdbv1.AuthMode(mode) + } + + // Create security with authentication if needed + var security *mdbv1.Security + if len(authModes) > 0 || internalClusterAuth != "" { + security = &mdbv1.Security{ + Authentication: &mdbv1.Authentication{ + Enabled: len(authModes) > 0, + Modes: authModesList, + InternalCluster: internalClusterAuth, + }, + } + } + + src := EnterpriseResourceSearchSource{ + MongoDB: &mdbv1.MongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mongodb", + Namespace: "test-namespace", + }, + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + Version: version, + ResourceType: resourceType, + Security: security, + }, + }, + }, + } + + // Set topology directly since it's inlined from DbCommonSpec + src.Spec.Topology = topology + return src +} + +func TestEnterpriseResourceSearchSource_Validate(t *testing.T) { + cases := []struct { + name string + version string + topology string + resourceType mdbv1.ResourceType + authModes []string + internalClusterAuth string + expectError bool + expectedErrMsg string + }{ + // Version validation tests + { + name: "Invalid version", + version: "invalid.version", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "error parsing MongoDB version", + }, + { + name: "Version too old", + version: "7.0.0", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Version just below minimum", + version: "8.0.9", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid minimum version", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + { + name: "Version above minimum", + version: "8.1.0", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + // Topology validation tests + { + name: "Invalid topology - MultiCluster", + version: "8.0.10", + topology: mdbv1.ClusterTopologyMultiCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for SingleCluster topology", + }, + { + name: "Valid topology - SingleCluster", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + { + name: "Empty topology defaults to SingleCluster", + version: "8.0.10", + topology: "", + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + // Resource type validation tests + { + name: "Invalid resource type - Standalone", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.Standalone, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for ReplicaSet resources", + }, + { + name: "Invalid resource type - ShardedCluster", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ShardedCluster, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for ReplicaSet resources", + }, + { + name: "Valid resource type - ReplicaSet", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + // Authentication mode tests + { + name: "No SCRAM authentication", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + { + name: "Empty authentication modes", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + { + name: "Nil authentication modes", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: nil, + expectError: false, + }, + { + name: "Valid SCRAM authentication", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Mixed auth modes with SCRAM", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"X509", "SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Case insensitive SCRAM", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"scram-sha-256"}, + expectError: false, + }, + { + name: "SCRAM variants", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM", "SCRAM-SHA-1", "SCRAM-SHA-256"}, + expectError: false, + }, + // Internal cluster authentication tests + { + name: "X509 internal cluster auth not supported", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + internalClusterAuth: "X509", + expectError: true, + expectedErrMsg: "MongoDBSearch does not support X.509 internal cluster authentication", + }, + { + name: "Valid internal cluster auth - empty", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + internalClusterAuth: "", + expectError: false, + }, + { + name: "Valid internal cluster auth - SCRAM", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + internalClusterAuth: "SCRAM", + expectError: false, + }, + // Combined validation tests + { + name: "Multiple validation failures - version takes precedence", + version: "7.0.0", + topology: mdbv1.ClusterTopologyMultiCluster, + resourceType: mdbv1.Standalone, + authModes: []string{"X509"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid version, invalid topology", + version: "8.0.10", + topology: mdbv1.ClusterTopologyMultiCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for SingleCluster topology", + }, + { + name: "Valid version and topology, invalid resource type", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.Standalone, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for ReplicaSet resources", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + src := newEnterpriseSearchSource(c.version, c.topology, c.resourceType, c.authModes, c.internalClusterAuth) + err := src.Validate() + + if c.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), c.expectedErrMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/controllers/search_controller/external_search_source.go b/controllers/search_controller/external_search_source.go new file mode 100644 index 000000000..5a408246e --- /dev/null +++ b/controllers/search_controller/external_search_source.go @@ -0,0 +1,49 @@ +package search_controller + +import ( + "k8s.io/apimachinery/pkg/types" + + searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" + "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" +) + +func NewExternalSearchSource(namespace string, spec *searchv1.ExternalMongoDBSource) SearchSourceDBResource { + return &externalSearchResource{namespace: namespace, spec: spec} +} + +// externalSearchResource implements SearchSourceDBResource for deployments managed outside the operator. +type externalSearchResource struct { + namespace string + spec *searchv1.ExternalMongoDBSource +} + +func (r *externalSearchResource) Validate() error { + return nil +} + +func (r *externalSearchResource) TLSConfig() *TLSSourceConfig { + if r.spec.TLS == nil || !r.spec.TLS.Enabled { + return nil + } + + return &TLSSourceConfig{ + CAFileName: "ca.crt", + CAVolume: statefulset.CreateVolumeFromSecret("ca", r.spec.TLS.CA.Name), + ResourcesToWatch: map[watch.Type][]types.NamespacedName{ + watch.Secret: { + {Namespace: r.namespace, Name: r.spec.TLS.CA.Name}, + }, + }, + } +} + +func (r *externalSearchResource) KeyfileSecretName() string { + if r.spec.KeyFileSecretKeyRef != nil { + return r.spec.KeyFileSecretKeyRef.Name + } + + return "" +} + +func (r *externalSearchResource) HostSeeds() []string { return r.spec.HostAndPorts } diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/search_controller/mongodbsearch_reconcile_helper.go index 59523ce8e..4b384a707 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper.go @@ -7,30 +7,42 @@ import ( "fmt" "strings" - "github.com/blang/semver" "github.com/ghodss/yaml" "go.uber.org/zap" "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/automationconfig" kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/podtemplatespec" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/service" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/mongot" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/tls" + "github.com/mongodb/mongodb-kubernetes/pkg/kube" "github.com/mongodb/mongodb-kubernetes/pkg/kube/commoncontroller" "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" ) const ( - MongoDBSearchIndexFieldName = "mdbsearch-for-mongodbresourceref-index" + MongoDBSearchIndexFieldName = "mdbsearch-for-mongodbresourceref-index" + unsupportedSearchVersion = "1.47.0" + unsupportedSearchVersionErrorFmt = "MongoDBSearch version %s is not supported because of breaking changes. " + + "The operator will ignore this resource: it will not reconcile or reconfigure the workload. " + + "Existing deployments will continue to run, but cannot be managed by the operator. " + + "To regain operator management, you must delete and recreate the MongoDBSearch resource." ) type OperatorSearchConfig struct { @@ -72,7 +84,11 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S log = log.With("MongoDBSearch", r.mdbSearch.NamespacedName()) log.Infof("Reconciling MongoDBSearch") - if err := ValidateSearchSource(r.db); err != nil { + if err := r.db.Validate(); err != nil { + return workflow.Failed(err) + } + + if err := r.ValidateSearchImageVersion(); err != nil { return workflow.Failed(err) } @@ -80,39 +96,79 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S return workflow.Failed(err) } + keyfileStsModification, err := r.ensureSourceKeyfile(ctx, log) + if apierrors.IsNotFound(err) { + return workflow.Pending("Waiting for keyfile secret to be created") + } else if err != nil { + return workflow.Failed(err) + } + if err := r.ensureSearchService(ctx, r.mdbSearch); err != nil { return workflow.Failed(err) } - mongotConfig := createMongotConfig(r.mdbSearch, r.db) - configHash, err := r.ensureMongotConfig(ctx, mongotConfig) + ingressTlsMongotModification, ingressTlsStsModification, err := r.ensureIngressTlsConfig(ctx) if err != nil { return workflow.Failed(err) } - if err := r.createOrUpdateStatefulSet(ctx, log, configHash); err != nil { + egressTlsMongotModification, egressTlsStsModification, err := r.ensureEgressTlsConfig(ctx) + if err != nil { + return workflow.Failed(err) + } + + configHash, err := r.ensureMongotConfig(ctx, log, createMongotConfig(r.mdbSearch, r.db), ingressTlsMongotModification, egressTlsMongotModification) + if err != nil { return workflow.Failed(err) } - if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.db.NamespacedName().Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() { + configHashModification := statefulset.WithPodSpecTemplate(podtemplatespec.WithAnnotations( + map[string]string{ + "mongotConfigHash": configHash, + }, + )) + + if err := r.createOrUpdateStatefulSet(ctx, log, CreateSearchStatefulSetFunc(r.mdbSearch, r.db, r.buildImageString()), configHashModification, keyfileStsModification, ingressTlsStsModification, egressTlsStsModification); err != nil { + return workflow.Failed(err) + } + + if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.mdbSearch.Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() { return statefulSetStatus } return workflow.OK() } -func (r *MongoDBSearchReconcileHelper) createOrUpdateStatefulSet(ctx context.Context, log *zap.SugaredLogger, mongotConfigHash string) error { +func (r *MongoDBSearchReconcileHelper) ensureSourceKeyfile(ctx context.Context, log *zap.SugaredLogger) (statefulset.Modification, error) { + keyfileSecretName := kube.ObjectKey(r.mdbSearch.GetNamespace(), r.db.KeyfileSecretName()) + keyfileSecret := &corev1.Secret{} + if err := r.client.Get(ctx, keyfileSecretName, keyfileSecret); err != nil { + return nil, err + } + + return statefulset.Apply( + // make sure mongot pods get restarted if the keyfile changes + statefulset.WithPodSpecTemplate(podtemplatespec.WithAnnotations( + map[string]string{ + "keyfileHash": hashBytes(keyfileSecret.Data["keyfile"]), + }, + )), + ), nil +} + +func (r *MongoDBSearchReconcileHelper) buildImageString() string { imageVersion := r.mdbSearch.Spec.Version if imageVersion == "" { imageVersion = r.operatorSearchConfig.SearchVersion } - searchImage := fmt.Sprintf("%s/%s:%s", r.operatorSearchConfig.SearchRepo, r.operatorSearchConfig.SearchName, imageVersion) + return fmt.Sprintf("%s/%s:%s", r.operatorSearchConfig.SearchRepo, r.operatorSearchConfig.SearchName, imageVersion) +} +func (r *MongoDBSearchReconcileHelper) createOrUpdateStatefulSet(ctx context.Context, log *zap.SugaredLogger, modifications ...statefulset.Modification) error { stsName := r.mdbSearch.StatefulSetNamespacedName() sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: stsName.Name, Namespace: stsName.Namespace}} op, err := controllerutil.CreateOrUpdate(ctx, r.client, sts, func() error { - stsModification := CreateSearchStatefulSetFunc(r.mdbSearch, r.db, searchImage, mongotConfigHash) - stsModification(sts) + statefulset.Apply(modifications...)(sts) return nil }) if err != nil { @@ -142,7 +198,9 @@ func (r *MongoDBSearchReconcileHelper) ensureSearchService(ctx context.Context, return nil } -func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, mongotConfig mongot.Config) (string, error) { +func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, log *zap.SugaredLogger, modifications ...mongot.Modification) (string, error) { + mongotConfig := mongot.Config{} + mongot.Apply(modifications...)(&mongotConfig) configData, err := yaml.Marshal(mongotConfig) if err != nil { return "", err @@ -163,13 +221,89 @@ func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, m return "", err } - zap.S().Debugf("Updated mongot config yaml config map: %v (%s) with the following configuration: %s", cmName, op, string(configData)) + log.Debugf("Updated mongot config yaml config map: %v (%s) with the following configuration: %s", cmName, op, string(configData)) - return hashMongotConfig(configData), nil + return hashBytes(configData), nil } -func hashMongotConfig(mongotConfigYaml []byte) string { - hashBytes := sha256.Sum256(mongotConfigYaml) +func (r *MongoDBSearchReconcileHelper) ensureIngressTlsConfig(ctx context.Context) (mongot.Modification, statefulset.Modification, error) { + if !r.mdbSearch.Spec.Security.TLS.Enabled { + mongotModification := func(config *mongot.Config) { + config.Server.Wireproto.TLS.Mode = mongot.ConfigTLSModeDisabled + } + return mongotModification, statefulset.NOOP(), nil + } + + // TODO: validate that the certificate in the user-provided Secret in .spec.security.tls.certificateKeySecret is issued by the CA in the operator's CA Secret + + certFileName, err := tls.EnsureTLSSecret(ctx, r.client, r.mdbSearch) + if err != nil { + return nil, nil, err + } + + mongotModification := func(config *mongot.Config) { + certPath := tls.OperatorSecretMountPath + certFileName + config.Server.Wireproto.TLS.Mode = mongot.ConfigTLSModeTLS + config.Server.Wireproto.TLS.CertificateKeyFile = ptr.To(certPath) + } + + tlsSecret := r.mdbSearch.TLSOperatorSecretNamespacedName() + tlsVolume := statefulset.CreateVolumeFromSecret("tls", tlsSecret.Name) + tlsVolumeMount := statefulset.CreateVolumeMount("tls", tls.OperatorSecretMountPath, statefulset.WithReadOnly(true)) + statefulsetModification := statefulset.WithPodSpecTemplate(podtemplatespec.Apply( + podtemplatespec.WithVolume(tlsVolume), + podtemplatespec.WithContainer(MongotContainerName, container.Apply( + container.WithVolumeMounts([]corev1.VolumeMount{tlsVolumeMount}), + )), + )) + + return mongotModification, statefulsetModification, nil +} + +func (r *MongoDBSearchReconcileHelper) ensureEgressTlsConfig(ctx context.Context) (mongot.Modification, statefulset.Modification, error) { + tlsSourceConfig := r.db.TLSConfig() + if tlsSourceConfig == nil { + return mongot.NOOP(), statefulset.NOOP(), nil + } + + mongotModification := func(config *mongot.Config) { + config.SyncSource.ReplicaSet.TLS = ptr.To(true) + } + + _, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + caVolume := tlsSourceConfig.CAVolume + trustStoreVolume := statefulset.CreateVolumeFromEmptyDir("cacerts") + statefulsetModification := statefulset.WithPodSpecTemplate(podtemplatespec.Apply( + podtemplatespec.WithVolume(caVolume), + podtemplatespec.WithVolume(trustStoreVolume), + podtemplatespec.WithInitContainer("init-cacerts", container.Apply( + container.WithImage(r.buildImageString()), + containerSecurityContext, + container.WithVolumeMounts([]corev1.VolumeMount{ + statefulset.CreateVolumeMount(caVolume.Name, tls.CAMountPath, statefulset.WithReadOnly(true)), + statefulset.CreateVolumeMount(trustStoreVolume.Name, "/java/trust-store", statefulset.WithReadOnly(false)), + }), + container.WithCommand([]string{"sh"}), + container.WithArgs([]string{ + "-c", + fmt.Sprintf(` +cp /mongot-community/bin/jdk/lib/security/cacerts /java/trust-store/cacerts +/mongot-community/bin/jdk/bin/keytool -keystore /java/trust-store/cacerts -storepass changeit -noprompt -trustcacerts -importcert -alias mongodcert -file %s/%s + `, tls.CAMountPath, tlsSourceConfig.CAFileName), + }), + )), + podtemplatespec.WithContainer(MongotContainerName, container.Apply( + container.WithVolumeMounts([]corev1.VolumeMount{ + statefulset.CreateVolumeMount(trustStoreVolume.Name, "/mongot-community/bin/jdk/lib/security/cacerts", statefulset.WithReadOnly(true), statefulset.WithSubPath("cacerts")), + }), + )), + )) + + return mongotModification, statefulsetModification, nil +} + +func hashBytes(bytes []byte) string { + hashBytes := sha256.Sum256(bytes) return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hashBytes[:]) } @@ -203,30 +337,67 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service { TargetPort: intstr.FromInt32(search.GetMongotMetricsPort()), }) + serviceBuilder.AddPort(&corev1.ServicePort{ + Name: "healthcheck", + Protocol: corev1.ProtocolTCP, + Port: search.GetMongotHealthCheckPort(), + TargetPort: intstr.FromInt32(search.GetMongotHealthCheckPort()), + }) + return serviceBuilder.Build() } -func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Config { - return mongot.Config{CommunityPrivatePreview: mongot.CommunityPrivatePreview{ - MongodHostAndPort: fmt.Sprintf("%s.%s.svc.cluster.local:%d", db.DatabaseServiceName(), db.GetNamespace(), db.DatabasePort()), - QueryServerAddress: fmt.Sprintf("localhost:%d", search.GetMongotPort()), - KeyFilePath: "/mongot/keyfile/keyfile", - DataPath: "/mongot/data/config.yml", - Metrics: mongot.Metrics{ +func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Modification { + return func(config *mongot.Config) { + hostAndPorts := db.HostSeeds() + + config.SyncSource = mongot.ConfigSyncSource{ + ReplicaSet: mongot.ConfigReplicaSet{ + HostAndPort: hostAndPorts, + Username: search.SourceUsername(), + PasswordFile: "/tmp/sourceUserPassword", + TLS: ptr.To(false), + ReadPreference: ptr.To("secondaryPreferred"), + AuthSource: ptr.To("admin"), + }, + } + config.Storage = mongot.ConfigStorage{ + DataPath: "/mongot/data/config.yml", + } + config.Server = mongot.ConfigServer{ + Wireproto: &mongot.ConfigWireproto{ + Address: "0.0.0.0:27027", + Authentication: &mongot.ConfigAuthentication{ + Mode: "keyfile", + KeyFile: "/tmp/keyfile", + }, + }, + } + config.Metrics = mongot.ConfigMetrics{ Enabled: true, - Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()), - }, - Logging: mongot.Logging{ - Verbosity: "DEBUG", - }, - }} + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotMetricsPort()), + } + config.HealthCheck = mongot.ConfigHealthCheck{ + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotHealthCheckPort()), + } + config.Logging = mongot.ConfigLogging{ + Verbosity: "TRACE", + LogPath: nil, + } + } } -func GetMongodConfigParameters(search *searchv1.MongoDBSearch) map[string]interface{} { - return map[string]interface{}{ - "setParameter": map[string]interface{}{ - "mongotHost": mongotHostAndPort(search), - "searchIndexManagementHostAndPort": mongotHostAndPort(search), +func GetMongodConfigParameters(search *searchv1.MongoDBSearch) map[string]any { + searchTLSMode := automationconfig.TLSModeDisabled + if search.Spec.Security.TLS.Enabled { + searchTLSMode = automationconfig.TLSModeRequired + } + return map[string]any{ + "setParameter": map[string]any{ + "mongotHost": mongotHostAndPort(search), + "searchIndexManagementHostAndPort": mongotHostAndPort(search), + "skipAuthenticationToSearchIndexManagementServer": false, + "searchTLSMode": string(searchTLSMode), }, } } @@ -236,27 +407,17 @@ func mongotHostAndPort(search *searchv1.MongoDBSearch) string { return fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName.Name, svcName.Namespace, search.GetMongotPort()) } -func ValidateSearchSource(db SearchSourceDBResource) error { - version, err := semver.ParseTolerant(db.GetMongoDBVersion()) - if err != nil { - return xerrors.Errorf("error parsing MongoDB version '%s': %w", db.GetMongoDBVersion(), err) - } else if version.Major < 8 { - return xerrors.New("MongoDB version must be 8.0 or higher") - } - - if db.IsSecurityTLSConfigEnabled() { - return xerrors.New("MongoDBSearch does not support TLS-enabled sources") +func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error { + if r.mdbSearch.Spec.Source != nil && r.mdbSearch.Spec.Source.ExternalMongoDBSource != nil { + return nil } - return nil -} - -func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error { + ref := r.mdbSearch.GetMongoDBResourceRef() searchList := &searchv1.MongoDBSearchList{} if err := r.client.List(ctx, searchList, &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, r.db.GetNamespace()+"/"+r.db.Name()), + FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, ref.Namespace+"/"+ref.Name), }); err != nil { - return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", r.db.Name(), err) + return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", ref.Name, err) } if len(searchList.Items) > 1 { @@ -264,8 +425,92 @@ func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSourc for i, search := range searchList.Items { resourceNames[i] = search.Name } - return xerrors.Errorf("Found multiple MongoDBSearch resources for search source '%s': %s", r.db.Name(), strings.Join(resourceNames, ", ")) + return xerrors.Errorf( + "Found multiple MongoDBSearch resources for search source '%s': %s", ref.Name, + strings.Join(resourceNames, ", "), + ) } return nil } + +func (r *MongoDBSearchReconcileHelper) ValidateSearchImageVersion() error { + version := r.getMongotImage() + + if strings.Contains(version, unsupportedSearchVersion) { + return xerrors.Errorf(unsupportedSearchVersionErrorFmt, unsupportedSearchVersion) + } + + return nil +} + +func (r *MongoDBSearchReconcileHelper) getMongotImage() string { + version := strings.TrimSpace(r.mdbSearch.Spec.Version) + if version != "" { + return version + } + + if r.operatorSearchConfig.SearchVersion != "" { + return r.operatorSearchConfig.SearchVersion + } + + if r.mdbSearch.Spec.StatefulSetConfiguration == nil { + return "" + } + + for _, container := range r.mdbSearch.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers { + if container.Name == MongotContainerName { + return container.Image + } + } + + return "" +} + +func SearchCoordinatorRole() mdbv1.MongoDBRole { + // direct translation of https://github.com/10gen/mongo/blob/6f8d95a513eea8f91ea9f5d895dd8a288dfcf725/src/mongo/db/auth/builtin_roles.yml#L652 + return mdbv1.MongoDBRole{ + Role: "searchCoordinator", + Db: "admin", + Roles: []mdbv1.InheritedRole{ + { + Role: "clusterMonitor", + Db: "admin", + }, + { + Role: "directShardOperations", + Db: "admin", + }, + { + Role: "readAnyDatabase", + Db: "admin", + }, + }, + Privileges: []mdbv1.Privilege{ + { + Resource: mdbv1.Resource{ + Db: "__mdb_internal_search", + }, + Actions: []string{ + "changeStream", "collStats", "dbHash", "dbStats", "find", + "killCursors", "listCollections", "listIndexes", "listSearchIndexes", + // performRawDataOperations is available only on mongod master + // "performRawDataOperations", + "planCacheRead", "cleanupStructuredEncryptionData", + "compactStructuredEncryptionData", "convertToCapped", "createCollection", + "createIndex", "createSearchIndexes", "dropCollection", "dropIndex", + "dropSearchIndex", "insert", "remove", "renameCollectionSameDB", + "update", "updateSearchIndex", + }, + }, + // TODO: this causes the error "(BadValue) resource: {cluster: true} conflicts with resource type 'db'" + // { + // Resource: mdbv1.Resource{ + // Cluster: ptr.To(true), + // }, + // Actions: []string{"bypassDefaultMaxTimeMS"}, + // }, + }, + AuthenticationRestrictions: nil, + } +} diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go b/controllers/search_controller/mongodbsearch_reconcile_helper_test.go index fe03ceb70..5a2a757ce 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper_test.go @@ -15,66 +15,6 @@ import ( kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" ) -func TestMongoDBSearchReconcileHelper_ValidateSearchSource(t *testing.T) { - mdbcMeta := metav1.ObjectMeta{ - Name: "test-mongodb", - Namespace: "test", - } - - cases := []struct { - name string - mdbc mdbcv1.MongoDBCommunity - expectedError string - }{ - { - name: "Invalid version", - mdbc: mdbcv1.MongoDBCommunity{ - ObjectMeta: mdbcMeta, - Spec: mdbcv1.MongoDBCommunitySpec{ - Version: "4.4.0", - }, - }, - expectedError: "MongoDB version must be 8.0 or higher", - }, - { - name: "Valid version", - mdbc: mdbcv1.MongoDBCommunity{ - ObjectMeta: mdbcMeta, - Spec: mdbcv1.MongoDBCommunitySpec{ - Version: "8.0", - }, - }, - }, - { - name: "TLS enabled", - mdbc: mdbcv1.MongoDBCommunity{ - ObjectMeta: mdbcMeta, - Spec: mdbcv1.MongoDBCommunitySpec{ - Version: "8.0", - Security: mdbcv1.Security{ - TLS: mdbcv1.TLS{ - Enabled: true, - }, - }, - }, - }, - expectedError: "MongoDBSearch does not support TLS-enabled sources", - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - db := NewSearchSourceDBResourceFromMongoDBCommunity(&c.mdbc) - err := ValidateSearchSource(db) - if c.expectedError == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, c.expectedError) - } - }) - } -} - func TestMongoDBSearchReconcileHelper_ValidateSingleMongoDBSearchForSearchSource(t *testing.T) { mdbSearchSpec := searchv1.MongoDBSearchSpec{ Source: &searchv1.MongoDBSource{ @@ -145,7 +85,7 @@ func TestMongoDBSearchReconcileHelper_ValidateSingleMongoDBSearchForSearchSource clientBuilder.WithObjects(v) } - helper := NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(clientBuilder.Build()), mdbSearch, NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), OperatorSearchConfig{}) + helper := NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(clientBuilder.Build()), mdbSearch, NewCommunityResourceSearchSource(mdbc), OperatorSearchConfig{}) err := helper.ValidateSingleMongoDBSearchForSearchSource(t.Context()) if c.expectedError == "" { assert.NoError(t, err) diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index ea169339d..ffaa72358 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -9,7 +9,7 @@ import ( searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/construct" - mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1/common" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/podtemplatespec" @@ -19,73 +19,30 @@ import ( ) const ( - MongotContainerName = "mongot" + MongotContainerName = "mongot" + SearchLivenessProbePath = "/health" + SearchReadinessProbePath = "/health" // Todo: Update this when search GA is available ) // SearchSourceDBResource is an object wrapping a MongoDBCommunity object // Its purpose is to: // - isolate and identify all the data we need to get from the CR in order to reconcile search resources // - implement search reconcile logic in a generic way that is working for any types of MongoDB databases (all database CRs). -// -// TODO check if we could use already existing interface (DbCommon, MongoDBStatefulSetOwner, etc.) type SearchSourceDBResource interface { - Name() string - NamespacedName() types.NamespacedName KeyfileSecretName() string - GetNamespace() string - HasSeparateDataAndLogsVolumes() bool - DatabaseServiceName() string - DatabasePort() int - GetMongoDBVersion() string - IsSecurityTLSConfigEnabled() bool + TLSConfig() *TLSSourceConfig + HostSeeds() []string + Validate() error } -func NewSearchSourceDBResourceFromMongoDBCommunity(mdbc *mdbcv1.MongoDBCommunity) SearchSourceDBResource { - return &mdbcSearchResource{db: mdbc} -} - -type mdbcSearchResource struct { - db *mdbcv1.MongoDBCommunity -} - -func (r *mdbcSearchResource) Name() string { - return r.db.Name -} - -func (r *mdbcSearchResource) NamespacedName() types.NamespacedName { - return r.db.NamespacedName() -} - -func (r *mdbcSearchResource) KeyfileSecretName() string { - return r.db.GetAgentKeyfileSecretNamespacedName().Name -} - -func (r *mdbcSearchResource) GetNamespace() string { - return r.db.Namespace -} - -func (r *mdbcSearchResource) HasSeparateDataAndLogsVolumes() bool { - return r.db.HasSeparateDataAndLogsVolumes() -} - -func (r *mdbcSearchResource) DatabaseServiceName() string { - return r.db.ServiceName() -} - -func (r *mdbcSearchResource) GetMongoDBVersion() string { - return r.db.Spec.Version -} - -func (r *mdbcSearchResource) IsSecurityTLSConfigEnabled() bool { - return r.db.Spec.Security.TLS.Enabled -} - -func (r *mdbcSearchResource) DatabasePort() int { - return r.db.GetMongodConfiguration().GetDBPort() +type TLSSourceConfig struct { + CAFileName string + CAVolume corev1.Volume + ResourcesToWatch map[watch.Type][]types.NamespacedName } // ReplicaSetOptions returns a set of options which will configure a ReplicaSet StatefulSet -func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBResource SearchSourceDBResource, searchImage string, mongotConfigHash string) statefulset.Modification { +func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBResource SearchSourceDBResource, searchImage string) statefulset.Modification { labels := map[string]string{ "app": mdbSearch.SearchServiceNamespacedName().Name, } @@ -95,14 +52,18 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso dataVolumeName := "data" keyfileVolumeName := "keyfile" + sourceUserPasswordVolumeName := "password" mongotConfigVolumeName := "config" pvcVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/mongot/data", statefulset.WithSubPath("data")) - keyfileVolume := statefulset.CreateVolumeFromSecret("keyfile", sourceDBResource.KeyfileSecretName()) + keyfileVolume := statefulset.CreateVolumeFromSecret(keyfileVolumeName, sourceDBResource.KeyfileSecretName()) keyfileVolumeMount := statefulset.CreateVolumeMount(keyfileVolumeName, "/mongot/keyfile", statefulset.WithReadOnly(true)) - mongotConfigVolume := statefulset.CreateVolumeFromConfigMap("config", mdbSearch.MongotConfigConfigMapNamespacedName().Name) + sourceUserPasswordVolume := statefulset.CreateVolumeFromSecret(sourceUserPasswordVolumeName, mdbSearch.SourceUserPasswordSecretRef().Name) + sourceUserPasswordVolumeMount := statefulset.CreateVolumeMount(sourceUserPasswordVolumeName, "/mongot/sourceUserPassword", statefulset.WithReadOnly(true)) + + mongotConfigVolume := statefulset.CreateVolumeFromConfigMap(mongotConfigVolumeName, mdbSearch.MongotConfigConfigMapNamespacedName().Name) mongotConfigVolumeMount := statefulset.CreateVolumeMount(mongotConfigVolumeName, "/mongot/config", statefulset.WithReadOnly(true)) var persistenceConfig *common.PersistenceConfig @@ -120,12 +81,14 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso keyfileVolumeMount, tmpVolumeMount, mongotConfigVolumeMount, + sourceUserPasswordVolumeMount, } volumes := []corev1.Volume{ tmpVolume, keyfileVolume, mongotConfigVolume, + sourceUserPasswordVolume, } stsModifications := []statefulset.Modification{ @@ -142,11 +105,7 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso podtemplatespec.Apply( podSecurityContext, podtemplatespec.WithPodLabels(labels), - podtemplatespec.WithAnnotations(map[string]string{ - "mongotConfigHash": mongotConfigHash, - }), podtemplatespec.WithVolumes(volumes), - podtemplatespec.WithServiceAccount(sourceDBResource.DatabaseServiceName()), podtemplatespec.WithServiceAccount(util.MongoDBServiceAccount), podtemplatespec.WithContainer(MongotContainerName, mongodbSearchContainer(mdbSearch, volumeMounts, searchImage)), ), @@ -170,22 +129,65 @@ func mongodbSearchContainer(mdbSearch *searchv1.MongoDBSearch, volumeMounts []co container.WithName(MongotContainerName), container.WithImage(searchImage), container.WithImagePullPolicy(corev1.PullAlways), - container.WithReadinessProbe(probes.Apply( - probes.WithTCPSocket("", intstr.FromInt32(mdbSearch.GetMongotPort())), - probes.WithInitialDelaySeconds(20), - probes.WithPeriodSeconds(10), - )), + container.WithLivenessProbe(mongotLivenessProbe(mdbSearch)), + container.WithReadinessProbe(mongotReadinessProbe(mdbSearch)), container.WithResourceRequirements(createSearchResourceRequirements(mdbSearch.Spec.ResourceRequirements)), container.WithVolumeMounts(volumeMounts), container.WithCommand([]string{"sh"}), container.WithArgs([]string{ "-c", - "/mongot-community/mongot --config /mongot/config/config.yml", + ` +cp /mongot/keyfile/keyfile /tmp/keyfile +chown 2000:2000 /tmp/keyfile +chmod 0600 /tmp/keyfile + +cp /mongot/sourceUserPassword/password /tmp/sourceUserPassword +chown 2000:2000 /tmp/sourceUserPassword +chmod 0600 /tmp/sourceUserPassword + +/mongot-community/mongot --config /mongot/config/config.yml +`, }), containerSecurityContext, ) } +func mongotLivenessProbe(search *searchv1.MongoDBSearch) func(*corev1.Probe) { + return probes.Apply( + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Port: intstr.FromInt32(search.GetMongotHealthCheckPort()), + Path: SearchLivenessProbePath, + }, + }), + probes.WithInitialDelaySeconds(10), + probes.WithPeriodSeconds(10), + probes.WithTimeoutSeconds(5), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(10), + ) +} + +// mongotReadinessProbe just uses the endpoint intended for liveness checks; +// readiness check endpoint may be available in search GA. +func mongotReadinessProbe(search *searchv1.MongoDBSearch) func(*corev1.Probe) { + return probes.Apply( + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Port: intstr.FromInt32(search.GetMongotHealthCheckPort()), + Path: SearchReadinessProbePath, + }, + }), + probes.WithInitialDelaySeconds(20), + probes.WithPeriodSeconds(10), + probes.WithTimeoutSeconds(5), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(3), + ) +} + func createSearchResourceRequirements(requirements *corev1.ResourceRequirements) corev1.ResourceRequirements { if requirements != nil { return *requirements diff --git a/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go b/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go index 793914d3a..ca7b9a3b0 100755 --- a/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go +++ b/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go @@ -145,7 +145,7 @@ func readLinesFromFile(name string) ([]string, error) { func writeLinesToFile(name string, lines []string) error { output := strings.Join(lines, lineBreak) - err := os.WriteFile(name, []byte(output), 0o775) + err := os.WriteFile(name, []byte(output), 0o644) if err != nil { return xerrors.Errorf("error writing to file %s: %w", name, err) } @@ -168,7 +168,7 @@ func appendLinesToFile(name string, lines string) error { func getOmPropertiesFromEnvVars() map[string]string { props := map[string]string{} - for _, pair := range os.Environ() { + for _, pair := range os.Environ() { // nolint:forbidigo if !strings.HasPrefix(pair, omPropertyPrefix) { continue } diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index da6887b81..9e0cca8fe 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -120,6 +120,9 @@ def helm_repo_add(repo_name: str, url: str): def process_run_and_check(args, **kwargs): + if "check" not in kwargs: + kwargs["check"] = True + try: logger.debug(f"subprocess.run: {args}") completed_process = subprocess.run(args, **kwargs) diff --git a/docker/mongodb-kubernetes-tests/kubetester/mongodb.py b/docker/mongodb-kubernetes-tests/kubetester/mongodb.py index c9970361a..b3f38239b 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/mongodb.py +++ b/docker/mongodb-kubernetes-tests/kubetester/mongodb.py @@ -278,10 +278,8 @@ def configure_custom_tls( tls_cert_secret_name: str, ): ensure_nested_objects(self, ["spec", "security", "tls"]) - self["spec"]["security"] = { - "certsSecretPrefix": tls_cert_secret_name, - "tls": {"enabled": True, "ca": issuer_ca_configmap_name}, - } + self["spec"]["security"]["certsSecretPrefix"] = tls_cert_secret_name + self["spec"]["security"]["tls"].update({"enabled": True, "ca": issuer_ca_configmap_name}) def build_list_of_hosts(self): """Returns the list of full_fqdn:27017 for every member of the mongodb resource""" diff --git a/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py b/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py index e5629bda0..be2151158 100644 --- a/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py +++ b/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py @@ -1,3 +1,7 @@ +import logging + +import pymongo.errors +from kubetester import kubetester from tests import test_logger from tests.common.search.search_tester import SearchTester @@ -26,9 +30,33 @@ def create_search_index(self): def wait_for_search_indexes(self): self.search_tester.wait_for_search_indexes_ready(self.db_name, self.col_name) - def assert_search_query(self): - # sample taken from: https://www.mongodb.com/docs/atlas/atlas-search/tutorial/run-query/#run-a-complex-query-on-the-movies-collection-7 - result = self.search_tester.client[self.db_name][self.col_name].aggregate( + def assert_search_query(self, retry_timeout: int = 1): + def wait_for_search_results(): + # sample taken from: https://www.mongodb.com/docs/atlas/atlas-search/tutorial/run-query/#run-a-complex-query-on-the-movies-collection-7 + count = 0 + status_msg = "" + try: + result = self.execute_example_search_query() + status_msg = f"{self.db_name}/{self.col_name}: search query results:\n" + for r in result: + status_msg += f"{r}\n" + count += 1 + status_msg += f"Count: {count}" + logger.debug(status_msg) + except pymongo.errors.PyMongoError as e: + logger.debug(f"error: {e}") + + return count == 4, status_msg + + kubetester.run_periodically( + fn=wait_for_search_results, + timeout=retry_timeout, + sleep_time=1, + msg="Search query to return correct data", + ) + + def execute_example_search_query(self): + return self.search_tester.client[self.db_name][self.col_name].aggregate( [ { "$search": { @@ -47,11 +75,3 @@ def assert_search_query(self): {"$project": {"title": 1, "plot": 1, "genres": 1, "_id": 0}}, ] ) - - logger.debug(f"{self.db_name}/{self.col_name}: search query results:") - count = 0 - for r in result: - logger.debug(r) - count += 1 - - assert count == 4 diff --git a/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py b/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py index bbca34b3b..6dccfb5ae 100644 --- a/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py +++ b/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py @@ -15,8 +15,10 @@ class SearchTester(MongoTester): def __init__( self, connection_string: str, + use_ssl: bool = False, + ca_path: str | None = None, ): - super().__init__(connection_string, False) + super().__init__(connection_string, use_ssl, ca_path) def mongorestore_from_url(self, archive_url: str, ns_include: str, mongodb_tools_dir: str = ""): logger.debug(f"running mongorestore from {archive_url}") @@ -26,7 +28,11 @@ def mongorestore_from_url(self, archive_url: str, ns_include: str, mongodb_tools logger.debug(f"Downloaded sample file from {archive_url} to {sample_file.name} (size: {size})") mongorestore_path = os.path.join(mongodb_tools_dir, "mongorestore") mongorestore_cmd = f"{mongorestore_path} --archive={sample_file.name} --verbose=1 --drop --nsInclude {ns_include} --uri={self.cnx_string}" - process_run_and_check(mongorestore_cmd.split()) + if self.default_opts.get("tls", False): + mongorestore_cmd += " --ssl" + if ca_path := self.default_opts.get("tlsCAFile"): + mongorestore_cmd += " --sslCAFile=" + ca_path + process_run_and_check(mongorestore_cmd.split(), capture_output=True) def create_search_index(self, database_name: str, collection_name: str): database = self.client[database_name] @@ -49,6 +55,10 @@ def wait_for_search_indexes_ready(self, database_name: str, collection_name: str def search_indexes_ready(self, database_name: str, collection_name: str): search_indexes = self.get_search_indexes(database_name, collection_name) + if len(search_indexes) == 0: + logger.error(f"There are no search indexes available in {database_name}.{collection_name}") + return False + for idx in search_indexes: if idx.get("status") != "READY": logger.debug(f"{database_name}/{collection_name}: search index {idx} is not ready") diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml new file mode 100644 index 000000000..45e332fd3 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: mdbc-rs +spec: + version: 8.0.10 + type: ReplicaSet + members: 3 + security: + authentication: + ignoreUnknownUsers: true + modes: + - SCRAM + roles: + - role: searchCoordinator + db: admin + roles: + - name: clusterMonitor + db: admin + - name: directShardOperations + db: admin + - name: readAnyDatabase + db: admin + privileges: + - resource: + db: "__mdb_internal_search" + collection: "" + actions: + - "changeStream" + - "collStats" + - "dbHash" + - "dbStats" + - "find" + - "killCursors" + - "listCollections" + - "listIndexes" + - "listSearchIndexes" + - "planCacheRead" + - "cleanupStructuredEncryptionData" + - "compactStructuredEncryptionData" + - "convertToCapped" + - "createCollection" + - "createIndex" + - "createSearchIndexes" + - "dropCollection" + - "dropIndex" + - "dropSearchIndex" + - "insert" + - "remove" + - "renameCollectionSameDB" + - "update" + - "updateSearchIndex" + - resource: + cluster: true + actions: + - "bypassDefaultMaxTimeMS" + agent: + logLevel: DEBUG + statefulSet: + spec: + template: + spec: + containers: + - name: mongod + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi + - name: mongodb-agent + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "0.5" + memory: 1Gi + users: + # admin user with root role + - name: mdb-admin + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-admin-user-password + scramCredentialsSecretName: mdb-admin-user + roles: + - name: root + db: admin + # user performing search queries + - name: mdb-user + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-user-password + scramCredentialsSecretName: mdb-user-scram + roles: + - name: restore + db: sample_mflix + - name: readWrite + db: sample_mflix + # user used by MongoDB Search to connect to MongoDB database to synchronize data from + # For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically + # From MongoDB 8.2, searchCoordinator role will be a built-in role. + - name: search-sync-source + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: mdbc-rs-search-sync-source-password + scramCredentialsSecretName: mdbc-rs-search-sync-source + roles: + - name: searchCoordinator + db: admin diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml index 4ba3d40b6..fe54d614d 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml @@ -4,27 +4,16 @@ kind: MongoDBCommunity metadata: name: mdbc-rs spec: - members: 3 + version: 8.0.10 type: ReplicaSet - version: "8.0.5" + members: 3 security: authentication: - modes: ["SCRAM"] + ignoreUnknownUsers: true + modes: + - SCRAM agent: logLevel: DEBUG - users: - - name: my-user - db: admin - passwordSecretRef: # a reference to the secret that will be used to generate the user's password - name: my-user-password - roles: - - name: clusterAdmin - db: admin - - name: userAdminAnyDatabase - db: admin - - name: readWrite - db: sample_mflix - scramCredentialsSecretName: my-scram statefulSet: spec: template: @@ -33,16 +22,48 @@ spec: - name: mongod resources: limits: - cpu: "3" - memory: 5Gi - requests: cpu: "2" - memory: 5Gi + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi - name: mongodb-agent resources: limits: - cpu: "3" - memory: 5Gi + cpu: "1" + memory: 2Gi requests: - cpu: "2" - memory: 5Gi + cpu: "0.5" + memory: 1Gi + users: + # admin user with root role + - name: mdb-admin + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-admin-user-password + scramCredentialsSecretName: mdb-admin-user + roles: + - name: root + db: admin + # user performing search queries + - name: mdb-user + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-user-password + scramCredentialsSecretName: mdb-user-scram + roles: + - name: restore + db: sample_mflix + - name: readWrite + db: sample_mflix + # user used by MongoDB Search to connect to MongoDB database to synchronize data from + # For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically + # From MongoDB 8.2, searchCoordinator role will be a built-in role. + - name: search-sync-source + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: mdbc-rs-search-sync-source-password + scramCredentialsSecretName: mdbc-rs-search-sync-source + roles: + - name: searchCoordinator + db: admin diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml new file mode 100644 index 000000000..29d455b51 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-rs +spec: + members: 3 + version: 8.0.10 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + security: + authentication: + enabled: true + ignoreUnknownUsers: true + modes: + - SCRAM + agent: + logLevel: DEBUG + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml new file mode 100644 index 000000000..0b3fe4c77 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml @@ -0,0 +1,16 @@ +# admin user with root role +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mdb-admin +spec: + username: mdb-admin + db: admin + mongodbResourceRef: + name: mdb-rs + passwordSecretKeyRef: + name: mdb-admin-user-password + key: password + roles: + - name: root + db: admin \ No newline at end of file diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml new file mode 100644 index 000000000..579cc58d0 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml @@ -0,0 +1,16 @@ +# user performing search queries +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mdb-user +spec: + username: mdb-user + db: admin + mongodbResourceRef: + name: mdb-rs + passwordSecretKeyRef: + name: mdb-user-password + key: password + roles: + - name: readWrite + db: sample_mflix diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml new file mode 100644 index 000000000..cd1eab1a5 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml @@ -0,0 +1,18 @@ +# user used by MongoDB Search to connect to MongoDB database to synchronize data from +# For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically +# From MongoDB 8.2, searchCoordinator role will be a built-in role. +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: search-sync-source-user +spec: + username: search-sync-source + db: admin + mongodbResourceRef: + name: mdb-rs + passwordSecretKeyRef: + name: mdb-rs-search-sync-source-password + key: password + roles: + - name: searchCoordinator + db: admin diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml new file mode 100644 index 000000000..33648e04a --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml @@ -0,0 +1,5 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBSearch +metadata: + name: mdbc-rs +spec: {} diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py index 69cbca895..97e9c1af7 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py @@ -12,21 +12,26 @@ logger = test_logger.get_test_logger(__name__) -USER_PASSWORD = "Passw0rd." -USER_NAME = "my-user" +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + MDBC_RESOURCE_NAME = "mdbc-rs" @fixture(scope="function") -def mdbc(namespace: str, custom_mdb_version: str) -> MongoDBCommunity: +def mdbc(namespace: str) -> MongoDBCommunity: resource = MongoDBCommunity.from_yaml( yaml_fixture("community-replicaset-sample-mflix.yaml"), name=MDBC_RESOURCE_NAME, namespace=namespace, ) - resource["spec"]["version"] = custom_mdb_version - if try_load(resource): return resource @@ -53,8 +58,14 @@ def test_install_operator(namespace: str, operator_installation_config: dict[str @mark.e2e_search_community_basic -def test_install_secret(namespace: str): - create_or_update_secret(namespace=namespace, name="my-user-password", data={"password": USER_PASSWORD}) +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) @mark.e2e_search_community_basic @@ -64,7 +75,7 @@ def test_create_database_resource(mdbc: MongoDBCommunity): @mark.e2e_search_community_basic -def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): +def test_create_search_resource(mdbs: MongoDBSearch): mdbs.update() mdbs.assert_reaches_phase(Phase.Running, timeout=300) @@ -76,7 +87,9 @@ def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): @fixture(scope="function") def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper: - return movies_search_helper.SampleMoviesSearchHelper(SearchTester(get_connection_string(mdbc))) + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)) + ) @mark.e2e_search_community_basic @@ -89,15 +102,10 @@ def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelp sample_movies_helper.create_search_index() -@mark.e2e_search_community_basic -def test_search_wait_for_search_indexes(sample_movies_helper: SampleMoviesSearchHelper): - sample_movies_helper.wait_for_search_indexes() - - @mark.e2e_search_community_basic def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): - sample_movies_helper.assert_search_query() + sample_movies_helper.assert_search_query(retry_timeout=60) -def get_connection_string(mdbc: MongoDBCommunity) -> str: - return f"mongodb://{USER_NAME}:{USER_PASSWORD}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py new file mode 100644 index 000000000..110e134dc --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -0,0 +1,144 @@ +from kubetester import create_or_update_secret, try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" +MDBS_RESOURCE_NAME = "mdbs" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix-external.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + mongot_host = f"{MDBS_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" + if "additionalMongodConfig" not in resource["spec"]: + resource["spec"]["additionalMongodConfig"] = {} + if "setParameter" not in resource["spec"]["additionalMongodConfig"]: + resource["spec"]["additionalMongodConfig"]["setParameter"] = {} + + resource["spec"]["additionalMongodConfig"]["setParameter"].update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "disabled", + } + ) + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + name=MDBS_RESOURCE_NAME, + namespace=namespace, + ) + + seeds = [ + f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" for i in range(mdbc["spec"]["members"]) + ] + + resource["spec"] = { + "source": { + "external": { + "hostAndPorts": seeds, + "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile", "key": "keyfile"}, + "tls": {"enabled": False}, + }, + "passwordSecretRef": {"name": f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", "key": "password"}, + "username": MONGOT_USER_NAME, + } + } + + return resource + + +@mark.e2e_search_external_basic +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_external_basic +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + + create_or_update_secret( + namespace=namespace, + name=f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", + data={"password": MONGOT_USER_PASSWORD}, + ) + + +@mark.e2e_search_external_basic +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_basic +def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_basic +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)) + ) + + +@mark.e2e_search_external_basic +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_external_basic +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_external_basic +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py new file mode 100644 index 000000000..f6a270a48 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -0,0 +1,190 @@ +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" +MDBS_RESOURCE_NAME = "mdbs" +TLS_SECRET_NAME = "tls-secret" +TLS_CA_SECRET_NAME = "tls-ca-secret" +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix-external.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + mongot_host = f"{MDBS_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" + if "additionalMongodConfig" not in resource["spec"]: + resource["spec"]["additionalMongodConfig"] = {} + if "setParameter" not in resource["spec"]["additionalMongodConfig"]: + resource["spec"]["additionalMongodConfig"]["setParameter"] = {} + + resource["spec"]["additionalMongodConfig"]["setParameter"].update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "requireTLS", + } + ) + + resource["spec"]["security"]["tls"] = { + "enabled": True, + "certificateKeySecretRef": {"name": TLS_SECRET_NAME}, + "caCertificateSecretRef": {"name": TLS_SECRET_NAME}, + } + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + name=MDBS_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + return resource + + +@mark.e2e_search_external_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_external_tls +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, + name=f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", + data={"password": MONGOT_USER_PASSWORD}, + ) + + +@mark.e2e_search_external_tls +def test_install_tls_secrets_and_configmaps( + namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str, issuer_ca_filepath: str +): + create_tls_certs(issuer, namespace, mdbc.name, mdbc["spec"]["members"], secret_name=TLS_SECRET_NAME) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + ca = open(issuer_ca_filepath).read() + + ca_secret_name = f"{mdbc.name}-ca" + create_or_update_secret(namespace=namespace, name=ca_secret_name, data={"ca.crt": ca}) + + +@mark.e2e_search_external_tls +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_tls +def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): + seeds = [ + f"{mdbc.name}-{i}.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017" + for i in range(mdbc["spec"]["members"]) + ] + + mdbs["spec"]["source"] = { + "external": { + "hostAndPorts": seeds, + "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, + "tls": { + "enabled": True, + "ca": {"name": f"{mdbc.name}-ca"}, + }, + }, + "passwordSecretRef": {"name": f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", "key": "password"}, + "username": MONGOT_USER_NAME, + } + + mdbs["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_external_tls +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester( + get_connection_string(mdbc, USER_NAME, USER_PASSWORD), + use_ssl=True, + ca_path=issuer_ca_filepath, + ) + ) + + +@mark.e2e_search_external_tls +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_external_tls +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_external_tls +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return ( + f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/" + f"?replicaSet={mdbc.name}" + ) diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py new file mode 100644 index 000000000..44418b793 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py @@ -0,0 +1,173 @@ +import pymongo +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" + +TLS_SECRET_NAME = "tls-secret" + +# MongoDBSearch TLS configuration +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + # Add TLS configuration + resource["spec"]["security"]["tls"] = { + "enabled": True, + "certificateKeySecretRef": {"name": TLS_SECRET_NAME}, + "caCertificateSecretRef": {"name": TLS_SECRET_NAME}, + } + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + # Add TLS configuration to MongoDBSearch + if "spec" not in resource: + resource["spec"] = {} + + resource["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + return resource + + +@mark.e2e_search_community_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_community_tls +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + # Create user password secrets + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) + + +@mark.e2e_search_community_tls +def test_install_tls_secrets_and_configmaps(namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str): + create_tls_certs(issuer, namespace, mdbc.name, mdbc["spec"]["members"], secret_name=TLS_SECRET_NAME) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + +@mark.e2e_search_community_tls +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_community_tls +def test_create_search_resource(mdbs: MongoDBSearch): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_community_tls +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@mark.e2e_search_community_tls +def test_validate_tls_connections(mdbc: MongoDBCommunity, mdbs: MongoDBSearch, namespace: str, issuer_ca_filepath: str): + with pymongo.MongoClient( + f"mongodb://{mdbc.name}-0.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=30000, + connectTimeoutMS=20000, + ) as mongodb_client: + mongodb_info = mongodb_client.admin.command("hello") + assert mongodb_info.get("ok") == 1, "MongoDBCommunity connection failed" + + with pymongo.MongoClient( + f"mongodb://{mdbs.name}-search-svc.{namespace}.svc.cluster.local:27027", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=10000, + connectTimeoutMS=10000, + ) as search_client: + search_info = search_client.admin.command("hello") + assert search_info.get("ok") == 1, "MongoDBSearch connection failed" + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath), + ) + + +@mark.e2e_search_community_tls +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_community_tls +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_community_tls +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py new file mode 100644 index 000000000..a3de8dcf8 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py @@ -0,0 +1,183 @@ +import yaml +from kubetester import create_or_update_secret, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB +from kubetester.mongodb_search import MongoDBSearch +from kubetester.mongodb_user import MongoDBUser +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = f"{ADMIN_USER_NAME}-password" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = f"{MONGOT_USER_NAME}-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = f"{USER_NAME}-password" + +MDB_RESOURCE_NAME = "mdb-rs" + + +@fixture(scope="function") +def mdb(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("enterprise-replicaset-sample-mflix.yaml"), + name=MDB_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml(yaml_fixture("search-minimal.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME) + + if try_load(resource): + return resource + + return resource + + +@fixture(scope="function") +def admin_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-mdb-admin.yaml"), namespace=namespace, name=ADMIN_USER_NAME + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodbuser-mdb-user.yaml"), namespace=namespace, name=USER_NAME) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def mongot_user(namespace: str, mdbs: MongoDBSearch) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-search-sync-source-user.yaml"), + namespace=namespace, + name=f"{mdbs.name}-{MONGOT_USER_NAME}", + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = MONGOT_USER_NAME + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@mark.e2e_search_enterprise_basic +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_enterprise_basic +def test_create_database_resource(mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_enterprise_basic +def test_create_users( + namespace: str, admin_user: MongoDBUser, user: MongoDBUser, mongot_user: MongoDBUser, mdb: MongoDB +): + create_or_update_secret( + namespace, name=admin_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": ADMIN_USER_PASSWORD} + ) + admin_user.create() + admin_user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=user["spec"]["passwordSecretKeyRef"]["name"], data={"password": USER_PASSWORD} + ) + user.create() + user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=mongot_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": MONGOT_USER_PASSWORD} + ) + mongot_user.create() + # we deliberately don't wait for this user to be ready, because to be reconciled successfully it needs the searchCoordinator role + # which the ReplicaSet reconciler will only define in the automation config after the MongoDBSearch resource is created. + + +@mark.e2e_search_enterprise_basic +def test_create_search_resource(mdbs: MongoDBSearch): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_enterprise_basic +def test_wait_for_database_resource_ready(mdb: MongoDB): + mdb.assert_abandons_phase(Phase.Running, timeout=1800) + mdb.assert_reaches_phase(Phase.Running, timeout=1800) + + for idx in range(mdb.get_members()): + mongod_config = yaml.safe_load( + KubernetesTester.run_command_in_pod_container( + f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"] + ) + ) + setParameter = mongod_config.get("setParameter", {}) + assert ( + "mongotHost" in setParameter and "searchIndexManagementHostAndPort" in setParameter + ), "mongot parameters not found in mongod config" + + +@mark.e2e_search_enterprise_basic +def test_search_restore_sample_database(mdb: MongoDB): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD)) + ) + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_enterprise_basic +def test_search_create_search_index(mdb: MongoDB): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD)) + ) + sample_movies_helper.create_search_index() + + +@mark.e2e_search_enterprise_basic +def test_search_assert_search_query(mdb: MongoDB): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD)) + ) + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py new file mode 100644 index 000000000..f4c0bc96f --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py @@ -0,0 +1,239 @@ +import pymongo +import yaml +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_mongodb_tls_certs, create_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB +from kubetester.mongodb_search import MongoDBSearch +from kubetester.mongodb_user import MongoDBUser +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = f"{ADMIN_USER_NAME}-password" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = f"{MONGOT_USER_NAME}-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = f"{USER_NAME}-password" + +MDB_RESOURCE_NAME = "mdb-rs" + +# MongoDBSearch TLS configuration +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdb(namespace: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("enterprise-replicaset-sample-mflix.yaml"), + name=MDB_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.configure_custom_tls(issuer_ca_configmap, "certs") + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml(yaml_fixture("search-minimal.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME) + + if try_load(resource): + return resource + + # Add TLS configuration to MongoDBSearch + if "spec" not in resource: + resource["spec"] = {} + + resource["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + return resource + + +@fixture(scope="function") +def admin_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-mdb-admin.yaml"), namespace=namespace, name=ADMIN_USER_NAME + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodbuser-mdb-user.yaml"), namespace=namespace, name=USER_NAME) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def mongot_user(namespace: str, mdbs: MongoDBSearch) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-search-sync-source-user.yaml"), + namespace=namespace, + name=f"{mdbs.name}-{MONGOT_USER_NAME}", + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = MONGOT_USER_NAME + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@mark.e2e_search_enterprise_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_enterprise_tls +def test_install_tls_secrets_and_configmaps(namespace: str, mdb: MongoDB, mdbs: MongoDBSearch, issuer: str): + create_mongodb_tls_certs(issuer, namespace, mdb.name, f"certs-{mdb.name}-cert", mdb.get_members()) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + +@mark.e2e_search_enterprise_tls +def test_create_database_resource(mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_enterprise_tls +def test_create_users( + namespace: str, admin_user: MongoDBUser, user: MongoDBUser, mongot_user: MongoDBUser, mdb: MongoDB +): + create_or_update_secret( + namespace, name=admin_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": ADMIN_USER_PASSWORD} + ) + admin_user.create() + admin_user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=user["spec"]["passwordSecretKeyRef"]["name"], data={"password": USER_PASSWORD} + ) + user.create() + user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=mongot_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": MONGOT_USER_PASSWORD} + ) + mongot_user.create() + # we deliberately don't wait for this user to be ready, because to be reconciled successfully it needs the searchCoordinator role + # which the ReplicaSet reconciler will only define in the automation config after the MongoDBSearch resource is created. + + +@mark.e2e_search_enterprise_tls +def test_create_search_resource(mdbs: MongoDBSearch): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_enterprise_tls +def test_wait_for_database_resource_ready(mdb: MongoDB): + mdb.assert_abandons_phase(Phase.Running, timeout=1800) + mdb.assert_reaches_phase(Phase.Running, timeout=1800) + + for idx in range(mdb.get_members()): + mongod_config = yaml.safe_load( + KubernetesTester.run_command_in_pod_container( + f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"] + ) + ) + setParameter = mongod_config.get("setParameter", {}) + assert ( + "mongotHost" in setParameter and "searchIndexManagementHostAndPort" in setParameter + ), "mongot parameters not found in mongod config" + + +@mark.e2e_search_enterprise_tls +def test_validate_tls_connections(mdb: MongoDB, mdbs: MongoDBSearch, namespace: str, issuer_ca_filepath: str): + with pymongo.MongoClient( + f"mongodb://{mdb.name}-0.{mdb.name}-svc.{namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=30000, + connectTimeoutMS=20000, + ) as mongodb_client: + mongodb_info = mongodb_client.admin.command("hello") + assert mongodb_info.get("ok") == 1, "MongoDB connection failed" + + with pymongo.MongoClient( + f"mongodb://{mdbs.name}-search-svc.{namespace}.svc.cluster.local:27027", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=10000, + connectTimeoutMS=10000, + ) as search_client: + search_info = search_client.admin.command("hello") + assert search_info.get("ok") == 1, "MongoDBSearch connection failed" + + +@mark.e2e_search_enterprise_tls +def test_search_restore_sample_database(mdb: MongoDB, issuer_ca_filepath: str): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester( + get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath + ) + ) + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_enterprise_tls +def test_search_create_search_index(mdb: MongoDB, issuer_ca_filepath: str): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath) + ) + sample_movies_helper.create_search_index() + + +@mark.e2e_search_enterprise_tls +def test_search_assert_search_query(mdb: MongoDB, issuer_ca_filepath: str): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath) + ) + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}" diff --git a/helm_chart/crds/mongodb.com_mongodbsearch.yaml b/helm_chart/crds/mongodb.com_mongodbsearch.yaml index 72ea0e50e..38dc38f2a 100644 --- a/helm_chart/crds/mongodb.com_mongodbsearch.yaml +++ b/helm_chart/crds/mongodb.com_mongodbsearch.yaml @@ -49,6 +49,8 @@ spec: spec: properties: persistence: + description: Configure MongoDB Search's persistent volume. If not + defined, the operator will request 10GB of storage. properties: multiple: properties: @@ -95,7 +97,8 @@ spec: type: object type: object resourceRequirements: - description: ResourceRequirements describes the compute resource requirements. + description: Configure resource requests and limits for the MongoDB + Search pods. properties: claims: description: |- @@ -153,8 +156,88 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + security: + description: Configure security settings of the MongoDB Search server + that MongoDB database is connecting to when performing search queries. + properties: + tls: + properties: + certificateKeySecretRef: + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object source: + description: MongoDB database connection details from which MongoDB + Search will synchronize data to build indexes. properties: + external: + properties: + hostAndPorts: + items: + type: string + type: array + keyFileSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + tls: + properties: + ca: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object mongodbResourceRef: properties: name: @@ -164,11 +247,26 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. + StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + which aren't exposed as fields in the MongoDBSearch.spec. properties: metadata: description: StatefulSetMetadataWrapper is a wrapper around Labels @@ -190,6 +288,9 @@ spec: - spec type: object version: + description: Optional version of MongoDB Search component (mongot). + If not set, then the operator will set the most appropriate version + of MongoDB Search. type: string type: object status: diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index ba3e5b168..67e062648 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -88,6 +89,9 @@ func NewReconciler(mgr manager.Manager, mongodbRepoUrl, mongodbImage, mongodbIma func findMdbcForSearch(ctx context.Context, rawObj k8sClient.Object) []reconcile.Request { mdbSearch := rawObj.(*searchv1.MongoDBSearch) + if mdbSearch.GetMongoDBResourceRef() == nil { + return nil + } return []reconcile.Request{ {NamespacedName: types.NamespacedName{Namespace: mdbSearch.GetMongoDBResourceRef().Namespace, Name: mdbSearch.GetMongoDBResourceRef().Name}}, } @@ -711,8 +715,8 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb // and that this resource passes search validations. If either fails, proceed without a search target // for the mongod automation config. if len(searchList.Items) == 1 { - searchSource := search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(&mdb) - if search_controller.ValidateSearchSource(searchSource) == nil { + searchSource := search_controller.NewCommunityResourceSearchSource(&mdb) + if searchSource.Validate() == nil { search = &searchList.Items[0] } } @@ -727,6 +731,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb prometheusModification, processPortManager.GetPortsModification(), getMongodConfigSearchModification(search), + searchCoordinatorCustomRoleModification(search), ) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not create an automation config: %s", err) @@ -739,6 +744,66 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb return automationConfig, nil } +// TODO: remove this as soon as searchCoordinator builtin role is backported +func searchCoordinatorCustomRoleModification(search *searchv1.MongoDBSearch) automationconfig.Modification { + if search == nil { + return automationconfig.NOOP() + } + + return func(ac *automationconfig.AutomationConfig) { + searchCoordinatorRole := searchCoordinatorCustomRoleStruct() + ac.Roles = append(ac.Roles, searchCoordinatorRole) + } +} + +func searchCoordinatorCustomRoleStruct() automationconfig.CustomRole { + // direct translation of https://github.com/10gen/mongo/blob/6f8d95a513eea8f91ea9f5d895dd8a288dfcf725/src/mongo/db/auth/builtin_roles.yml#L652 + return automationconfig.CustomRole{ + Role: "searchCoordinator", + DB: "admin", + Roles: []automationconfig.Role{ + { + Role: "clusterMonitor", + Database: "admin", + }, + { + Role: "directShardOperations", + Database: "admin", + }, + { + Role: "readAnyDatabase", + Database: "admin", + }, + }, + Privileges: []automationconfig.Privilege{ + { + Resource: automationconfig.Resource{ + DB: ptr.To("__mdb_internal_search"), + Collection: ptr.To(""), + }, + Actions: []string{ + "changeStream", "collStats", "dbHash", "dbStats", "find", + "killCursors", "listCollections", "listIndexes", "listSearchIndexes", + // performRawDataOperations is available only on mongod master + // "performRawDataOperations", + "planCacheRead", "cleanupStructuredEncryptionData", + "compactStructuredEncryptionData", "convertToCapped", "createCollection", + "createIndex", "createSearchIndexes", "dropCollection", "dropIndex", + "dropSearchIndex", "insert", "remove", "renameCollectionSameDB", + "update", "updateSearchIndex", + }, + }, + { + Resource: automationconfig.Resource{ + Cluster: true, + }, + Actions: []string{"bypassDefaultMaxTimeMS"}, + }, + }, + AuthenticationRestrictions: nil, + } +} + // OverrideToAutomationConfig turns an automation config override from the resource spec into an automation config // which can be used to merge. func OverrideToAutomationConfig(override mdbv1.AutomationConfigOverride) automationconfig.AutomationConfig { @@ -772,10 +837,9 @@ func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Mo // getMongodConfigModification will merge the additional configuration in the CRD // into the configuration set up by the operator. func getMongodConfigSearchModification(search *searchv1.MongoDBSearch) automationconfig.Modification { - if search == nil { - return func(config *automationconfig.AutomationConfig) { - // do nothing - } + // Condition for skipping add parameter if it is external mongod + if search == nil || search.IsExternalMongoDBSource() { + return automationconfig.NOOP() } searchConfigParameters := search_controller.GetMongodConfigParameters(search) diff --git a/mongodb-community-operator/pkg/mongot/mongot_config.go b/mongodb-community-operator/pkg/mongot/mongot_config.go index 80d4c1421..c8126b173 100644 --- a/mongodb-community-operator/pkg/mongot/mongot_config.go +++ b/mongodb-community-operator/pkg/mongot/mongot_config.go @@ -1,41 +1,82 @@ package mongot -type Config struct { - CommunityPrivatePreview CommunityPrivatePreview `json:"communityPrivatePreview"` +type Modification func(*Config) + +func NOOP() Modification { + return func(config *Config) {} } -// CommunityPrivatePreview structure reflects private preview configuration from mongot: -// https://github.com/10gen/mongot/blob/060ec179af062ac2639678f4a613b8ab02c21597/src/main/java/com/xgen/mongot/config/provider/community/CommunityConfig.java#L100 -// Comments are from the default config file: https://github.com/10gen/mongot/blob/375379e56a580916695a2f53e12fd4a99aa24f0b/deploy/community-resources/config.default.yml#L1-L0 -type CommunityPrivatePreview struct { - // Socket (IPv4/6) address of the sync source mongod - MongodHostAndPort string `json:"mongodHostAndPort"` +func Apply(modifications ...Modification) func(*Config) { + return func(config *Config) { + for _, mod := range modifications { + mod(config) + } + } +} - // Socket (IPv4/6) address on which to listen for wire protocol connections - QueryServerAddress string `json:"queryServerAddress"` +type Config struct { + SyncSource ConfigSyncSource `json:"syncSource"` + Storage ConfigStorage `json:"storage"` + Server ConfigServer `json:"server"` + Metrics ConfigMetrics `json:"metrics"` + HealthCheck ConfigHealthCheck `json:"healthCheck"` + Logging ConfigLogging `json:"logging"` +} - // Keyfile used for mongod -> mongot authentication - KeyFilePath string `json:"keyFilePath"` +type ConfigSyncSource struct { + ReplicaSet ConfigReplicaSet `json:"replicaSet"` +} + +type ConfigReplicaSet struct { + HostAndPort []string `json:"hostAndPort"` + Username string `json:"username"` + PasswordFile string `json:"passwordFile"` + TLS *bool `json:"tls,omitempty"` + ReadPreference *string `json:"readPreference,omitempty"` + AuthSource *string `json:"authSource,omitempty"` +} - // Filesystem path that all mongot data will be stored at +type ConfigStorage struct { DataPath string `json:"dataPath"` +} - // Options for metrics - Metrics Metrics `json:"metrics,omitempty"` +type ConfigServer struct { + Wireproto *ConfigWireproto `json:"wireproto,omitempty"` +} + +type ConfigWireproto struct { + Address string `json:"address"` + Authentication *ConfigAuthentication `json:"authentication,omitempty"` + TLS ConfigTLS `json:"tls"` +} - // Options for logging - Logging Logging `json:"logging,omitempty"` +type ConfigAuthentication struct { + Mode string `json:"mode"` + KeyFile string `json:"keyFile"` } -type Metrics struct { - // Whether to enable the Prometheus metrics endpoint - Enabled bool `json:"enabled"` +type ConfigTLSMode string + +const ( + ConfigTLSModeTLS ConfigTLSMode = "TLS" + ConfigTLSModeDisabled ConfigTLSMode = "Disabled" +) + +type ConfigTLS struct { + Mode ConfigTLSMode `json:"mode"` + CertificateKeyFile *string `json:"certificateKeyFile,omitempty"` +} + +type ConfigMetrics struct { + Enabled bool `json:"enabled"` + Address string `json:"address"` +} - // Socket address (IPv4/6) on which the Prometheus /metrics endpoint will be exposed +type ConfigHealthCheck struct { Address string `json:"address"` } -type Logging struct { - // Log level - Verbosity string `json:"verbosity"` +type ConfigLogging struct { + Verbosity string `json:"verbosity"` + LogPath *string `json:"logPath,omitempty"` } diff --git a/mongodb-community-operator/pkg/tls/tls.go b/mongodb-community-operator/pkg/tls/tls.go new file mode 100644 index 000000000..77c49ed2a --- /dev/null +++ b/mongodb-community-operator/pkg/tls/tls.go @@ -0,0 +1,113 @@ +package tls + +import ( + "context" + "crypto/sha256" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/types" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/secret" +) + +const ( + CAMountPath = "/var/lib/tls/ca/" + OperatorSecretMountPath = "/var/lib/tls/server/" //nolint + + tlsSecretCertName = "tls.crt" + tlsSecretKeyName = "tls.key" + tlsSecretPemName = "tls.pem" +) + +type TLSConfigurableResource interface { + metav1.Object + TLSSecretNamespacedName() types.NamespacedName + TLSOperatorSecretNamespacedName() types.NamespacedName +} + +// ensureTLSSecret will create or update the operator-managed Secret containing +// the concatenated certificate and key from the user-provided Secret. +// Returns the file name of the concatenated certificate and key +func EnsureTLSSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, resource TLSConfigurableResource) (string, error) { + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, resource.TLSSecretNamespacedName()) + if err != nil { + return "", err + } + // Calculate file name from certificate and key + fileName := OperatorSecretFileName(certKey) + + operatorSecret := secret.Builder(). + SetName(resource.TLSOperatorSecretNamespacedName().Name). + SetNamespace(resource.TLSOperatorSecretNamespacedName().Namespace). + SetField(fileName, certKey). + SetOwnerReferences(resource.GetOwnerReferences()). + Build() + + return fileName, secret.CreateOrUpdate(ctx, getUpdateCreator, operatorSecret) +} + +// getCertAndKey will fetch the certificate and key from the user-provided Secret. +func getCertAndKey(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) string { + cert, err := secret.ReadKey(ctx, getter, tlsSecretCertName, secretName) + if err != nil { + return "" + } + + key, err := secret.ReadKey(ctx, getter, tlsSecretKeyName, secretName) + if err != nil { + return "" + } + + return combineCertificateAndKey(cert, key) +} + +// getPem will fetch the pem from the user-provided secret +func getPem(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) string { + pem, err := secret.ReadKey(ctx, getter, tlsSecretPemName, secretName) + if err != nil { + return "" + } + return pem +} + +func combineCertificateAndKey(cert, key string) string { + trimmedCert := strings.TrimRight(cert, "\n") + trimmedKey := strings.TrimRight(key, "\n") + return fmt.Sprintf("%s\n%s", trimmedCert, trimmedKey) +} + +// getPemOrConcatenatedCrtAndKey will get the final PEM to write to the secret. +// This is either the tls.pem entry in the given secret, or the concatenation +// of tls.crt and tls.key +// It performs a basic validation on the entries. +func getPemOrConcatenatedCrtAndKey(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) (string, error) { + certKey := getCertAndKey(ctx, getter, secretName) + pem := getPem(ctx, getter, secretName) + if certKey == "" && pem == "" { + return "", fmt.Errorf(`neither "%s" nor the pair "%s"/"%s" were present in the TLS secret`, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) + } + if certKey == "" { + return pem, nil + } + if pem == "" { + return certKey, nil + } + if certKey != pem { + return "", fmt.Errorf(`if all of "%s", "%s" and "%s" are present in the secret, the entry for "%s" must be equal to the concatenation of "%s" with "%s"`, tlsSecretCertName, tlsSecretKeyName, tlsSecretPemName, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) + } + return certKey, nil +} + +// OperatorSecretFileName calculates the file name to use for the mounted +// certificate-key file. The name is based on the hash of the combined cert and key. +// If the certificate or key changes, the file path changes as well which will trigger +// the agent to perform a restart. +// The user-provided secret is being watched and will trigger a reconciliation +// on changes. This enables the operator to automatically handle cert rotations. +func OperatorSecretFileName(certKey string) string { + hash := sha256.Sum256([]byte(certKey)) + return fmt.Sprintf("%x.pem", hash) +} diff --git a/pkg/telemetry/collector.go b/pkg/telemetry/collector.go index f1d0e8712..1e6a2ed09 100644 --- a/pkg/telemetry/collector.go +++ b/pkg/telemetry/collector.go @@ -21,6 +21,7 @@ import ( mdbmultiv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdbmulti" omv1 "github.com/mongodb/mongodb-kubernetes/api/v1/om" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + userv1 "github.com/mongodb/mongodb-kubernetes/api/v1/user" mcov1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/util/envvar" "github.com/mongodb/mongodb-kubernetes/pkg/images" @@ -364,25 +365,50 @@ func addCommunityEvents(ctx context.Context, operatorClusterClient kubeclient.Cl return events } +func resolveSearchSource(ctx context.Context, operatorClusterClient kubeclient.Client, source *userv1.MongoDBResourceRef) (architecture string, isEnterprise bool, ok bool) { + if source == nil { + return "external", false, true // we cheat and hijack the Architecture field to indicate this Search resource is configured with an external MongoDB source + } + + key := kubeclient.ObjectKey{Namespace: source.Namespace, Name: source.Name} + + mdb := &mdbv1.MongoDB{} + if err := operatorClusterClient.Get(ctx, key, mdb); err == nil { + return string(architectures.GetArchitecture(mdb.Annotations)), true, true + } + + mdbc := &mcov1.MongoDBCommunity{} + if err := operatorClusterClient.Get(ctx, key, mdbc); err == nil { + return "static", false, true // Community is always static + } + + return "", false, false // likely the database resource doesn't exist yet, skip telemetry for this item for now +} + func addSearchEvents(ctx context.Context, operatorClusterClient kubeclient.Client, operatorUUID string, now time.Time) []Event { var events []Event searchList := &searchv1.MongoDBSearchList{} if err := operatorClusterClient.List(ctx, searchList); err != nil { Logger.Warnf("failed to fetch MongoDBSearchList from Kubernetes: %v", err) - } else { - for _, item := range searchList.Items { - properties := DeploymentUsageSnapshotProperties{ - DeploymentUID: string(item.UID), - OperatorID: operatorUUID, - Architecture: string(architectures.Static), // Community Search is always static - IsMultiCluster: false, // Community Search doesn't support multi-cluster - Type: "Search", - IsRunningEnterpriseImage: false, // Community search doesn't run enterprise - } - if event := createEvent(properties, now, Deployments); event != nil { - events = append(events, *event) - } + return nil + } + + for _, item := range searchList.Items { + architecture, isEnterprise, ok := resolveSearchSource(ctx, operatorClusterClient, item.GetMongoDBResourceRef()) + if !ok { // search source doesn't exist yet, don't generate a telemetry event + continue + } + properties := DeploymentUsageSnapshotProperties{ + DeploymentUID: string(item.UID), + OperatorID: operatorUUID, + Architecture: architecture, + IsMultiCluster: false, // Search doesn't support multi-cluster + Type: "Search", + IsRunningEnterpriseImage: isEnterprise, + } + if event := createEvent(properties, now, Deployments); event != nil { + events = append(events, *event) } } return events diff --git a/pkg/telemetry/collector_test.go b/pkg/telemetry/collector_test.go index 9b3a8c638..f0ca25986 100644 --- a/pkg/telemetry/collector_test.go +++ b/pkg/telemetry/collector_test.go @@ -2,7 +2,9 @@ package telemetry import ( "context" + "encoding/json" "errors" + "reflect" "runtime" "testing" "time" @@ -21,6 +23,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/api/v1/mdbmulti" omv1 "github.com/mongodb/mongodb-kubernetes/api/v1/om" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + userv1 "github.com/mongodb/mongodb-kubernetes/api/v1/user" "github.com/mongodb/mongodb-kubernetes/controllers/operator/mock" mcov1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" mockClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" @@ -1157,12 +1160,17 @@ func findEventWithDeploymentUID(events []Event, deploymentUID string) *Event { type MockClient struct { client.Client MockList func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error + MockGet func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error } func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { return m.MockList(ctx, list, opts...) } +func (m *MockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return m.MockGet(ctx, key, obj, opts...) +} + func TestAddCommunityEvents(t *testing.T) { operatorUUID := "test-operator-uuid" @@ -1261,55 +1269,67 @@ func TestAddCommunityEvents(t *testing.T) { func TestAddSearchEvents(t *testing.T) { operatorUUID := "test-operator-uuid" - now := time.Now() + mdbStatic := &mdbv1.MongoDB{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "mdb-static", Annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.Static)}}} + mdbNonStatic := &mdbv1.MongoDB{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "mdb-nonstatic", Annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.NonStatic)}}} + community := &mcov1.MongoDBCommunity{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "community-db"}} + testCases := []struct { - name string - resources searchv1.MongoDBSearchList - events []DeploymentUsageSnapshotProperties + name string + searchItems []searchv1.MongoDBSearch + events []DeploymentUsageSnapshotProperties + sources map[reflect.Type][]client.Object }{ { - name: "With resources", - resources: searchv1.MongoDBSearchList{ - Items: []searchv1.MongoDBSearch{ - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID("search-1"), - Name: "test-search-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID("search-2"), - Name: "test-search-2", - }, - }, - }, + name: "External source", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-external"), Name: "search-external", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{ExternalMongoDBSource: &searchv1.ExternalMongoDBSource{}}}}, + }, + events: []DeploymentUsageSnapshotProperties{{ + DeploymentUID: "search-external", + OperatorID: operatorUUID, + Architecture: "external", + IsMultiCluster: false, + Type: "Search", + IsRunningEnterpriseImage: false, + }}, + }, + { + name: "Enterprise static and non-static", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-static"), Name: "search-static", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "mdb-static"}}}}, + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-nonstatic"), Name: "search-nonstatic", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "mdb-nonstatic"}}}}, }, events: []DeploymentUsageSnapshotProperties{ - { - DeploymentUID: "search-1", - OperatorID: operatorUUID, - Architecture: string(architectures.Static), - IsMultiCluster: false, - Type: "Search", - IsRunningEnterpriseImage: false, - }, - { - DeploymentUID: "search-2", - OperatorID: operatorUUID, - Architecture: string(architectures.Static), - IsMultiCluster: false, - Type: "Search", - IsRunningEnterpriseImage: false, - }, + {DeploymentUID: "search-static", OperatorID: operatorUUID, Architecture: string(architectures.Static), IsMultiCluster: false, Type: "Search", IsRunningEnterpriseImage: true}, + {DeploymentUID: "search-nonstatic", OperatorID: operatorUUID, Architecture: string(architectures.NonStatic), IsMultiCluster: false, Type: "Search", IsRunningEnterpriseImage: true}, + }, + sources: map[reflect.Type][]client.Object{ + reflect.TypeOf(&mdbv1.MongoDB{}): {mdbStatic, mdbNonStatic}, }, }, { - name: "With no resources", - resources: searchv1.MongoDBSearchList{}, - events: []DeploymentUsageSnapshotProperties{}, + name: "Community source", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-community"), Name: "search-community", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "community-db"}}}}, + }, + events: []DeploymentUsageSnapshotProperties{{DeploymentUID: "search-community", OperatorID: operatorUUID, Architecture: "static", IsMultiCluster: false, Type: "Search", IsRunningEnterpriseImage: false}}, + sources: map[reflect.Type][]client.Object{ + reflect.TypeOf(&mcov1.MongoDBCommunity{}): {community}, + }, + }, + { + name: "Missing underlying resource (skipped)", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-missing"), Name: "search-missing", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "does-not-exist"}}}}, + }, + events: []DeploymentUsageSnapshotProperties{}, + }, + { + name: "No search resources", + searchItems: []searchv1.MongoDBSearch{}, + events: []DeploymentUsageSnapshotProperties{}, }, } @@ -1318,10 +1338,23 @@ func TestAddSearchEvents(t *testing.T) { mc := &MockClient{ MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { if l, ok := list.(*searchv1.MongoDBSearchList); ok { - *l = tc.resources + *l = searchv1.MongoDBSearchList{Items: tc.searchItems} } return nil }, + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + objects := tc.sources[reflect.TypeOf(obj)] + for _, o := range objects { + if o.GetName() == key.Name && o.GetNamespace() == key.Namespace { + // copy the arranged object into the obj pointer like controller-runtime's pkg/client/fake does + bytes, _ := json.Marshal(o) + json.Unmarshal(bytes, obj) + return nil + } + } + + return errors.New("not found") + }, } events := addSearchEvents(context.Background(), mc, operatorUUID, now) diff --git a/public/crds.yaml b/public/crds.yaml index 0bdd5e315..4afc63e97 100644 --- a/public/crds.yaml +++ b/public/crds.yaml @@ -4071,6 +4071,8 @@ spec: spec: properties: persistence: + description: Configure MongoDB Search's persistent volume. If not + defined, the operator will request 10GB of storage. properties: multiple: properties: @@ -4117,7 +4119,8 @@ spec: type: object type: object resourceRequirements: - description: ResourceRequirements describes the compute resource requirements. + description: Configure resource requests and limits for the MongoDB + Search pods. properties: claims: description: |- @@ -4175,8 +4178,88 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + security: + description: Configure security settings of the MongoDB Search server + that MongoDB database is connecting to when performing search queries. + properties: + tls: + properties: + certificateKeySecretRef: + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object source: + description: MongoDB database connection details from which MongoDB + Search will synchronize data to build indexes. properties: + external: + properties: + hostAndPorts: + items: + type: string + type: array + keyFileSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + tls: + properties: + ca: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object mongodbResourceRef: properties: name: @@ -4186,11 +4269,26 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. + StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + which aren't exposed as fields in the MongoDBSearch.spec. properties: metadata: description: StatefulSetMetadataWrapper is a wrapper around Labels @@ -4212,6 +4310,9 @@ spec: - spec type: object version: + description: Optional version of MongoDB Search component (mongot). + If not set, then the operator will set the most appropriate version + of MongoDB Search. type: string type: object status: diff --git a/scripts/dev/contexts/e2e_mdb_community b/scripts/dev/contexts/e2e_mdb_community index 4f096c2d7..b38563179 100644 --- a/scripts/dev/contexts/e2e_mdb_community +++ b/scripts/dev/contexts/e2e_mdb_community @@ -10,3 +10,5 @@ source "${script_dir}/variables/mongodb_latest" # This variable is needed otherwise the `fetch_om_information.sh` script is called and fails the test export OM_EXTERNALLY_CONFIGURED="true" + +source "${script_dir}/variables/mongodb_search_dev" diff --git a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa index e1f13453c..886ecb8be 100644 --- a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa +++ b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa @@ -16,3 +16,5 @@ export CUSTOM_OM_VERSION export CUSTOM_MDB_VERSION=6.0.5 export CUSTOM_MDB_PREV_VERSION=5.0.7 + +source "${script_dir}/variables/mongodb_search_dev" diff --git a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa index c007157d7..f4fe1ebdb 100644 --- a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa +++ b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa @@ -17,3 +17,5 @@ export CUSTOM_OM_VERSION export CUSTOM_MDB_PREV_VERSION=6.0.16 export CUSTOM_MDB_VERSION=7.0.5 + +source "${script_dir}/variables/mongodb_search_dev" diff --git a/scripts/dev/contexts/evg-private-context b/scripts/dev/contexts/evg-private-context index dec163d8c..af2a42479 100644 --- a/scripts/dev/contexts/evg-private-context +++ b/scripts/dev/contexts/evg-private-context @@ -110,8 +110,8 @@ export CODE_SNIPPETS_COMMIT_OUTPUT=${code_snippets_commit_output:-"false"} export READINESS_PROBE_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-readinessprobe:${version_id}" export VERSION_UPGRADE_HOOK_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-operator-version-upgrade-post-start-hook:${version_id}" -# TODO to be removed at public preview stage of community-search -export COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON="${community_private_preview_pullsecret_dockerconfigjson}" +# shellcheck disable=SC2154 +export PRERELEASE_PULLSECRET_DOCKERCONFIGJSON="${community_private_preview_pullsecret_dockerconfigjson}" export cognito_user_pool_id="${cognito_user_pool_id}" export cognito_workload_federation_client_id="${cognito_workload_federation_client_id}" diff --git a/scripts/dev/contexts/root-context b/scripts/dev/contexts/root-context index 24570ac93..542190d4e 100644 --- a/scripts/dev/contexts/root-context +++ b/scripts/dev/contexts/root-context @@ -6,10 +6,12 @@ script_name=$(readlink -f "${BASH_SOURCE[0]}") script_dir=$(dirname "${script_name}") source "${script_dir}/private-context" -export PROJECT_DIR="${PWD}" +PROJECT_DIR="$(realpath "${script_dir}/../../..")" +export PROJECT_DIR export IMAGE_TYPE=ubi export UBI_IMAGE_WITHOUT_SUFFIX=true export WATCH_NAMESPACE=${WATCH_NAMESPACE:-${NAMESPACE}} +export OPERATOR_NAME="mongodb-kubernetes-operator" # # changing variables below should not be necessary @@ -35,7 +37,7 @@ fi export OPERATOR_ENV=${OPERATOR_ENV:-"dev"} -AGENT_VERSION="$(jq -r '.agentVersion' release.json)" +AGENT_VERSION="$(jq -r '.agentVersion' "${PROJECT_DIR}/release.json")" export AGENT_VERSION export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" @@ -111,7 +113,7 @@ export MDB_COMMUNITY_REPO_URL=quay.io/mongodb export MDB_COMMUNITY_AGENT_IMAGE=${AGENT_IMAGE} export MDB_COMMUNITY_IMAGE_TYPE=ubi8 -MDB_SEARCH_COMMUNITY_VERSION="$(jq -r '.search.community.version' release.json)" +MDB_SEARCH_COMMUNITY_VERSION="$(jq -r '.search.community.version' "${PROJECT_DIR}/release.json")" export MDB_SEARCH_COMMUNITY_VERSION export MDB_SEARCH_COMMUNITY_NAME="mongodb-search-community" diff --git a/scripts/dev/contexts/variables/mongodb_search_dev b/scripts/dev/contexts/variables/mongodb_search_dev new file mode 100644 index 000000000..1eda41398 --- /dev/null +++ b/scripts/dev/contexts/variables/mongodb_search_dev @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# Temporary development images built from mongot master +export MDB_SEARCH_COMMUNITY_VERSION="1.53.0-78-g227f2593f" # Sep 2nd mongot master +export MDB_SEARCH_COMMUNITY_NAME="mongot/community" +export MDB_SEARCH_COMMUNITY_REPO_URL="268558157000.dkr.ecr.eu-west-1.amazonaws.com" diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 8549b34da..7d7f51e25 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -6,6 +6,29 @@ test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x source scripts/dev/set_env_context.sh source scripts/funcs/kubernetes +usage() { + echo "Deploy local registry and create kind cluster configured to use this registry. Local Docker registry is deployed at localhost:5000. + +Usage: + setup_kind_cluster.sh [-n ] [-r] + setup_kind_cluster.sh [-h] + setup_kind_cluster.sh [-n ] [-e] [-r] + +Options: + -n (optional) Set kind cluster name to . Creates kubeconfig in ~/.kube/. The default name is 'kind' if not set. + -e (optional) Export newly created kind cluster's credentials to ~/.kube/ and set current kubectl context. + -h (optional) Shows this screen. + -r (optional) Recreate cluster if needed + -p (optional) Network reserved for Pods, e.g. 10.244.0.0/16 + -s (optional) Network reserved for Services, e.g. 10.96.0.0/16 + -l (optional) MetalLB IP range, e.g. 172.18.255.200-172.18.255.250 + -c (optional) Cluster domain. If not supplied, cluster.local will be used + -g (optional) Run docker registry before installing kind + -k (optional) Create docker network before installing kind +" + exit 0 +} + cluster_name=${CLUSTER_NAME:-"kind"} cluster_domain="cluster.local" export_kubeconfig=0 @@ -47,29 +70,6 @@ reg_name='kind-registry' reg_port='5000' kind_image="${registry}/kindest/node:v1.33.2@sha256:c55080dc5be4f2cc242e6966fdf97bb62282e1cd818a28223cf536db8b0fddf4" -usage() { - echo "Deploy local registry and create kind cluster configured to use this registry. Local Docker registry is deployed at localhost:5000. - -Usage: - setup_kind_cluster.sh [-n ] [-r] - setup_kind_cluster.sh [-h] - setup_kind_cluster.sh [-n ] [-e] [-r] - -Options: - -n (optional) Set kind cluster name to . Creates kubeconfig in ~/.kube/. The default name is 'kind' if not set. - -e (optional) Export newly created kind cluster's credentials to ~/.kube/ and set current kubectl context. - -h (optional) Shows this screen. - -r (optional) Recreate cluster if needed - -p (optional) Network reserved for Pods, e.g. 10.244.0.0/16 - -s (optional) Network reserved for Services, e.g. 10.96.0.0/16 - -l (optional) MetalLB IP range, e.g. 172.18.255.200-172.18.255.250 - -c (optional) Cluster domain. If not supplied, cluster.local will be used - -g (optional) Run docker registry before installing kind - -k (optional) Create docker network before installing kind -" - exit 0 -} - kind_delete_cluster() { kind delete cluster --name "${cluster_name}" || true } diff --git a/scripts/funcs/operator_deployment b/scripts/funcs/operator_deployment index f13b31f23..d8c7a5ed3 100644 --- a/scripts/funcs/operator_deployment +++ b/scripts/funcs/operator_deployment @@ -34,6 +34,9 @@ get_operator_helm_values() { "operator.telemetry.send.enabled=${MDB_OPERATOR_TELEMETRY_SEND_ENABLED:-false}" # lets collect and save in the configmap as frequently as we can "operator.telemetry.collection.frequency=${MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY:-1m}" + "search.community.repo=${MDB_SEARCH_COMMUNITY_REPO_URL}" + "search.community.name=${MDB_SEARCH_COMMUNITY_NAME}" + "search.community.version=${MDB_SEARCH_COMMUNITY_VERSION}" "community.registry.agent=${AGENT_BASE_REGISTRY:-${REGISTRY}}" ) @@ -67,9 +70,11 @@ get_operator_helm_values() { # shellcheck disable=SC2154 if [[ "${KUBE_ENVIRONMENT_NAME-}" = "multi" ]]; then - comma_separated_list="$(echo "${MEMBER_CLUSTERS}" | tr ' ' ',')" - # shellcheck disable=SC2154 - config+=("multiCluster.clusters={${comma_separated_list}}") + if [[ -n "${MEMBER_CLUSTERS:-}" ]]; then + comma_separated_list="$(echo "${MEMBER_CLUSTERS}" | tr ' ' ',')" + # shellcheck disable=SC2154 + config+=("multiCluster.clusters={${comma_separated_list}}") + fi config+=("operator.createOperatorServiceAccount=false") fi From 8ae2ff3d2297630f083e7a6f74c322337f14ebcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 8 Sep 2025 22:43:34 +0200 Subject: [PATCH 03/17] Search snippets --- .evergreen-functions.yml | 4 +- .evergreen-snippets.yml | 153 +++++ .evergreen-tasks.yml | 8 +- .evergreen.yml | 84 +-- .githooks/pre-commit | 5 + .gitignore | 2 + docs/community-search/quick-start/README.md | 442 ------------- .../community-search/quick-start/README.md.j2 | 206 ------ .../code_snippets/0045_create_namespaces.sh | 1 - .../code_snippets/0100_install_operator.sh | 6 - ...0_configure_community_search_pullsecret.sh | 25 - ...0210_verify_community_search_pullsecret.sh | 10 - ...5_create_mongodb_community_user_secrets.sh | 7 - .../0310_create_mongodb_community_resource.sh | 55 -- .../0315_wait_for_community_resource.sh | 6 - .../0325_wait_for_search_resource.sh | 2 - .../0330_wait_for_community_resource.sh | 2 - .../code_snippets/0335_show_running_pods.sh | 6 - .../0410_run_mongodb_tools_pod.sh | 20 - .../0420_import_movies_mflix_database.sh | 9 - .../code_snippets/0430_create_search_index.sh | 6 - .../0440_wait_for_search_index_ready.sh | 20 - .../code_snippets/9010_delete_namespace.sh | 1 - .../community_search_snippets_test.sh.run.log | 16 - .../quick-start/env_variables.sh | 16 - .../env_variables_e2e_prerelease.sh | 6 - .../quick-start/env_variables_e2e_private.sh | 8 - .../quick-start/env_variables_e2e_public.sh | 2 - ..._configure_community_search_pullsecret.out | 18 - ...210_verify_community_search_pullsecret.out | 3 - .../output/0335_show_running_pods.out | 16 - .../0440_wait_for_search_index_ready.out | 4 - docs/community-search/quick-start/test.sh | 33 - .../01_0045_create_namespaces.sh | 1 + .../01_0046_create_image_pull_secrets.sh} | 2 +- ...8_configure_prerelease_image_pullsecret.sh | 11 + .../01_0090_helm_add_mogodb_repo.sh} | 0 .../code_snippets/01_0100_install_operator.sh | 6 + .../01_0110_wait_for_operator_deployment.sh | 5 + ...5_create_mongodb_community_user_secrets.sh | 12 + ..._0310_create_mongodb_community_resource.sh | 76 +++ .../01_0315_wait_for_community_resource.sh | 7 + ...01_0320_create_mongodb_search_resource.sh} | 2 +- .../01_0325_wait_for_search_resource.sh | 3 + .../01_0330_wait_for_community_resource.sh | 3 + .../01_0335_show_running_pods.sh | 6 + .../code_snippets/01_9010_delete_namespace.sh | 1 + .../env_variables.sh | 21 + .../env_variables_e2e_prerelease.sh | 7 + .../env_variables_e2e_private.sh | 7 + .../env_variables_e2e_private_dev.sh | 36 ++ .../env_variables_e2e_public.sh | 1 + .../search/01-search-community-deploy/test.sh | 28 + .../02_0045_create_namespaces.sh | 1 + .../02_0046_create_image_pull_secrets.sh | 3 + ...8_configure_prerelease_image_pullsecret.sh | 11 + .../02_0050_create_ops_manager_resources.sh | 7 + ..._0080_helm_login_to_prerelease_registry.sh | 8 + .../02_0090_helm_add_mogodb_repo.sh | 3 + .../code_snippets/02_0100_install_operator.sh | 6 + ...2_0305_create_mongodb_database_resource.sh | 35 ++ .../02_0310_wait_for_database_resource.sh | 6 + .../02_0315_create_mongodb_users.sh | 69 +++ .../02_0320_create_mongodb_search_resource.sh | 16 + .../02_0325_wait_for_search_resource.sh | 2 + .../02_0330_wait_for_database_resource.sh | 2 + .../02_0335_show_running_pods.sh | 6 + .../code_snippets/02_9010_delete_namespace.sh | 1 + .../env_variables.sh | 37 ++ .../env_variables_e2e_prerelease.sh | 14 + .../env_variables_e2e_private.sh | 14 + .../env_variables_e2e_private_dev.sh | 36 ++ .../env_variables_e2e_public.sh | 1 + .../env_variables_prerelease.sh | 25 + .../output/02_0090_helm_add_mogodb_repo.out} | 2 +- .../output/02_0100_install_operator.out | 585 ++++++++++++++++++ .../02_0310_wait_for_database_resource.out | 13 + .../output/02_0335_show_running_pods.out | 16 + .../02-search-enterprise-deploy/test.sh | 30 + .../03_0410_run_mongodb_tools_pod.sh | 19 + .../03_0420_import_movies_mflix_database.sh | 14 + .../03_0430_create_search_index.sh | 5 + .../03_0435_create_vector_search_index.sh | 12 + .../03_0440_wait_for_search_index_ready.sh | 3 + .../03_0444_list_search_indexes.sh | 4 + .../03_0445_list_vector_search_indexes.sh | 4 + .../03_0450_execute_search_query.sh} | 7 +- .../03_0455_execute_vector_search_query.sh | 30 + .../code_snippets/03_9010_delete_namespace.sh | 1 + .../03-search-query-usage/env_variables.sh | 20 + .../03_0420_import_movies_mflix_database.out | 2 + .../03_0440_wait_for_search_index_ready.out | 1 + .../output/03_0444_list_search_indexes.out | 32 + .../03_0445_list_vector_search_indexes.out | 40 ++ .../output/03_0450_execute_search_query.out | 22 + .../03_0455_execute_vector_search_query.out | 54 ++ docs/search/03-search-query-usage/test.sh | 24 + .../code_snippets/code_snippets_cleanup.sh | 38 +- .../find_snippets_directories.sh | 14 + ...mmunity_search_snippets_render_template.sh | 7 - .../kind_search_snippets_render_template.sh | 6 + scripts/code_snippets/sample_commit_output.sh | 8 +- scripts/code_snippets/sample_test_runner.sh | 21 +- ...ask_kind_community_search_snippets_test.sh | 20 - .../01_0090_helm_add_mogodb_repo.out | 6 + .../01_0100_install_operator.out | 179 +++--- .../01_0110_wait_for_operator_deployment.out | 9 + .../01_0315_wait_for_community_resource.out | 16 + .../01_0330_wait_for_community_resource.out | 2 + .../01_0335_show_running_pods.out | 20 + .../03_0420_import_movies_mflix_database.out | 2 + .../03_0440_wait_for_search_index_ready.out | 1 + .../03_0444_list_search_indexes.out | 32 + .../03_0445_list_vector_search_indexes.out | 40 ++ .../03_0450_execute_search_query.out | 0 .../03_0455_execute_vector_search_query.out | 54 ++ .../02_0090_helm_add_mogodb_repo.out | 6 + .../02_0100_install_operator.out | 585 ++++++++++++++++++ .../02_0310_wait_for_database_resource.out | 16 + .../02_0335_show_running_pods.out | 20 + .../03_0420_import_movies_mflix_database.out | 2 + .../03_0440_wait_for_search_index_ready.out | 1 + .../03_0444_list_search_indexes.out | 32 + .../03_0445_list_vector_search_indexes.out | 40 ++ .../03_0450_execute_search_query.out | 22 + .../03_0455_execute_vector_search_query.out | 54 ++ .../test_kind_search_community_snippets.sh | 37 ++ .../test_kind_search_enterprise_snippets.sh | 35 ++ scripts/code_snippets/validate_snippets.py | 68 ++ scripts/dev/configure_container_auth.sh | 4 +- .../contexts/prerelease_kind_code_snippets | 2 + scripts/dev/contexts/private-context-template | 3 +- .../dev/contexts/private_kind_code_snippets | 3 + scripts/dev/update_docs_snippets.sh | 46 +- 134 files changed, 2952 insertions(+), 1179 deletions(-) create mode 100644 .evergreen-snippets.yml delete mode 100644 docs/community-search/quick-start/README.md delete mode 100644 docs/community-search/quick-start/README.md.j2 delete mode 100755 docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0100_install_operator.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0200_configure_community_search_pullsecret.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0210_verify_community_search_pullsecret.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0305_create_mongodb_community_user_secrets.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0310_create_mongodb_community_resource.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0315_wait_for_community_resource.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0325_wait_for_search_resource.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0330_wait_for_community_resource.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0335_show_running_pods.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0410_run_mongodb_tools_pod.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0420_import_movies_mflix_database.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0430_create_search_index.sh delete mode 100644 docs/community-search/quick-start/code_snippets/0440_wait_for_search_index_ready.sh delete mode 100644 docs/community-search/quick-start/code_snippets/9010_delete_namespace.sh delete mode 100644 docs/community-search/quick-start/community_search_snippets_test.sh.run.log delete mode 100644 docs/community-search/quick-start/env_variables.sh delete mode 100644 docs/community-search/quick-start/env_variables_e2e_prerelease.sh delete mode 100644 docs/community-search/quick-start/env_variables_e2e_private.sh delete mode 100644 docs/community-search/quick-start/env_variables_e2e_public.sh delete mode 100644 docs/community-search/quick-start/output/0200_configure_community_search_pullsecret.out delete mode 100644 docs/community-search/quick-start/output/0210_verify_community_search_pullsecret.out delete mode 100644 docs/community-search/quick-start/output/0335_show_running_pods.out delete mode 100644 docs/community-search/quick-start/output/0440_wait_for_search_index_ready.out delete mode 100755 docs/community-search/quick-start/test.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0045_create_namespaces.sh rename docs/{community-search/quick-start/code_snippets/0046_create_image_pull_secrets.sh => search/01-search-community-deploy/code_snippets/01_0046_create_image_pull_secrets.sh} (68%) mode change 100755 => 100644 create mode 100644 docs/search/01-search-community-deploy/code_snippets/01_0048_configure_prerelease_image_pullsecret.sh rename docs/{community-search/quick-start/code_snippets/090_helm_add_mogodb_repo.sh => search/01-search-community-deploy/code_snippets/01_0090_helm_add_mogodb_repo.sh} (100%) mode change 100644 => 100755 create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0100_install_operator.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0110_wait_for_operator_deployment.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0305_create_mongodb_community_user_secrets.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0310_create_mongodb_community_resource.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0315_wait_for_community_resource.sh rename docs/{community-search/quick-start/code_snippets/0320_create_mongodb_search_resource.sh => search/01-search-community-deploy/code_snippets/01_0320_create_mongodb_search_resource.sh} (69%) mode change 100644 => 100755 create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0325_wait_for_search_resource.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0330_wait_for_community_resource.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_0335_show_running_pods.sh create mode 100755 docs/search/01-search-community-deploy/code_snippets/01_9010_delete_namespace.sh create mode 100644 docs/search/01-search-community-deploy/env_variables.sh create mode 100644 docs/search/01-search-community-deploy/env_variables_e2e_prerelease.sh create mode 100644 docs/search/01-search-community-deploy/env_variables_e2e_private.sh create mode 100644 docs/search/01-search-community-deploy/env_variables_e2e_private_dev.sh create mode 100644 docs/search/01-search-community-deploy/env_variables_e2e_public.sh create mode 100755 docs/search/01-search-community-deploy/test.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0045_create_namespaces.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0046_create_image_pull_secrets.sh create mode 100644 docs/search/02-search-enterprise-deploy/code_snippets/02_0048_configure_prerelease_image_pullsecret.sh create mode 100644 docs/search/02-search-enterprise-deploy/code_snippets/02_0050_create_ops_manager_resources.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0080_helm_login_to_prerelease_registry.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0090_helm_add_mogodb_repo.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0100_install_operator.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0305_create_mongodb_database_resource.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0310_wait_for_database_resource.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0315_create_mongodb_users.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0320_create_mongodb_search_resource.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0325_wait_for_search_resource.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0330_wait_for_database_resource.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_0335_show_running_pods.sh create mode 100755 docs/search/02-search-enterprise-deploy/code_snippets/02_9010_delete_namespace.sh create mode 100644 docs/search/02-search-enterprise-deploy/env_variables.sh create mode 100644 docs/search/02-search-enterprise-deploy/env_variables_e2e_prerelease.sh create mode 100644 docs/search/02-search-enterprise-deploy/env_variables_e2e_private.sh create mode 100644 docs/search/02-search-enterprise-deploy/env_variables_e2e_private_dev.sh create mode 100644 docs/search/02-search-enterprise-deploy/env_variables_e2e_public.sh create mode 100644 docs/search/02-search-enterprise-deploy/env_variables_prerelease.sh rename docs/{community-search/quick-start/output/090_helm_add_mogodb_repo.out => search/02-search-enterprise-deploy/output/02_0090_helm_add_mogodb_repo.out} (85%) create mode 100644 docs/search/02-search-enterprise-deploy/output/02_0100_install_operator.out create mode 100644 docs/search/02-search-enterprise-deploy/output/02_0310_wait_for_database_resource.out create mode 100644 docs/search/02-search-enterprise-deploy/output/02_0335_show_running_pods.out create mode 100755 docs/search/02-search-enterprise-deploy/test.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0410_run_mongodb_tools_pod.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0420_import_movies_mflix_database.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0430_create_search_index.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0435_create_vector_search_index.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0440_wait_for_search_index_ready.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0444_list_search_indexes.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_0445_list_vector_search_indexes.sh rename docs/{community-search/quick-start/code_snippets/0450_execute_search_query.sh => search/03-search-query-usage/code_snippets/03_0450_execute_search_query.sh} (67%) mode change 100644 => 100755 create mode 100644 docs/search/03-search-query-usage/code_snippets/03_0455_execute_vector_search_query.sh create mode 100755 docs/search/03-search-query-usage/code_snippets/03_9010_delete_namespace.sh create mode 100644 docs/search/03-search-query-usage/env_variables.sh create mode 100644 docs/search/03-search-query-usage/output/03_0420_import_movies_mflix_database.out create mode 100644 docs/search/03-search-query-usage/output/03_0440_wait_for_search_index_ready.out create mode 100644 docs/search/03-search-query-usage/output/03_0444_list_search_indexes.out create mode 100644 docs/search/03-search-query-usage/output/03_0445_list_vector_search_indexes.out create mode 100644 docs/search/03-search-query-usage/output/03_0450_execute_search_query.out create mode 100644 docs/search/03-search-query-usage/output/03_0455_execute_vector_search_query.out create mode 100755 docs/search/03-search-query-usage/test.sh create mode 100644 scripts/code_snippets/find_snippets_directories.sh delete mode 100755 scripts/code_snippets/kind_community_search_snippets_render_template.sh create mode 100755 scripts/code_snippets/kind_search_snippets_render_template.sh delete mode 100755 scripts/code_snippets/task_kind_community_search_snippets_test.sh create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0090_helm_add_mogodb_repo.out rename docs/community-search/quick-start/output/0100_install_operator.out => scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out (80%) create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0110_wait_for_operator_deployment.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0315_wait_for_community_resource.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0330_wait_for_community_resource.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0335_show_running_pods.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0420_import_movies_mflix_database.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0440_wait_for_search_index_ready.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0444_list_search_indexes.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0445_list_vector_search_indexes.out rename docs/community-search/quick-start/output/0450_execute_search_query.out => scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0450_execute_search_query.out (100%) create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0455_execute_vector_search_query.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0090_helm_add_mogodb_repo.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0100_install_operator.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0310_wait_for_database_resource.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0335_show_running_pods.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0420_import_movies_mflix_database.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0440_wait_for_search_index_ready.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0444_list_search_indexes.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0445_list_vector_search_indexes.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0450_execute_search_query.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0455_execute_vector_search_query.out create mode 100755 scripts/code_snippets/tests/test_kind_search_community_snippets.sh create mode 100755 scripts/code_snippets/tests/test_kind_search_enterprise_snippets.sh create mode 100755 scripts/code_snippets/validate_snippets.py diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 12c52e62b..373938a82 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -817,7 +817,7 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/code_snippets/sample_commit_output.sh - # it executes a script by convention: ./scripts/code_snippets/${task_name}_test.sh + # it executes a script by convention: ./scripts/code_snippets/tests/${task_name} test_code_snippets: - *switch_context - command: shell.exec @@ -830,7 +830,7 @@ functions: - code_snippets_reset - task_name script: | - ./scripts/code_snippets/${task_name}_test.sh + ./scripts/code_snippets/tests/${task_name} # # kubectl mongodb plugin release functions diff --git a/.evergreen-snippets.yml b/.evergreen-snippets.yml new file mode 100644 index 000000000..2acaa2e0d --- /dev/null +++ b/.evergreen-snippets.yml @@ -0,0 +1,153 @@ +variables: + - &setup_and_teardown_group_gke_code_snippets + setup_task_can_fail_task: true + setup_group: + - func: clone + - func: setup_gcloud_cli + - func: setup_mongosh + - func: download_kube_tools + - func: build_multi_cluster_binary + teardown_task: + - func: upload_e2e_logs + - func: upload_code_snippets_logs + + - &setup_and_teardown_group_kind_code_snippets + setup_task_can_fail_task: true + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: cleanup_exec_environment + - func: download_kube_tools + - func: configure_docker_auth + - func: setup_kubernetes_environment + - func: python_venv + - func: setup_cloud_qa + teardown_task: + - func: upload_e2e_logs + - func: upload_code_snippets_logs + - func: teardown_kubernetes_environment + - func: teardown_cloud_qa + +# This variable is copied over from .evergreen.yml because anchors don't work for included files + - &base_om8_dependency + depends_on: + - name: build_om_images + variant: build_om80_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + +tasks: + # Code snippets tasks + # Each scripts/code_snippets/tests/test_*.sh should have its task defined here. + # TODO: it should be autogenerated + # Each task executes test_code_snippets functon executes scripts/code_snippets/tests/${task_name} by task name convention + - name: test_gke_multi_cluster_snippets.sh + tags: [ "code_snippets" ] + commands: + - func: test_code_snippets + - func: sample_commit_output + + - name: test_gke_multi_cluster_no_mesh_snippets.sh + tags: [ "code_snippets" ] + commands: + - func: test_code_snippets + - func: sample_commit_output + + - name: test_kind_search_community_snippets.sh + tags: [ "code_snippets", "patch-run" ] + commands: + - func: test_code_snippets + - func: sample_commit_output + + - name: test_kind_search_enterprise_snippets.sh + tags: [ "code_snippets", "patch-run" ] + commands: + - func: test_code_snippets + - func: sample_commit_output + +task_groups: + - name: gke_code_snippets_task_group + <<: *setup_and_teardown_group_gke_code_snippets + max_hosts: -1 + tasks: + - test_gke_multi_cluster_snippets.sh + - test_gke_multi_cluster_no_mesh_snippets.sh + + - name: kind_code_snippets_task_group + <<: *setup_and_teardown_group_kind_code_snippets + max_hosts: -1 + tasks: + - test_kind_search_community_snippets.sh + - test_kind_search_enterprise_snippets.sh + +buildvariants: + # These variants are used to test the code snippets and each one can be used in patches + # Prerelease is especially used when the repo is tagged + # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe + - name: public_gke_code_snippets + display_name: public_gke_code_snippets + allowed_requesters: [ "patch" ] + run_on: + - ubuntu2204-small + tasks: + - name: gke_code_snippets_task_group + + - name: prerelease_gke_code_snippets + display_name: prerelease_gke_code_snippets + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - variant: release_images + name: '*' + patch_optional: true + run_on: + - ubuntu2204-small + tasks: + - name: gke_code_snippets_task_group + + - name: private_gke_code_snippets + display_name: private_gke_code_snippets + allowed_requesters: [ "patch" ] + run_on: + - ubuntu2204-small + <<: *base_om8_dependency + tasks: + - name: gke_code_snippets_task_group + + - name: private_kind_code_snippets + display_name: private_kind_code_snippets + tags: [ "e2e_test_suite" ] + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: kind_code_snippets_task_group + + - name: prerelease_kind_code_snippets + display_name: prerelease_kind_code_snippets + allowed_requesters: [ "patch" ] + run_on: + - ubuntu2204-large + tasks: + - name: kind_code_snippets_task_group + + - name: public_kind_code_snippets + display_name: public_kind_code_snippets + allowed_requesters: [ "patch" ] + run_on: + - ubuntu2204-large + tasks: + - name: kind_code_snippets_task_group diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml index 6ec158a41..154defd76 100644 --- a/.evergreen-tasks.yml +++ b/.evergreen-tasks.yml @@ -86,7 +86,13 @@ tasks: - func: test_code_snippets - func: sample_commit_output - - name: task_kind_community_search_snippets + - name: task_kind_search_community_snippets + tags: [ "code_snippets", "patch-run" ] + commands: + - func: test_code_snippets + - func: sample_commit_output + + - name: task_kind_search_enterprise_snippets tags: [ "code_snippets", "patch-run" ] commands: - func: test_code_snippets diff --git a/.evergreen.yml b/.evergreen.yml index cde2a1674..dd3d3eaed 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -5,6 +5,7 @@ include: - filename: .evergreen-functions.yml - filename: .evergreen-tasks.yml - filename: .evergreen-mco.yml + - filename: .evergreen-snippets.yml - filename: .evergreen-release.yml variables: @@ -87,29 +88,6 @@ variables: - func: setup_building_host - func: build_multi_cluster_binary - - &setup_and_teardown_group_gke_code_snippets - setup_task_can_fail_task: true - setup_group: - - func: clone - - func: setup_gcloud_cli - - func: setup_mongosh - - func: download_kube_tools - - func: build_multi_cluster_binary - teardown_group: - - func: upload_code_snippets_logs - - - &setup_and_teardown_group_kind_code_snippets - setup_task_can_fail_task: true - setup_group: - - func: clone - - func: cleanup_exec_environment - - func: download_kube_tools - - func: configure_docker_auth - - func: setup_kubernetes_environment - teardown_task: - - func: upload_e2e_logs - - func: upload_code_snippets_logs - - &setup_and_teardown_task_cloudqa setup_task_can_fail_task: true setup_task: @@ -173,6 +151,7 @@ variables: - name: build_init_om_images_ubi variant: init_test_run + # Any change to base_om8_dependency should be reflected to its copy in .evergreen-snippets.yml - &base_om8_dependency depends_on: - name: build_om_images @@ -580,19 +559,6 @@ task_groups: - unit_tests_helm - sbom_tests - - name: gke_code_snippets_task_group - <<: *setup_and_teardown_group_gke_code_snippets - max_hosts: -1 - tasks: - - task_gke_multi_cluster_snippets - - task_gke_multi_cluster_no_mesh_snippets - - - name: kind_code_snippets_task_group - <<: *setup_and_teardown_group_kind_code_snippets - max_hosts: -1 - tasks: - - task_kind_community_search_snippets - # Task group for deploying mongodbcommunity resources and testing the (former) MCO - name: e2e_mdb_community_task_group max_hosts: -1 @@ -1833,52 +1799,6 @@ buildvariants: tasks: - name: release_all_currently_used_agents_on_ecr - # These variants are used to test the code snippets and each one can be used in patches - # Prerelease is especially used when the repo is tagged - # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe - - name: public_gke_code_snippets - display_name: public_gke_code_snippets - allowed_requesters: [ "patch" ] - run_on: - - ubuntu2404-small - tasks: - - name: gke_code_snippets_task_group - - - name: private_gke_code_snippets - display_name: private_gke_code_snippets - allowed_requesters: [ "patch" ] - run_on: - - ubuntu2404-small - <<: *base_om8_dependency - tasks: - - name: gke_code_snippets_task_group - - - name: private_kind_code_snippets - display_name: private_kind_code_snippets - tags: [ "e2e_test_suite" ] - allowed_requesters: [ "patch", "github_pr" ] - run_on: - - ubuntu2404-large - <<: *base_om8_dependency - tasks: - - name: kind_code_snippets_task_group - - - name: prerelease_kind_code_snippets - display_name: prerelease_kind_code_snippets - allowed_requesters: [ "patch" ] - run_on: - - ubuntu2404-large - tasks: - - name: kind_code_snippets_task_group - - - name: public_kind_code_snippets - display_name: public_kind_code_snippets - allowed_requesters: [ "patch" ] - run_on: - - ubuntu2404-large - tasks: - - name: kind_code_snippets_task_group - ### Build variants for manual patch only - name: backup_csv_images diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 681e2e229..e885c25f5 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -163,6 +163,10 @@ lint_code() { scripts/evergreen/lint_code.sh } +function validate_snippets() { + scripts/code_snippets/validate_snippets.py +} + # bg_job_ vars are global; run_job_in_background function is appending to them on each call bg_job_pids=() bg_job_pids_with_names=() @@ -217,6 +221,7 @@ pre_commit() { run_job_in_background "regenerate_public_rbac_multi_cluster" run_job_in_background "python_formatting" run_job_in_background "check_erroneous_kubebuilder_annotations" + run_job_in_background "validate_snippets" if wait_for_all_background_jobs; then echo -e "${GREEN}pre-commit: All checks passed!${NO_COLOR}" diff --git a/.gitignore b/.gitignore index b2fc0d5e4..be38e569d 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,5 @@ docs/**/test.sh.run.log # goreleaser generated files dist +logs +*.run.log diff --git a/docs/community-search/quick-start/README.md b/docs/community-search/quick-start/README.md deleted file mode 100644 index 3934db1ab..000000000 --- a/docs/community-search/quick-start/README.md +++ /dev/null @@ -1,442 +0,0 @@ -# MongoDB Community Search on Kubernetes - Quick Start - -This guide provides instructions for deploying MongoDB Community Edition along with its Search capabilities onto a Kubernetes cluster. By following these steps, you will set up a MongoDB instance and configure search indexes to perform full-text search queries against your data. - -## Prerequisites - -Community Search is currently in private preview, and access to the image requires a secret to pull the search container image from Quay.io. This secret is specified during the first step of the process below, and must be obtained from MongoDB when requesting access to the private preview. - -Before you begin, ensure you have the following tools and configurations in place: - -- **Kubernetes cluster**: A running Kubernetes cluster (e.g., Minikube, Kind, GKE, EKS, AKS). -- **kubectl**: The Kubernetes command-line tool, configured to communicate with your cluster. -- **Helm**: The package manager for Kubernetes, used here to install the MongoDB Kubernetes Operator. -- **Bash 5.1+**: All shell commands in this guide are intended to be run in Bash. Scripts in this guide are automatically tested on Linux with Bash 5.1. - -## Setup Steps - -The following steps guide you through deploying MongoDB Community with Search. Each step provides a shell script. -**It is important to first source the `env_variables.sh` script provided and customize its values for your environment.** -The subsequent script snippets rely on the environment variables defined in `env_variables.sh`. You should copy and paste each script into your Bash terminal. - -### 1. Configure Environment Variables - -First, you need to set up your environment. The `env_variables.sh` script, shown below, contains variables for the subsequent steps. You should create this file locally or use the linked one. - -Download or copy the content of `env_variables.sh`: -[env_variables.sh](env_variables.sh) -```shell copy -# set it to the context name of the k8s cluster -export K8S_CLUSTER_0_CONTEXT_NAME="" - -# At the private preview stage the community search image is accessible only from a private repository. -# Please contact MongoDB Support to get access. -export PRIVATE_PREVIEW_IMAGE_PULLSECRET="<.dockerconfigjson>" - -# the following namespace will be created if not exists -export MDB_NAMESPACE="mongodb" - -export MDB_ADMIN_USER_PASSWORD="admin-user-password-CHANGE-ME" -export MDB_SEARCH_USER_PASSWORD="search-user-password-CHANGE-ME" - -export OPERATOR_HELM_CHART="mongodb/mongodb-kubernetes" -# comma-separated key=value pairs for additional parameters passed to the helm-chart installing the operator -export OPERATOR_ADDITIONAL_HELM_VALUES="" -``` -This will load the variables into your current shell session, making them available for the commands in the following steps. - -### 2. Add MongoDB Helm Repository - -First, add the MongoDB Helm repository. This repository contains the Helm chart required to install the MongoDB Kubernetes Operator. The operator automates the deployment and management of MongoDB instances (both Community and Enterprise editions) on Kubernetes. - -[code_snippets/090_helm_add_mogodb_repo.sh](code_snippets/090_helm_add_mogodb_repo.sh) -```shell copy -helm repo add mongodb https://mongodb.github.io/helm-charts -helm repo update mongodb -helm search repo mongodb/mongodb-kubernetes -``` - -### 3. Install MongoDB Kubernetes Operator - -Next, install the MongoDB Kubernetes Operator from the Helm repository you just added. The Operator will watch for MongoDBCommunity and MongoDBSearch custom resources and manage the lifecycle of your MongoDB deployments. - -[code_snippets/0100_install_operator.sh](code_snippets/0100_install_operator.sh) -```shell copy -helm upgrade --install --debug --kube-context "${K8S_CLUSTER_0_CONTEXT_NAME}" \ - --create-namespace \ - --namespace="${MDB_NAMESPACE}" \ - mongodb-kubernetes \ - --set "${OPERATOR_ADDITIONAL_HELM_VALUES:-"dummy=value"}" \ - "${OPERATOR_HELM_CHART}" -``` -This command installs the operator in the `mongodb` namespace (creating it if it doesn't exist) and names the release `community-operator`. - -### 4. Configure Pull Secret for MongoDB Community Search - -To use MongoDB Search, your Kubernetes cluster needs to pull the necessary container images. This step creates a Kubernetes secret named `community-private-preview-pullsecret`. This secret stores the credentials required to access the image repository for MongoDB Search. The script then patches the `mongodb-kubernetes-database-pods` service account to include this pull secret, allowing pods managed by this service account to pull the required images. - -[code_snippets/0200_configure_community_search_pullsecret.sh](code_snippets/0200_configure_community_search_pullsecret.sh) -```shell copy -kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - < /tmp/mdb_script.js -mongosh --quiet "mongodb://search-user:${MDB_SEARCH_USER_PASSWORD}@mdbc-rs-0.mdbc-rs-svc.${MDB_NAMESPACE}.svc.cluster.local:27017/?replicaSet=mdbc-rs" < /tmp/mdb_script.js -EOF -)" -``` \ No newline at end of file diff --git a/docs/community-search/quick-start/README.md.j2 b/docs/community-search/quick-start/README.md.j2 deleted file mode 100644 index 0e3125075..000000000 --- a/docs/community-search/quick-start/README.md.j2 +++ /dev/null @@ -1,206 +0,0 @@ -# MongoDB Community Search on Kubernetes - Quick Start - -This guide provides instructions for deploying MongoDB Community Edition along with its Search capabilities onto a Kubernetes cluster. By following these steps, you will set up a MongoDB instance and configure search indexes to perform full-text search queries against your data. - -## Prerequisites - -Community Search is currently in private preview, and access to the image requires a secret to pull the search container image from Quay.io. This secret is specified during the first step of the process below, and must be obtained from MongoDB when requesting access to the private preview. - -Before you begin, ensure you have the following tools and configurations in place: - -- **Kubernetes cluster**: A running Kubernetes cluster (e.g., Minikube, Kind, GKE, EKS, AKS). -- **kubectl**: The Kubernetes command-line tool, configured to communicate with your cluster. -- **Helm**: The package manager for Kubernetes, used here to install the MongoDB Kubernetes Operator. -- **Bash 5.1+**: All shell commands in this guide are intended to be run in Bash. Scripts in this guide are automatically tested on Linux with Bash 5.1. - -## Setup Steps - -The following steps guide you through deploying MongoDB Community with Search. Each step provides a shell script. -**It is important to first source the `env_variables.sh` script provided and customize its values for your environment.** -The subsequent script snippets rely on the environment variables defined in `env_variables.sh`. You should copy and paste each script into your Bash terminal. - -### 1. Configure Environment Variables - -First, you need to set up your environment. The `env_variables.sh` script, shown below, contains variables for the subsequent steps. You should create this file locally or use the linked one. - -Download or copy the content of `env_variables.sh`: -[env_variables.sh](env_variables.sh) -```shell copy -{% include "env_variables.sh" %} -``` -This will load the variables into your current shell session, making them available for the commands in the following steps. - -### 2. Add MongoDB Helm Repository - -First, add the MongoDB Helm repository. This repository contains the Helm chart required to install the MongoDB Kubernetes Operator. The operator automates the deployment and management of MongoDB instances (both Community and Enterprise editions) on Kubernetes. - -[code_snippets/090_helm_add_mogodb_repo.sh](code_snippets/090_helm_add_mogodb_repo.sh) -```shell copy -{% include "code_snippets/090_helm_add_mogodb_repo.sh" %} -``` - -### 3. Install MongoDB Kubernetes Operator - -Next, install the MongoDB Kubernetes Operator from the Helm repository you just added. The Operator will watch for MongoDBCommunity and MongoDBSearch custom resources and manage the lifecycle of your MongoDB deployments. - -[code_snippets/0100_install_operator.sh](code_snippets/0100_install_operator.sh) -```shell copy -{% include "code_snippets/0100_install_operator.sh" %} -``` -This command installs the operator in the `mongodb` namespace (creating it if it doesn't exist) and names the release `community-operator`. - -### 4. Configure Pull Secret for MongoDB Community Search - -To use MongoDB Search, your Kubernetes cluster needs to pull the necessary container images. This step creates a Kubernetes secret named `community-private-preview-pullsecret`. This secret stores the credentials required to access the image repository for MongoDB Search. The script then patches the `mongodb-kubernetes-database-pods` service account to include this pull secret, allowing pods managed by this service account to pull the required images. - -[code_snippets/0200_configure_community_search_pullsecret.sh](code_snippets/0200_configure_community_search_pullsecret.sh) -```shell copy -{% include "code_snippets/0200_configure_community_search_pullsecret.sh" %} -``` -This script creates a `community-private-preview-pullsecret` secret in your Kubernetes namespace and associates it with the service account used for MongoDB pods. - -### 5. Verify Pull Secret Configuration - -Confirm that the `community-private-preview-pullsecret` has been successfully added to the `mongodb-kubernetes-database-pods` service account. This ensures that Kubernetes can authenticate with the container registry when pulling images for MongoDB Search pods. - -[code_snippets/0210_verify_community_search_pullsecret.sh](code_snippets/0210_verify_community_search_pullsecret.sh) -```shell copy -{% include "code_snippets/0210_verify_community_search_pullsecret.sh" %} -``` -This command checks the `mongodb-kubernetes-database-pods` service account to confirm the presence of `community-private-preview-pullsecret`. - -## Creating a MongoDB Community Search Deployment - -With the prerequisites and initial setup complete, you can now deploy MongoDB Community Edition and enable Search. - -### 6. Create MongoDB User Secrets - -MongoDB requires authentication for secure access. This step creates two Kubernetes secrets: `admin-user-password` and `search-user-password`. These secrets store the credentials for the MongoDB administrative user and a dedicated search user, respectively. These secrets will be mounted into the MongoDB pods. - -[code_snippets/0305_create_mongodb_community_user_secrets.sh](code_snippets/0305_create_mongodb_community_user_secrets.sh) -```shell copy -{% include "code_snippets/0305_create_mongodb_community_user_secrets.sh" %} -``` -Ensure these secrets are created in the same namespace where you plan to deploy MongoDB. - -### 7. Create MongoDB Community Resource - -Now, deploy MongoDB Community by creating a `MongoDBCommunity` custom resource named `mdbc-rs`. This resource definition instructs the MongoDB Kubernetes Operator to configure a MongoDB replica set with 3 members, running version 8.0.6. MongoDB Community Search is supported only from MongoDB Community Server version 8.0. It also defines CPU and memory resources for the `mongod` and `mongodb-agent` containers, and sets up two users (`admin-user` and `search-user`) with their respective roles and password secrets. User `search-user` will be used to restore, connect and perform search queries on the `sample_mflix` database. - -[code_snippets/0310_create_mongodb_community_resource.sh](code_snippets/0310_create_mongodb_community_resource.sh) -```yaml copy -{% include "code_snippets/0310_create_mongodb_community_resource.sh" %} -``` - -### 8. Wait for MongoDB Community Resource to be Ready - -After applying the `MongoDBCommunity` custom resource, the operator begins deploying the MongoDB nodes (pods). This step uses `kubectl wait` to pause execution until the `mdbc-rs` resource's status phase becomes `Running`, indicating that the MongoDB Community replica set is operational. - -[code_snippets/0315_wait_for_community_resource.sh](code_snippets/0315_wait_for_community_resource.sh) -```shell copy -{% include "code_snippets/0315_wait_for_community_resource.sh" %} -``` - -### 9. Create MongoDB Search Resource - -Once your MongoDB deployment is ready, enable Search capabilities by creating a `MongoDBSearch` custom resource, also named `mdbc-rs` to associate it with the MongoDB instance. This resource specifies the CPU and memory resource requirements for the search nodes. - -Note: Private preview of MongoDB Community Search comes with some limitations, and it is not suitable for production use: -* TLS cannot be enabled in MongoDB Community deployment (MongoD communicates with MongoT with plain text). -* Only one node of search node is supported (load balancing not supported) - -[code_snippets/0320_create_mongodb_search_resource.sh](code_snippets/0320_create_mongodb_search_resource.sh) -```shell copy -{% include "code_snippets/0320_create_mongodb_search_resource.sh" %} -``` - -The `MongoDBSearch.spec` fields are supported: -* `spec.source.mongodbResourceRef.name` - omitted in the example as the MongoDBSearch CR has the same name as MongoDBCommunity CR allowing to integrate both using naming convention. While keeping the same name is recommended (you cannot have more than one MongoDBSearch resources referencing the same MongoDBCommunity resource - it's 1:1 relationship) it's not enforced. The name can be different, but then you must explicitly point to the MongoDBCommunity you would like to enable search in. Note that you enable search capabilities by deploying search component (with MongoDBSearch CR) and nothing is necessary to define in MongoDBCommunity CR to configure it for search - it will be configured automatically by recognising there is related MongoDBSearch pointing to it. -* `spec.version`: Version of mongodb-community-search. By default, the operator chooses the MongoDB Search version automatically, but it is possible to specify it explicitly. Currently, the default value is `1.47.0`. -* `spec.statefulSet`: Optional statefulset overrides, which are applied last to the mongot's statefulset. It is possible to adjust any statefulset configuration that was create by the operator (the overrides are applied last). The type of the field is [apps/v1/StatefulSet](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#statefulset-v1-apps) and both `spec.statefulSet.spec` and `spec.statefulSet.metadata` fields are supported. -* `spec.persistence.single`: optional storage configuration for MongoDB Search persistence volume containing storing search indexes. See [here](https://www.mongodb.com/docs/kubernetes/current/reference/k8s-operator-specification/#mongodb-setting-spec.podSpec.persistence.single) for more information about storage settings. MongoDBSearch reuses the same persistence type as in other custom resources (e.g. `MongoDB`), but supports only `single` persistence field. If not set, the operator sets `spec.persistence.single.storage = 10G`. -* `spec.resourceRequirements` - resource requests and limits for mongodb-search container. It's recommended to use this field to customize resource allocations instead of overriding it via `spec.statefulSet` overrides. If not set, the operator sets the following values (no limits, only requests): -```yaml -requests: - cpu: 2 - memory: 2G -``` - -### 10. Wait for Search Resource to be Ready - -Similar to the MongoDB deployment, the Search deployment needs time to initialize. This step uses `kubectl wait` to pause until the `MongoDBSearch` resource `mdbc-rs` reports a `Running` status in its `.status.phase` field, indicating that the search nodes are operational and integrated. - -[code_snippets/0325_wait_for_search_resource.sh](code_snippets/0325_wait_for_search_resource.sh) -```shell copy -{% include "code_snippets/0325_wait_for_search_resource.sh" %} -``` -This command polls the status of the `MongoDBSearch` resource `mdbc-rs`. - -### 11. Verify MongoDB Community Resource Status - -Double-check the status of your `MongoDBCommunity` resource to ensure it remains healthy and that the integration with the Search resource is reflected if applicable. - -[code_snippets/0330_wait_for_community_resource.sh](code_snippets/0330_wait_for_community_resource.sh) -```shell copy -{% include "code_snippets/0330_wait_for_community_resource.sh" %} -``` -This provides a final confirmation that the core database is operational. - -### 12. List Running Pods - -View all the running pods in your namespace. You should see pods for the MongoDB replica set members, the MongoDB Kubernetes Operator, and the MongoDB Search nodes. - -[code_snippets/0335_show_running_pods.sh](code_snippets/0335_show_running_pods.sh) -```shell copy -{% include "code_snippets/0335_show_running_pods.sh" %} -``` - -## Using MongoDB Search - -Now that your MongoDB Community database with Search is deployed, you can start using its search capabilities. - -### 13. Deploy MongoDB Tools Pod - -To interact with your MongoDB deployment, this step deploys a utility pod named `mongodb-tools-pod`. This pod runs a MongoDB Community Server image and is kept running with a `sleep infinity` command, allowing you to use `kubectl exec` to run MongoDB client tools like `mongosh` and `mongorestore` from within the Kubernetes cluster. Running steps in a pod inside the cluster simplifies connectivity to mongodb without neeeding to expose the database externally (provided steps directly connect to the *.cluster.local hostnames). - -[code_snippets/0410_run_mongodb_tools_pod.sh](code_snippets/0410_run_mongodb_tools_pod.sh) -```shell copy -{% include "code_snippets/0410_run_mongodb_tools_pod.sh" %} -``` - -### 14. Import Sample Data - -To test the search functionality, this step imports the `sample_mflix.movies` collection. It downloads the sample dataset and uses `mongorestore` to load the data into the `sample_mflix` database in your MongoDB deployment, connecting as the `search-user`. - -[code_snippets/0420_import_movies_mflix_database.sh](code_snippets/0420_import_movies_mflix_database.sh) -```shell copy -{% include "code_snippets/0420_import_movies_mflix_database.sh" %} -``` -This command uses `mongorestore` from the `mongodb-tools-pod` to load data from the downloaded `sample_mflix.archive` file. - -### 15. Create Search Index - -Before performing search queries, create a search index. This step uses `kubectl exec` to run `mongosh` in the `mongodb-tools-pod`. It connects to the `sample_mflix` database as `search-user` and calls `db.movies.createSearchIndex()` to create a search index named "default" with dynamic mappings on the `movies` collection. Dynamic mapping automatically indexes all fields with supported types. MongoDB Search offers flexible index definitions, allowing for dynamic and static field mappings, various analyzer types (standard, language-specific, custom), and features like synonyms and faceted search. - -[code_snippets/0430_create_search_index.sh](code_snippets/0430_create_search_index.sh) -```shell copy -{% include "code_snippets/0430_create_search_index.sh" %} -``` - -### 16. Wait for Search Index to be Ready - -Creating a search index is an asynchronous operation. This script polls periodically the status by executing `db.movies.getSearchIndexes("default")`. - -[code_snippets/0440_wait_for_search_index_ready.sh](code_snippets/0440_wait_for_search_index_ready.sh) -```shell copy -{% include "code_snippets/0440_wait_for_search_index_ready.sh" %} -``` - -### 17. Execute a Search Query - -Once the search index is ready, execute search queries using the `$search` aggregation pipeline stage. MongoDB Search supports a query language, allowing for various types of queries such as text search, autocomplete, faceting, and more. You can combine `$search` with other aggregation stages to further refine and process your results. - -[code_snippets/0450_execute_search_query.sh](code_snippets/0450_execute_search_query.sh) -```shell copy -{% include "code_snippets/0450_execute_search_query.sh" %} -``` diff --git a/docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh b/docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh deleted file mode 100755 index 7d0899c63..000000000 --- a/docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh +++ /dev/null @@ -1 +0,0 @@ -kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create namespace "${MDB_NAMESPACE}" diff --git a/docs/community-search/quick-start/code_snippets/0100_install_operator.sh b/docs/community-search/quick-start/code_snippets/0100_install_operator.sh deleted file mode 100644 index 36715a6a2..000000000 --- a/docs/community-search/quick-start/code_snippets/0100_install_operator.sh +++ /dev/null @@ -1,6 +0,0 @@ -helm upgrade --install --debug --kube-context "${K8S_CLUSTER_0_CONTEXT_NAME}" \ - --create-namespace \ - --namespace="${MDB_NAMESPACE}" \ - mongodb-kubernetes \ - --set "${OPERATOR_ADDITIONAL_HELM_VALUES:-"dummy=value"}" \ - "${OPERATOR_HELM_CHART}" diff --git a/docs/community-search/quick-start/code_snippets/0200_configure_community_search_pullsecret.sh b/docs/community-search/quick-start/code_snippets/0200_configure_community_search_pullsecret.sh deleted file mode 100644 index 59310c81f..000000000 --- a/docs/community-search/quick-start/code_snippets/0200_configure_community_search_pullsecret.sh +++ /dev/null @@ -1,25 +0,0 @@ -kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - < /tmp/mdb_script.js -mongosh --quiet "mongodb://search-user:${MDB_SEARCH_USER_PASSWORD}@mdbc-rs-0.mdbc-rs-svc.${MDB_NAMESPACE}.svc.cluster.local:27017/?replicaSet=mdbc-rs" < /tmp/mdb_script.js +mongosh --quiet "${MDB_CONNECTION_STRING}" < /tmp/mdb_script.js EOF )" diff --git a/docs/search/03-search-query-usage/code_snippets/03_0455_execute_vector_search_query.sh b/docs/search/03-search-query-usage/code_snippets/03_0455_execute_vector_search_query.sh new file mode 100644 index 000000000..fb522d697 --- /dev/null +++ b/docs/search/03-search-query-usage/code_snippets/03_0455_execute_vector_search_query.sh @@ -0,0 +1,30 @@ +mdb_script=$(cat <<'EOF' +use sample_mflix; +db.embedded_movies.aggregate([ + { + "$vectorSearch": { + "index": "vector_index", + "path": "plot_embedding_voyage_3_large", + "queryVector": [-0.034731735,0.008558298,-0.0153717,-0.029912498,0.011549547,0.010261648,-0.011964999,-0.023265276,0.010303194,-0.006896493,-0.00054528,0.003926015,-0.025757983,0.027419789,0.001199616,-0.036227357,-0.005297005,0.021935832,0.010303194,-0.019193852,0.025093261,-0.040049512,-0.033900831,-0.011466458,-0.01827986,-0.0153717,0.023265276,0.007727395,0.000114249,0.005317777,-0.043871664,-0.02127111,-0.019609304,0.016368784,-0.004756918,0.003552109,0.006522586,-0.005400868,-0.015620971,-0.034565553,-0.018695312,-0.023099095,0.050851244,-0.034731735,0.004819236,0.022268193,-0.095719993,0.05517194,-0.046198189,-0.036393538,0.007187308,-0.02459472,-0.036725901,0.009472291,0.019027673,0.020938748,-0.011051006,0.027087428,0.04586583,-0.022600554,-0.05517194,0.044204023,0.01213118,0.047859997,-0.03938479,0.002928932,0.002056484,0.019443123,-0.028583053,0.013543714,0.022932915,0.011632638,0.004923099,0.000389486,0.020024756,-0.024096178,-0.022766734,0.011217186,-0.003198975,0.007104218,-0.047195274,-0.013377533,0.013294443,0.024096178,-0.056501385,-0.026755067,-0.008433662,-0.001911076,0.007976666,-0.008101301,-0.014042255,0.008641388,-0.02176965,0.010012378,-0.000607598,-0.024927082,0.024927082,-0.018612221,-0.001184036,0.005567048,0.001324251,-0.019526213,-0.023597637,0.060489718,-0.010178559,-0.019609304,0.004112968,-0.011217186,-0.031574301,-0.008766023,0.005483958,-0.061819162,-0.023431456,-0.040714234,0.015039339,0.026422706,0.016202603,0.004653055,0.041046593,-0.018030589,0.040381871,-0.002638116,0.013045172,0.004216831,0.005650138,0.027419789,0.003926015,-0.028749233,0.004798463,-0.030244859,0.063813329,0.007145763,-0.017448956,0.025591804,-0.045201108,0.010718645,0.002804297,0.014291527,0.04586583,-0.015205519,-0.021603471,-0.035230275,0.00760276,0.033236109,0.016534964,-0.043206941,-0.003115885,-0.026256526,0.005940954,0.016534964,0.024262359,-0.001630647,0.028084511,-0.012795902,0.007270399,0.001381376,-0.009763107,-0.006896493,0.008433662,-0.019360034,0.000386889,0.030411039,0.025591804,0.010469374,0.037722982,-0.001147684,-0.005400868,0.052845411,-0.052513052,0.00768585,-0.004299921,0.00922302,0.011881908,0.012962082,-0.068798743,0.003593654,0.020938748,-0.013792985,-0.034565553,-0.007519669,-0.04021569,-0.020689478,0.006273315,0.046862911,0.006107135,0.002638116,-0.013792985,-0.005400868,-0.020274026,0.007644305,-0.010801735,0.026422706,0.043871664,0.003780607,0.010261648,-0.064145692,0.011881908,-0.009056839,0.009347656,-0.02459472,0.026422706,0.033236109,0.041212775,0.019027673,-0.00315743,0.004424557,0.020689478,-0.0153717,-0.015205519,-0.034897912,0.020274026,0.016867325,0.040714234,-0.022766734,-0.010967916,0.026256526,0.007062673,-0.015953332,-0.007727395,0.031574301,-0.002887387,-0.00614868,0.004569965,0.019027673,0.012878992,0.011798819,0.004258377,-0.019193852,-0.021437289,-0.021603471,0.000301202,-0.051183607,-0.004985416,-0.030078677,0.012629721,0.065142773,-0.031740483,-0.021104928,-0.03938479,-0.003365156,-0.016036423,0.036393538,0.009804652,-0.018612221,0.060489718,-0.003697517,0.000547876,0.063480966,0.02758597,0.010053922,-0.003655972,-0.001485239,0.018362951,0.021104928,-0.003905243,0.019443123,-0.002658889,-0.00380138,-0.013626805,0.035894997,0.035396457,-0.005691683,0.002762751,0.012878992,-0.009596926,-0.009970833,-0.015953332,0.022434372,0.00614868,-0.021188019,0.001557943,-0.020190936,0.009763107,0.017448956,0.006730312,0.005567048,0.019692395,-0.00218112,-0.016867325,0.006854947,0.007976666,0.019193852,0.040880412,0.007353489,-0.02127111,-0.031906664,-0.026755067,-0.017947499,0.040381871,0.042209856,0.00913993,-0.0307434,-0.017781317,-0.015039339,0.03057722,0.017532047,0.0187784,-0.060822077,0.002928932,-0.026422706,-0.005899409,0.039717149,0.026588887,-0.000971118,0.004923099,-0.013626805,0.0187784,-0.031408124,-0.000695881,0.050851244,-0.014457707,-0.007311944,-0.001293092,-0.002139574,-0.019276943,0.00290816,0.019360034,-0.017781317,0.002160347,0.016618054,-0.006522586,0.011798819,0.029247776,-0.02775215,0.010344739,-0.018362951,-0.036725901,-0.015870241,0.015704062,-0.012463541,0.02459472,-0.024096178,0.001152877,-0.031408124,0.025425622,0.027087428,0.00922302,0.034565553,0.015704062,-0.020689478,-0.00517237,-0.014706978,-0.001589101,0.026090344,0.014956249,0.011715728,0.004299921,-0.00913993,0.022434372,-0.03705826,0.048524719,-0.030411039,0.008433662,0.017033506,-0.000511525,-0.031408124,0.005940954,-0.012962082,-0.031574301,0.017448956,0.010178559,-0.011383367,-0.020107845,-0.005151597,0.006647222,0.013128263,0.007145763,0.008059756,-0.045201108,-0.004943871,0.015787151,-0.045201108,-0.020772567,-0.020274026,0.028250692,-0.024262359,-0.004424557,0.009804652,0.000472576,-0.005691683,0.001443693,-0.013294443,0.001412535,0.013211353,-0.01213118,-0.002118802,0.017781317,-0.007353489,-0.031075761,-0.004923099,0.011383367,-0.004486875,-0.010178559,0.016618054,0.014457707,0.023763817,-0.02459472,-0.00388447,0.012546631,-0.007519669,0.015704062,-0.014291527,0.009680017,-0.035562634,0.023763817,0.053510133,-0.0555043,-0.003572882,0.022102011,0.021603471,-0.017282777,-0.001474852,-0.043539301,0.007810486,-0.025757983,-0.005400868,0.029912498,-0.00760276,0.014125346,0.030909581,-0.03340229,-0.009680017,0.018030589,0.008849114,0.03057722,0.019775484,0.014125346,0.031906664,-0.03057722,-0.027087428,-0.023597637,-0.022434372,-0.012878992,0.016285693,-0.021603471,-0.029746316,0.029746316,0.020357117,0.006314861,-0.001158071,0.028749233,-0.045201108,0.011383367,0.011134096,-0.021437289,-0.035728816,0.001827986,0.008267482,-0.057498466,0.01213118,-0.01213118,-0.040548053,0.010718645,0.004798463,-0.004881553,-0.019526213,-0.008558298,0.0059825,-0.000262254,-0.017615138,0.005193142,0.019692395,-0.00198378,-0.002845842,0.012546631,0.006107135,-0.008225936,-0.008890659,0.015870241,0.00517237,0.002596571,-0.010427829,-0.019110762,0.024262359,0.012048089,-0.032405205,0.006522586,0.013211353,0.013211353,-0.038221523,-0.007727395,-0.008267482,-0.019276943,0.001474852,0.031408124,-0.035562634,0.017532047,-0.023431456,-0.015454791,-0.011383367,0.016534964,-0.02176965,0.008682934,0.027253609,0.020190936,-0.0247609,-0.007311944,0.009555381,-0.01852913,-0.011632638,0.011549547,0.027419789,-0.034067012,-0.01229736,0.0307434,0.003946788,0.0046946,0.037722982,0.03057722,-0.010427829,0.002284982,0.033236109,0.030078677,-0.013377533,0.007395034,-0.012048089,0.040714234,-0.028749233,-0.000102565,-0.0059825,-0.041046593,0.017698228,-0.006356406,0.003178203,0.009056839,0.023099095,0.00606559,0.011881908,-0.02127111,-0.001126912,-0.027087428,0.011134096,0.001204809,-0.017033506,0.011051006,-0.014374617,0.017864408,0.023431456,-0.002077257,-0.026755067,-0.043871664,0.025757983,-0.006190225,0.001152877,0.011798819,-0.024262359,0.006564131,-0.070128188,-0.004362239,0.012962082,-0.013626805,-0.001402148,-0.012214269,0.011217186,-0.015953332,0.015787151,0.011134096,0.027253609,0.024262359,-0.048192356,0.009970833,0.018944582,-0.00517237,0.021935832,0.02775215,0.003406701,-0.010884825,0.075113602,-0.015953332,0.007727395,0.026755067,-0.006190225,-0.012712811,0.013377533,0.005940954,-0.008309027,0.02459472,0.002316141,-0.022434372,-0.012712811,0.03057722,-0.015787151,0.026755067,-0.001069787,0.03988333,-0.003697517,0.039550968,-0.019027673,-0.0059825,-0.00031029,-0.012546631,-0.003614427,0.007478124,0.005525503,0.032571387,-0.011798819,-0.011466458,-0.00606559,-0.011798819,0.018446039,0.007976666,0.018944582,-0.02176965,0.026588887,-0.006273315,-0.012463541,-0.007395034,0.012048089,-0.029247776,0.015454791,-0.007145763,0.006481041,-0.015620971,-0.00388447,-0.025757983,-0.001651419,-0.032903746,-0.005068507,0.03938479,0.003926015,0.004715373,0.022600554,-0.012546631,0.022932915,0.007810486,0.040714234,0.019941665,0.013543714,0.003406701,0.010884825,-0.03988333,0.042209856,-0.022766734,0.027419789,-0.029580137,0.043206941,0.022932915,0.021104928,-0.056833744,0.005193142,0.036061179,-0.012878992,0.008516753,-0.02758597,-0.030244859,-0.011798819,0.001111332,-0.014125346,-0.014125346,0.019027673,0.029081594,0.018861491,0.013626805,0.06846638,0.023099095,0.041378956,0.001599488,-0.028749233,0.017781317,0.016285693,0.021603471,-0.018113678,0.011300277,-0.032239024,0.022434372,-0.02459472,-0.013626805,0.005483958,0.013460624,-0.031574301,-0.015620971,0.016451873,0.014790068,-0.008849114,0.011134096,0.00461151,0.015122429,0.036227357,0.00206687,0.000877641,0.022102011,-0.028250692,0.022600554,-0.026422706,0.004029878,-0.032072846,0.017116595,0.010884825,0.019609304,0.00614868,0.005733229,0.016119512,0.002866614,-0.014540797,0.012463541,-0.003905243,0.003759835,-0.000485559,-0.022766734,-0.016285693,0.037722982,0.009513836,0.001506011,0.011964999,0.004029878,0.019941665,-0.000965924,0.002129188,0.015205519,0.071125269,0.022932915,0.005940954,-0.00044661,0.010220103,-0.03423319,-0.016285693,-0.016867325,-0.000659529,-0.008018211,-0.011383367,0.000016634,0.004071423,-0.029413955,0.019941665,-0.00913993,-0.024096178,0.010635555,0.010594009,0.001547556,0.036227357,-0.030078677,0.020772567,0.022268193,-0.014125346,0.008766023,-0.012962082,-0.007187308,0.017033506,-0.007187308,-0.015205519,-0.005608593,0.044536386,-0.001235968,0.007852031,0.001599488,0.005857864,-0.005940954,-0.010510919,-0.005567048,0.006730312,0.016285693,-0.010801735,-0.024428539,0.015122429,-0.02176965,0.01528861,-0.007436579,0.00226421,-0.004715373,0.004507647,0.004341467,0.005525503,-0.031075761,-0.005899409,0.037556801,0.014873158,-0.000342747,0.009970833,-0.019443123,0.023597637,-0.012048089,-0.025259443,0.006024044,-0.01827986,0.010012378,0.016784234,0.013211353,-0.005400868,-0.024428539,-0.02176965,-0.035230275,0.009347656,0.028583053,-0.015704062,-0.017781317,0.00226421,0.001199616,-0.003385928,0.008267482,0.002326528,0.022434372,-0.020190936,-0.015787151,0.000789358,0.031241942,0.011300277,0.001506011,-0.023265276,-0.010967916,0.009056839,0.011300277,-0.030244859,0.007478124,0.001111332,-0.035894997,0.0153717,0.002700434,0.021104928,0.010884825,-0.003344383,0.00768585,0.010386284,0.00452842,-0.014706978,0.028084511,0.013377533,0.014873158,0.046862911,-0.015454791,0.021188019,0.013959166,0.012629721,0.025924165,-0.018695312,-0.00922302,-0.0093892,0.007727395,0.036892079,0.007228854,-0.01229736,0.029247776,-0.004943871,-0.027253609,-0.008433662,0.043206941,0.002825069,0.028583053,-0.023431456,0.034897912,-0.041545134,-0.016534964,0.003053567,-0.012712811,0.002741979,-0.007187308,-0.025093261,-0.045201108,-0.004424557,-0.016618054,-0.008890659,0.008018211,-0.05184833,-0.019526213,-0.013377533,-0.010469374,0.030244859,-0.005068507,0.051183607,0.005483958,-0.006024044,0.035064094,-0.011134096,0.014956249,0.002284982,0.001724123,-0.01229736,0.012629721,0.010261648,0.014540797,0.048857078,-0.029580137,-0.024927082,-0.008350573,-0.03988333,0.000939959,0.013543714,0.013626805,-0.021437289,-0.012962082,0.006771857,0.013709894,-0.0059825,0.035396457,-0.006439496,-0.029580137,0.0046946,0.019609304,-0.007270399,0.014291527,-0.015620971,0.00118923,-0.00760276,-0.017199686,0.023265276,0.026588887,-0.030078677,-0.016701145,-0.025757983,0.004964644,0.026588887,0.043206941,0.011051006,-0.009846197,0.028915415,0.031574301,0.023763817,0.009264565,-0.008433662,-0.035064094,-0.000579035,-0.0247609,0.014125346,0.016618054,0.028749233,-0.052513052,-0.016867325,-0.01238045,0.002741979,0.013709894,0.010718645,0.013626805,0.009596926,-0.004403784,-0.02758597,-0.000945152,0.000420645,0.003759835,0.012546631,-0.011881908,0.008392117,0.012795902,0.005483958,-0.009763107,0.006397951,-0.010801735,0.012795902,-0.03938479,0.005733229,0.005733229,-0.000433627,0.015454791,0.002357686,-0.006564131,0.030244859,-0.024428539,0.016036423,0.014291527,-0.004964644,0.029413955,0.040381871,0.012629721,-0.033568468,-0.026422706,-0.037889164,-0.034399372,-0.03423319,0.021935832,0.004133741,-0.014623888,-0.013543714,-0.05517194,0.004736145,0.006314861,0.00006037,0.006356406,0.003323611,-0.010344739,0.007062673,0.005899409,-0.00623177,-0.001973394,-0.0555043,0.011881908,0.001350217,-0.033069927,-0.026921248,0.022268193,0.028583053,-0.021021837,0.010884825,0.019692395,-0.005442413,0.031574301,-0.014956249,0.01238045,-0.006356406,0.006273315,-0.003095113,-0.014540797,-0.02176965,0.005006189,-0.002658889,0.042542219,-0.02176965,0.017199686,-0.016701145,-0.001599488,0.016950415,-0.021188019,0.017864408,0.023763817,-0.000669915,0.025093261,0.021104928,0.008807569,0.037390623,-0.025591804,-0.003178203,-0.001319058,0.020523297,0.005255459,0.019276943,-0.00226421,0.00760276,-0.057166107,-0.006896493,-0.034067012,0.043871664,0.038221523,0.008101301,0.03988333,0.015870241,0.000955538,-0.004299921,-0.002928932,-0.002118802,-0.020523297,-0.001168457,-0.011134096,-0.000685495,0.003323611,0.011549547,0.034565553,0.029247776,-0.029746316,0.005213914,0.019110762,-0.003302838,0.026422706,0.028915415,-0.036227357,0.033236109,0.038387705,-0.035230275,0.004071423,-0.021935832,0.002928932,0.000976311,0.000527104,-0.006854947,-0.003822153,-0.001199616,0.019858574,-0.002762751,0.039052427,-0.008641388,0.032239024,-0.002295369,0.035396457,0.044536386,-0.029413955,0.025093261,-0.03423319,-0.016867325,-0.008849114,0.008433662,-0.004486875,0.017033506,0.006730312,-0.008599843,-0.008225936,-0.024428539,0.006564131,-0.007561215,-0.032072846,-0.019941665,0.035396457,0.019276943,0.010261648,0.005857864,0.032239024,-0.044204023,-0.018944582,0.002409618,0.032903746,0.05517194,-0.03655972,0.007976666,0.030909581,-0.023929998,0.016368784,0.01528861,-0.00768585,0.02176965,0.013626805,-0.02459472,0.04021569,-0.032737568,0.006854947,-0.011383367,0.014873158,-0.02176965,0.00243039,0.0093892,0.0093892,-0.029580137,0.019858574,0.01827986,0.024428539,0.017864408,-0.028250692,-0.001111332,0.056169022,0.007478124,-0.010718645,0.041046593,-0.015704062,0.034731735,0.002523867,-0.032571387,0.004341467,-0.023597637,-0.011881908,-0.035562634,0.006688767,0.007810486,-0.012712811,0.022600554,0.03057722,0.022600554,0.010552464,0.0307434,-0.009638472,0.02176965,-0.018030589,0.024262359,-0.036227357,-0.020772567,0.001641033,-0.022932915,-0.014623888,0.018362951,0.002575798,0.006190225,-0.011051006,0.021021837,0.019110762,0.02127111,-0.028583053,-0.052180689,-0.014291527,-0.010552464,0.036393538,0.042542219,-0.04586583,-0.001869531,0.008350573,-0.008516753,-0.020772567,0.000294711,0.015704062,-0.014457707,-0.020772567,0.008766023,-0.026588887,-0.004736145,-0.028084511,-0.007519669,0.010552464,-0.016534964,0.006190225,0.012962082,-0.016618054,0.012546631,0.02459472,0.022932915,0.020440206,-0.027918331,-0.008059756,0.020689478,-0.014623888,-0.011466458,-0.006896493,-0.020024756,-0.031408124,0.021603471,0.007270399,-0.03057722,0.008350573,-0.021437289,0.00072704,-0.043871664,0.006314861,-0.017199686,0.02176965,0.024262359,-0.020357117,-0.000542683,-0.005213914,0.001963008,-0.00064395,-0.022434372,0.022102011,-0.006688767,-0.028583053,0.002191506,-0.005047734,0.002368073,0.014956249,0.023929998,-0.003302838,-0.032239024,0.022268193,-0.013377533,-0.010801735,0.003676744,0.009015295,-0.039550968,0.010884825,-0.033568468,0.013709894,-0.029413955,-0.006356406,-0.020274026,0.023597637,0.030909581,0.02176965,0.016285693,0.045533467,-0.024096178,-0.030909581,-0.026422706,0.002783524,-0.010594009,0.004362239,-0.070792913,0.009472291,-0.022102011,0.011134096,-0.017448956,-0.011549547,-0.056833744,0.00082571,0.026588887,-0.013709894,0.002575798,0.02176965,-0.000568649,-0.007270399,0.004279149,-0.042874578,-0.026588887,0.016784234,0.036725901,-0.028915415,-0.009513836,0.017448956,0.002035712,-0.007228854,0.011383367,0.011134096,0.028915415,0.0153717,-0.027087428,0.043871664,-0.005089279,0.006314861,0.014291527,-0.003240521,0.025924165,-0.001230775,-0.015454791,-0.012629721,0.031740483,-0.039717149,-0.031075761,0.006605676,-0.008641388,-0.032239024,0.037722982,-0.03705826,-0.024096178,0.001911076,0.018196769,-0.007353489,-0.011300277,-0.029081594,0.004590738,-0.018030589,-0.026588887,0.010261648,0.038221523,0.008392117,-0.01213118,0.018362951,-0.034731735,-0.017781317,-0.011632638,0.005255459,0.000851675,0.014208436,-0.000039922,-0.000228498,0.014790068,0.00913993,0.0004544,-0.011798819,-0.020440206,0.005899409,0.008350573,0.006314861,0.040548053,0.003427474,-0.010801735,0.008599843,0.002586185,-0.041212775,-0.016368784,0.020024756,0.000965924,-0.021021837,-0.008475208,0.0307434,0.00760276,0.003614427,0.003489791,-0.025924165,0.000799744,0.013460624,-0.020440206,0.048857078,0.004320694,-0.048857078,0.015039339,-0.029580137,0.025924165,0.018861491,-0.014706978,0.000576439,-0.031241942,0.0307434,0.0153717,0.014706978,0.028084511,-0.01238045,-0.031241942,0.018196769,-0.034897912,0.008142847,0.010718645,0.00922302,0.047859997,-0.00072704,-0.010427829,0.007104218,0.026256526,0.012214269,-0.013377533,-0.05184833,0.005276232,0.021935832,-0.007021128,0.009804652,0.007893575,0.024096178,-0.002357686,0.033900831,-0.031740483,0.034565553,-0.036892079,-0.015454791,0.030411039,0.010552464,-0.022268193,-0.001391762,-0.008184392,-0.008558298,0.008475208,-0.009929287,0.010427829,0.041378956,-0.009555381,-0.008724478,-0.039052427,0.034731735,-0.014291527,0.023099095,0.029081594,0.007519669,0.010967916,-0.008142847,0.006190225,-0.031075761,0.033734649,-0.001672192,0.047859997,-0.022434372,-0.007395034,0.01213118,0.056169022,0.002762751,-0.029413955,-0.000763392,-0.015787151,0.010801735,0.008142847,0.029912498,-0.0018176,0.033236109,-0.046198189,-0.002492708,-0.006730312,0.008807569,-0.03655972,0.009430746,-0.053842496,-0.060489718,0.046862911,0.002783524,-0.0187784,0.000571246,0.00760276,0.002482322,0.001319058,-0.014291527,0.001464466,-0.011632638,-0.012463541,-0.004902326,0.000841289,0.006688767,0.030244859,0.018944582,0.000532297,-0.015620971,0.007104218,0.005608593,0.002035712,-0.023763817,0.003032795,0.010594009,-0.023597637,-0.042376038,-0.005255459,0.001199616,-0.0247609,-0.007893575,-0.011632638,0.013045172,-0.005691683,-0.007104218,0.027419789,-0.004320694,-0.005525503,-0.026090344,0.031408124,-0.012795902,-0.007062673,0.000939959,0.000030185,0.004175286,0.014291527,0.033236109,-0.038720068,0.074116521,-0.019692395,0.001589101,0.013792985,-0.056169022,-0.028749233,-0.001599488,0.004175286,0.014790068,0.00162026,-0.007519669,-0.041378956,0.016534964,-0.003572882,-0.002575798,-0.019526213,-0.00922302,-0.033900831,-0.042043675,-0.014208436,0.010178559,0.017698228,0.032239024,0.00913993,0.009264565,-0.012463541,-0.005857864,-0.015870241,0.004486875,0.018861491,-0.000176567,-0.029912498,0.016784234,0.012546631,0.051183607,0.023597637,0.032903746,0.0153717,-0.013377533,-0.000016634,-0.061486799,-0.034565553,0.016119512,0.00380138,-0.003863698,0.004362239,-0.017532047,-0.002762751,0.000102565,-0.021437289,0.029247776,-0.010718645,-0.015870241,-0.016285693,0.010220103,-0.000373906,0.012962082,0.010137013,-0.007228854,0.02127111,-0.029247776,0.018113678,0.009181475,0.002233051,0.014374617,-0.00396756,0.010801735,0.007644305,0.020855658,0.014790068,0.032737568,-0.037390623,0.003032795,0.010801735,-0.01553788,-0.014790068,0.019526213,-0.017947499,-0.007893575,-0.011964999,-0.00614868,-0.005857864,-0.032072846,-0.025924165,0.001163264,-0.013294443,-0.01553788,0.016701145,-0.013460624,-0.001111332,0.00760276,0.01553788,-0.033734649,0.048192356,-0.003282066,0.031906664,0.002845842,0.003240521,0.017116595,-0.01827986,0.006896493,-0.00760276,-0.009680017,-0.02459472,-0.020689478,-0.053510133,0.00614868,-0.010552464,-0.032405205,-0.0307434,0.025093261,0.003635199,-0.008101301,-0.00606559,-0.007436579,0.00606559,-0.012962082,0.026921248,0.009098385,0.046530552,-0.011632638,0.032571387,-0.033900831,0.009846197,0.002866614,0.032903746,0.008973749,0.012712811,0.040049512,0.013626805,-0.026256526,-0.031408124,0.036227357,0.011964999,-0.006024044,-0.001848759,0.015704062,-0.021188019,-0.035064094,-0.013377533,-0.009721561,-0.01553788,0.008766023,0.005400868,0.004507647,-0.018362951,-0.026588887,-0.00913993,-0.025591804,0.035894997,0.021935832,-0.031906664,-0.000602404,0.026422706,-0.006397951,0.006647222,0.0093892,0.020606387,0.00913993,0.015620971,-0.024096178,0.00063616,-0.006564131,0.01238045,-0.013709894,0.000563456,-0.009887742,0.016618054,-0.003323611,0.000451803,0.001609874,0.008682934,0.025259443,0.020024756,-0.027253609,0.010884825,0.028250692,-0.054839578,0.033568468,-0.004902326,0.003053567,0.020274026,-0.015704062,-0.00614868,-0.063813329,0.002482322,0.009763107,-0.001609874,-0.012214269,0.020107845,0.001921462,0.018695312,-0.004923099,0.007270399,-0.023763817,0.005234687,0.003406701,0.002565412,0.007104218,0.000841289,0.016202603,0.01827986,-0.031075761,-0.035562634,-0.025259443,-0.007021128,0.000641353,-0.033069927,0.010718645,0.005650138,0.024927082,-0.002658889,0.00380138,0.009929287,-0.004258377,-0.039717149,-0.022434372,0.025425622,0.00198378,0.006356406,0.017615138,-0.032072846,0.046862911,-0.026921248,0.005940954,0.021603471,-0.002253824,0.002825069,-0.030411039,-0.003115885,0.023597637,-0.004320694,-0.007852031,0.018030589,-0.008724478,-0.005733229,0.032903746,0.013876075,0.015454791,-0.023597637,0.005151597,-0.035396457,0.02176965,-0.012463541,0.025591804,0.014540797,-0.027918331,0.004154514,0.008724478,0.016036423,-0.015870241,0.005400868,-0.017365867,-0.044868745,-0.000485559,0.020357117,-0.00760276,-0.023265276,-0.012048089,0.008433662,0.018362951,-0.006979583,0.0307434,0.008392117,0.027087428,-0.019360034,0.016119512,0.02127111,0.010801735,0.00299125,0.002949705,0.012463541,-0.000025966,0.015953332,0.029413955,0.020024756,0.003780607,0.022102011,-0.031740483,0.01553788,0.010386284,0.028749233,-0.010884825,0.008682934,-0.003531337,-0.05517194,-0.019360034,-0.009347656,-0.002025325,0.003261293,-0.025425622,-0.01553788,-0.000251867,0.014291527,0.012546631,0.035728816,-0.007062673,-0.006605676,0.000384293,-0.005047734,-0.032571387,-0.021188019,-0.02127111,-0.016036423,0.008475208,-0.004009106,0.014291527,-0.008101301,0.004424557,-0.038221523,-0.019360034,0.015039339,-0.015454791,-0.029580137,0.035728816,0.004466102,-0.000778971,-0.005068507,-0.017781317,0.00477769,0.001838372,0.030244859,0.01213118,-0.022932915,-0.005359322,0.037390623,0.005899409,0.002046098,0.037889164,0.016701145,0.010303194,0.02127111,-0.009513836,-0.022268193,-0.005650138,-0.00388447,0.016534964,-0.023265276,-0.00054528,0.004819236,0.004715373,-0.001178843,-0.051183607,-0.00614868,-0.010552464,-0.002741979,-0.009181475,0.023597637,0.019193852,0.017199686,-0.036393538,-0.00243039,-0.015870241,-0.014706978,-0.00145408,0.016368784,-0.011632638,-0.014623888,-0.01229736,-0.01553788,0.040880412,0.023929998,-0.014623888,0.002648502,0.031906664,-0.033734649,-0.026755067,0.002783524,0.005359322,0.009970833,0.001412535,0.016950415,0.016285693,-0.006730312,-0.02459472,0.050851244,-0.001827986,-0.020855658,0.020938748,0.004071423,-0.021603471,-0.007852031,-0.023929998,-0.029912498,-0.003365156,0.017365867,-0.010427829,-0.011715728,0.014956249,0.011383367,0.032405205,-0.028583053,-0.017448956,0.018446039,0.017615138,0.035728816,-0.010095468,-0.00254464,0.010012378,0.028250692,-0.020855658,-0.002305755,-0.001002276,-0.014125346,-0.007021128,-0.028583053,-0.045533467,-0.02758597,-0.020440206,0.001350217,0.010053922,0.020689478,-0.017615138,0.026422706,0.040880412,0.012463541,-0.010718645,-0.014706978,0.068134025,0.038720068,0.047859997,-0.012546631,0.015704062,-0.002087643,-0.010303194,0.014790068,0.018612221,0.007395034,-0.014790068,-0.017864408,-0.005068507,-0.054507218,0.004902326,-0.004050651,0.021603471,0.019775484,-0.024262359,-0.012795902,0.021935832,-0.004009106,-0.039717149,0.037556801,-0.016701145,-0.025757983,0.005483958,-0.005110051,-0.021935832,-0.003406701,0.010594009,0.015787151,-0.049854163,0.007727395,-0.008392117,-0.017199686,0.009970833,-0.008849114,-0.013876075,-0.0059825,-0.015870241,-0.007104218,0.028250692,-0.029081594,0.026921248,0.00299125,-0.017781317,0.042542219,0.018196769,0.052845411,-0.004819236,-0.014125346,0.02459472,-0.011715728,0.015787151,-0.005774774,0.004902326,-0.004964644,-0.02758597,-0.013959166,-0.033568468,-0.027918331,-0.017698228,0.003489791,-0.020024756,-0.021603471,0.019360034,0.028084511,-0.002503094,-0.018861491,-0.002295369,0.050851244,-0.020689478,-0.000459593,-0.026090344,0.002783524,-0.005899409,-0.026921248,-0.0093892,-0.004112968,0.031574301,0.003926015,-0.032903746,-0.046198189,-0.019027673,-0.00913993,0.030411039,-0.019443123,0.001963008,-0.005193142,0.010884825,-0.02127111,-0.025259443,0.032737568,0.00089322,0.003282066,0.001713737,-0.006439496,0.016867325,-0.031574301,0.031075761,-0.009970833,0.022600554,-0.023597637,-0.014956249,0.004009106,0.00198378,0.026588887,-0.023431456,-0.023763817,-0.013294443,-0.029746316,0.001381376,-0.042874578,-0.00913993,0.014873158,0.016202603,0.012878992,-0.006024044,0.009638472,0.010552464,-0.017033506,-0.027087428,0.044536386,-0.038055345,0.001329444,-0.019609304,0.023597637,-0.043206941,0.040049512,0.017615138,0.046862911,0.02127111,0.013294443,-0.039550968,-0.018861491,-0.019609304,-0.033734649,0.00623177,-0.017199686,0.041212775,-0.017781317,-0.024262359,0.054507218,-0.009721561,0.005816319,-0.00206687,-0.008766023,0.017365867,-0.000737426,0.018362951,-0.023597637,-0.019110762,0.021935832,0.041545134,-0.020357117,-0.017615138,0.044868745,-0.018030589,-0.032405205,-0.050186522,-0.014540797,0.005213914,-0.006688767,0.047527634,0.040714234], + "numCandidates": 150, + "limit": 10, + "quantization": "scalar" + } + }, + { + "$project": { + "_id": 0, + "plot": 1, + "title": 1, + "score": { $meta: "vectorSearchScore" } + } + } +]); +EOF +) + +kubectl exec --context "${K8S_CTX}" -n "${MDB_NS}" mongodb-tools-pod -- /bin/bash -eu -c "$(cat < /tmp/mdb_script.js +mongosh --quiet "${MDB_CONNECTION_STRING}" < /tmp/mdb_script.js +EOF +)" diff --git a/docs/search/03-search-query-usage/code_snippets/03_9010_delete_namespace.sh b/docs/search/03-search-query-usage/code_snippets/03_9010_delete_namespace.sh new file mode 100755 index 000000000..2d28e4bb7 --- /dev/null +++ b/docs/search/03-search-query-usage/code_snippets/03_9010_delete_namespace.sh @@ -0,0 +1 @@ +kubectl --context "${K8S_CTX}" delete namespace "${MDB_NS}" diff --git a/docs/search/03-search-query-usage/env_variables.sh b/docs/search/03-search-query-usage/env_variables.sh new file mode 100644 index 000000000..af6bd874c --- /dev/null +++ b/docs/search/03-search-query-usage/env_variables.sh @@ -0,0 +1,20 @@ +# The env vars here are all commented out because this snippets module +# is reusable across different MongoDB deployments and all the necessary variables +# should be already defined there. + +# set it to the context name of the k8s cluster +#export K8S_CTX="" +# the following namespace will be used to deploy mongodb tools pod +#export MDB_NS="mongodb" + +# regular user performing restore and search queries on sample mflix database +# the user should be able to restore database using mongorestore +#export MDB_USER_PASSWORD="mdb-user-password-CHANGE-ME" + +# name of MongoDB or MongoDBCommunity resource in case it's deployed in the same cluster +# user only for the connection string in MDB_CONNECTION_STRING env var below +#export MDB_RESOURCE_NAME="mdbc-rs" + +# default connection string if MongoDB database is deployed using the operator +#export MDB_CONNECTION_STRING="mongodb://mdb-user:${MDB_USER_PASSWORD}@${MDB_RESOURCE_NAME}-0.${MDB_RESOURCE_NAME}-svc.${MDB_NS}.svc.cluster.local:27017/?replicaSet=${MDB_RESOURCE_NAME}" + diff --git a/docs/search/03-search-query-usage/output/03_0420_import_movies_mflix_database.out b/docs/search/03-search-query-usage/output/03_0420_import_movies_mflix_database.out new file mode 100644 index 000000000..a7432faf0 --- /dev/null +++ b/docs/search/03-search-query-usage/output/03_0420_import_movies_mflix_database.out @@ -0,0 +1,2 @@ +Downloading sample database archive... +Restoring sample database diff --git a/docs/search/03-search-query-usage/output/03_0440_wait_for_search_index_ready.out b/docs/search/03-search-query-usage/output/03_0440_wait_for_search_index_ready.out new file mode 100644 index 000000000..b264580af --- /dev/null +++ b/docs/search/03-search-query-usage/output/03_0440_wait_for_search_index_ready.out @@ -0,0 +1 @@ +Sleeping to wait for search indexes to be created diff --git a/docs/search/03-search-query-usage/output/03_0444_list_search_indexes.out b/docs/search/03-search-query-usage/output/03_0444_list_search_indexes.out new file mode 100644 index 000000000..d236b3b7c --- /dev/null +++ b/docs/search/03-search-query-usage/output/03_0444_list_search_indexes.out @@ -0,0 +1,32 @@ +{ + cursor: { + id: 0, + ns: 'sample_mflix.movies', + firstBatch: [ + { + id: '68ba080ea2d54a79efca0bcc', + name: 'default', + type: 'search', + latestDefinition: { + indexID: ObjectId('68ba080ea2d54a79efca0bcc'), + name: 'default', + database: 'sample_mflix', + lastObservedCollectionName: 'movies', + collectionUUID: UUID('1946f612-6643-42ee-b3fe-a8c70dcb6c8f'), + numPartitions: 1, + mappings: { dynamic: true, fields: {} }, + indexFeatureVersion: 3 + } + } + ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1757022273, i: 7 }), + signature: { + hash: Binary.createFromBase64('kdWxyz35pdSme00nFy6jvg0fMQ0=', 0), + keyId: Long('7546352109956890629') + } + }, + operationTime: Timestamp({ t: 1757022273, i: 7 }) +} diff --git a/docs/search/03-search-query-usage/output/03_0445_list_vector_search_indexes.out b/docs/search/03-search-query-usage/output/03_0445_list_vector_search_indexes.out new file mode 100644 index 000000000..e7c92f46e --- /dev/null +++ b/docs/search/03-search-query-usage/output/03_0445_list_vector_search_indexes.out @@ -0,0 +1,40 @@ +{ + cursor: { + id: 0, + ns: 'sample_mflix.embedded_movies', + firstBatch: [ + { + id: '68ba080ea2d54a79efca0bcd', + name: 'vector_index', + type: 'vectorSearch', + latestDefinition: { + type: 'vectorSearch', + indexID: ObjectId('68ba080ea2d54a79efca0bcd'), + name: 'vector_index', + database: 'sample_mflix', + lastObservedCollectionName: 'embedded_movies', + collectionUUID: UUID('cf7c49d4-6299-4da8-a36c-a5a2017c3077'), + numPartitions: 1, + fields: [ + { + type: 'vector', + path: 'plot_embedding_voyage_3_large', + numDimensions: 2048, + similarity: 'dotProduct', + quantization: 'scalar' + } + ] + } + } + ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1757022273, i: 7 }), + signature: { + hash: Binary.createFromBase64('kdWxyz35pdSme00nFy6jvg0fMQ0=', 0), + keyId: Long('7546352109956890629') + } + }, + operationTime: Timestamp({ t: 1757022273, i: 7 }) +} diff --git a/docs/search/03-search-query-usage/output/03_0450_execute_search_query.out b/docs/search/03-search-query-usage/output/03_0450_execute_search_query.out new file mode 100644 index 000000000..f425a96d8 --- /dev/null +++ b/docs/search/03-search-query-usage/output/03_0450_execute_search_query.out @@ -0,0 +1,22 @@ +mdb-rs [primary] test> switched to db sample_mflix +mdb-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'A sports agent stages an unconventional recruitment strategy to get talented Indian cricket players to play Major League Baseball.', + genres: [ 'Biography', 'Drama', 'Sport' ], + title: 'Million Dollar Arm', + released: ISODate('2014-05-16T00:00:00.000Z') + }, + { + plot: 'A Taiwanese high school baseball team travels to Japan in 1931 to compete in a national tournament.', + genres: [ 'Biography', 'Drama', 'History' ], + title: 'Kano', + released: ISODate('2014-02-27T00:00:00.000Z') + }, + { + plot: "12-year-old Josh is a mixed race boy and a promising baseball player. He is abused by his mother's boyfriend Byrd, and neglected by his mother Debbie. He forges his own path in life when ...", + genres: [ 'Drama' ], + title: 'Calloused Hands', + released: ISODate('2013-03-03T00:00:00.000Z') + } +] +mdb-rs [primary] sample_mflix> \ No newline at end of file diff --git a/docs/search/03-search-query-usage/output/03_0455_execute_vector_search_query.out b/docs/search/03-search-query-usage/output/03_0455_execute_vector_search_query.out new file mode 100644 index 000000000..1623a6fdc --- /dev/null +++ b/docs/search/03-search-query-usage/output/03_0455_execute_vector_search_query.out @@ -0,0 +1,54 @@ +mdb-rs [primary] test> switched to db sample_mflix +mdb-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'At the age of 21, Tim discovers he can travel in time and change what happens and has happened in his own life. His decision to make his world a better place by getting a girlfriend turns out not to be as easy as you might think.', + title: 'About Time', + score: 0.7704131603240967 + }, + { + plot: 'A psychiatrist makes multiple trips through time to save a woman that was murdered by her brutal husband.', + title: 'Retroactive', + score: 0.7597770690917969 + }, + { + plot: 'An officer for a security agency that regulates time travel, must fend for his life against a shady politician who has a tie to his past.', + title: 'Timecop', + score: 0.7574796676635742 + }, + { + plot: 'A time-travel experiment in which a robot probe is sent from the year 2073 to the year 1973 goes terribly wrong thrusting one of the project scientists, a man named Nicholas Sinclair into a...', + title: 'A.P.E.X.', + score: 0.7573235034942627 + }, + { + plot: 'After visiting 2015, Marty McFly must repeat his visit to 1955 to prevent disastrous changes to 1985... without interfering with his first trip.', + title: 'Back to the Future Part II', + score: 0.751945972442627 + }, + { + plot: 'A reporter, learning of time travelers visiting 20th century disasters, tries to change the history they know by averting upcoming disasters.', + title: 'Thrill Seekers', + score: 0.7503504753112793 + }, + { + plot: 'Hoping to alter the events of the past, a 19th century inventor instead travels 800,000 years into the future, where he finds humankind divided into two warring races.', + title: 'The Time Machine', + score: 0.750007152557373 + }, + { + plot: 'Lyle, a motorcycle champion is traveling the Mexican desert, when he find himself in the action radius of a time machine. So he find himself one century back in the past between rapists, ...', + title: 'Timerider: The Adventure of Lyle Swann', + score: 0.7499568462371826 + }, + { + plot: 'A romantic drama about a Chicago librarian with a gene that causes him to involuntarily time travel, and the complications it creates for his marriage.', + title: "The Time Traveler's Wife", + score: 0.7492842674255371 + }, + { + plot: 'A modern aircraft carrier is thrown back in time to 1941 near Hawaii, just hours before the Japanese attack on Pearl Harbor.', + title: 'The Final Countdown', + score: 0.7472751140594482 + } +] +mdb-rs [primary] sample_mflix> \ No newline at end of file diff --git a/docs/search/03-search-query-usage/test.sh b/docs/search/03-search-query-usage/test.sh new file mode 100755 index 000000000..28e430eb0 --- /dev/null +++ b/docs/search/03-search-query-usage/test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/../../../scripts/code_snippets/sample_test_runner.sh" + +cd "${script_dir}" + +prepare_snippets + +run 03_0410_run_mongodb_tools_pod.sh +run_for_output 03_0420_import_movies_mflix_database.sh +run 03_0430_create_search_index.sh +run 03_0435_create_vector_search_index.sh +run_for_output 03_0440_wait_for_search_index_ready.sh +run_for_output 03_0444_list_search_indexes.sh +run_for_output 03_0445_list_vector_search_indexes.sh +run_for_output 03_0450_execute_search_query.sh +run_for_output 03_0455_execute_vector_search_query.sh + +cd - diff --git a/scripts/code_snippets/code_snippets_cleanup.sh b/scripts/code_snippets/code_snippets_cleanup.sh index 5c4736bb4..f88f9f36a 100755 --- a/scripts/code_snippets/code_snippets_cleanup.sh +++ b/scripts/code_snippets/code_snippets_cleanup.sh @@ -1,16 +1,36 @@ -#!/usr/bin/env bash +#!/bin/bash set -eou pipefail -find public/architectures -name "test.sh" -exec sh -c ' - source scripts/code_snippets/sample_test_runner.sh +script_name=$(readlink -f "${BASH_SOURCE[0]}") +# shellcheck disable=SC2034 +script_dir=$(dirname "${script_name}") - pushd "$(dirname $1)" +source scripts/code_snippets/sample_test_runner.sh + +cleanup_directory() { + local test_file="$1" + local dir + dir="$(dirname "${test_file}")" + + echo " ${dir}" + + pushd "${dir}" >/dev/null run_cleanup "test.sh" run_cleanup "teardown.sh" - rm -rf istio* - rm -rf certs - rm -rf secrets + rm -rf .generated 2>/dev/null || true + rm -rf istio* 2>/dev/null || true + rm -rf certs 2>/dev/null || true + rm -rf secrets 2>/dev/null || true + rm ./*.run.log 2>/dev/null || true + popd >/dev/null +} + +echo "Cleaning up from snippets runtime files from the following directories..." +for snippet_dir in $(bash "${script_dir}/find_snippets_directories.sh"); do + cleanup_directory "${snippet_dir}/test.sh" & +done + +wait +echo "Cleaning up done." - popd - ' sh {} \; diff --git a/scripts/code_snippets/find_snippets_directories.sh b/scripts/code_snippets/find_snippets_directories.sh new file mode 100644 index 000000000..463a42ea5 --- /dev/null +++ b/scripts/code_snippets/find_snippets_directories.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -eou pipefail +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + +source scripts/dev/set_env_context.sh + +# It finds all directories containing both test.sh and code_snippets subdirectory +find . -path "*/code_snippets" -type d | while read -r code_snippets_dir; do + parent_dir="$(dirname "${code_snippets_dir}")" + if [[ -f "${parent_dir}/test.sh" ]]; then + echo "${parent_dir}" + fi +done diff --git a/scripts/code_snippets/kind_community_search_snippets_render_template.sh b/scripts/code_snippets/kind_community_search_snippets_render_template.sh deleted file mode 100755 index f7c86d1f7..000000000 --- a/scripts/code_snippets/kind_community_search_snippets_render_template.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail -source scripts/dev/set_env_context.sh - -test_dir="docs/community-search/quick-start" -python scripts/code_snippets/render_template.py "${test_dir}/README.md.j2" "${test_dir}/README.md" diff --git a/scripts/code_snippets/kind_search_snippets_render_template.sh b/scripts/code_snippets/kind_search_snippets_render_template.sh new file mode 100755 index 000000000..57d2fcc28 --- /dev/null +++ b/scripts/code_snippets/kind_search_snippets_render_template.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -eou pipefail + +test_dir="$1" +scripts/evergreen/run_python.sh scripts/code_snippets/render_template.py "${test_dir}/README.md.j2" "${test_dir}/README.md" diff --git a/scripts/code_snippets/sample_commit_output.sh b/scripts/code_snippets/sample_commit_output.sh index 0eb12227b..12790b0b5 100755 --- a/scripts/code_snippets/sample_commit_output.sh +++ b/scripts/code_snippets/sample_commit_output.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -set -Eeou pipefail +set -eou pipefail +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + source scripts/dev/set_env_context.sh if [[ "${CODE_SNIPPETS_COMMIT_OUTPUT:-"false"}" == "true" ]]; then @@ -8,9 +10,7 @@ if [[ "${CODE_SNIPPETS_COMMIT_OUTPUT:-"false"}" == "true" ]]; then branch="meko-snippets-update-$(date "+%Y%m%d%H%M%S")" git checkout -b "${branch}" git reset - git add public/architectures/**/*.out - git add docs/**/output/*.out - git add docs/**/*.md + git add scripts/code_snippets/tests/outputs/test_* git commit -m "Update code snippets outputs" git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/mongodb/mongodb-kubernetes.git" git push origin "${branch}" diff --git a/scripts/code_snippets/sample_test_runner.sh b/scripts/code_snippets/sample_test_runner.sh index bc579b732..0e9c17f0b 100755 --- a/scripts/code_snippets/sample_test_runner.sh +++ b/scripts/code_snippets/sample_test_runner.sh @@ -2,12 +2,22 @@ set -eou pipefail -log_file="$(basename "$0").run.log" +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +# we reuse script_dir if already set by the script that is sourcing this file +script_dir=${script_dir:-$(dirname "${script_name}")} + +# script_dir will be set from the file that is sourcing this file +log_file="${RUN_LOG_FILE:-$(basename "${script_dir}").run.log}" + snippets_src_dir="code_snippets" snippets_run_dir=".generated" DEBUG=${DEBUG:-"false"} +_SNIPPETS_OUTPUT_DIR=${_SNIPPETS_OUTPUT_DIR:-"output"} + function snippets_list() { src_dir=$1 # shellcheck disable=SC2012 @@ -18,7 +28,8 @@ function run_cleanup() { script_file=$1 rm -rf "${snippets_run_dir}" 2>/dev/null || true rm -rf "log" 2>/dev/null || true - git restore --staged --worktree rm -rf "output" 2>/dev/null || true + rm -rf ".generated" 2>/dev/null || true + git restore --staged --worktree rm -rf "${_SNIPPETS_OUTPUT_DIR}" 2>/dev/null || true rm -rf "${script_file}.run.log" 2>/dev/null || true } @@ -27,7 +38,7 @@ function prepare_snippets() { touch "${log_file}" mkdir log 2>/dev/null || true - mkdir output 2>/dev/null || true + mkdir "${_SNIPPETS_OUTPUT_DIR}" 2>/dev/null || true rm -rf "${snippets_run_dir}" 2>/dev/null || true mkdir "${snippets_run_dir}" 2>/dev/null || true @@ -36,7 +47,7 @@ function prepare_snippets() { while IFS= read -r file_name; do file_path="${snippets_run_dir}/${file_name}" ( - echo "# This file is generated automatically from ${file_path}" + echo "# This file is generated automatically from ${snippets_src_dir}/${file_name}" echo "# DO NOT EDIT" echo "function ${file_name%.sh}() {" cat "${snippets_src_dir}/${file_name}" @@ -95,7 +106,7 @@ function run_for_output() { ret=$? set -e if [[ ${ret} == 0 ]]; then - tee "output/${cmd}.out" < "${stdout_file}" + tee "${_SNIPPETS_OUTPUT_DIR}/${cmd}.out" < "${stdout_file}" else echo "Error running: ${cmd}" fi diff --git a/scripts/code_snippets/task_kind_community_search_snippets_test.sh b/scripts/code_snippets/task_kind_community_search_snippets_test.sh deleted file mode 100755 index 4b4501c2c..000000000 --- a/scripts/code_snippets/task_kind_community_search_snippets_test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail -source scripts/dev/set_env_context.sh - -dump_logs() { - source scripts/evergreen/e2e/dump_diagnostic_information.sh - dump_all_non_default_namespaces "$@" -} -trap dump_logs EXIT - -test_dir="./docs/community-search/quick-start" - -source "${test_dir}/env_variables.sh" -echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" -# shellcheck disable=SC1090 -test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" - -${test_dir}/test.sh -scripts/code_snippets/kind_community_search_snippets_render_template.sh diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0090_helm_add_mogodb_repo.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0090_helm_add_mogodb_repo.out new file mode 100644 index 000000000..b1576c236 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0090_helm_add_mogodb_repo.out @@ -0,0 +1,6 @@ +"mongodb" already exists with the same configuration, skipping +Hang tight while we grab the latest from your chart repositories... +...Successfully got an update from the "mongodb" chart repository +Update Complete. ⎈Happy Helming!⎈ +NAME CHART VERSION APP VERSION DESCRIPTION +mongodb/mongodb-kubernetes 1.2.0 MongoDB Controllers for Kubernetes translate th... diff --git a/docs/community-search/quick-start/output/0100_install_operator.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out similarity index 80% rename from docs/community-search/quick-start/output/0100_install_operator.out rename to scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out index c54b9c92e..9c027d887 100644 --- a/docs/community-search/quick-start/output/0100_install_operator.out +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out @@ -1,17 +1,18 @@ Release "mongodb-kubernetes" does not exist. Installing it now. NAME: mongodb-kubernetes -LAST DEPLOYED: Tue Jul 8 07:04:51 2025 +LAST DEPLOYED: Mon Sep 1 19:22:18 2025 NAMESPACE: mongodb STATUS: deployed REVISION: 1 TEST SUITE: None USER-SUPPLIED VALUES: -dummy: value +registry: + imagePullSecrets: prerelease-image-pullsecret COMPUTED VALUES: agent: name: mongodb-agent - version: 108.0.2.8729-1 + version: 108.0.12.8846-1 community: agent: name: mongodb-agent @@ -39,17 +40,16 @@ community: version: 4.4.0 database: name: mongodb-kubernetes-database - version: 1.2.0 -dummy: value + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 initAppDb: name: mongodb-kubernetes-init-appdb - version: 1.2.0 + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 initDatabase: name: mongodb-kubernetes-init-database - version: 1.2.0 + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 initOpsManager: name: mongodb-kubernetes-init-ops-manager - version: 1.2.0 + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 managedSecurityContext: false mongodb: appdbAssumeOldFormat: false @@ -96,7 +96,7 @@ operator: vaultSecretBackend: enabled: false tlsSecretRef: "" - version: 1.2.0 + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 watchedResources: - mongodb - opsmanagers @@ -113,23 +113,22 @@ readinessProbe: version: 1.0.22 registry: agent: quay.io/mongodb - appDb: quay.io/mongodb - database: quay.io/mongodb - imagePullSecrets: null - initAppDb: quay.io/mongodb - initDatabase: quay.io/mongodb - initOpsManager: quay.io/mongodb - operator: quay.io/mongodb + appDb: quay.io/mongodb/staging + database: quay.io/mongodb/staging + imagePullSecrets: prerelease-image-pullsecret + initAppDb: quay.io/mongodb/staging + initDatabase: quay.io/mongodb/staging + initOpsManager: quay.io/mongodb/staging + operator: quay.io/mongodb/staging opsManager: quay.io/mongodb pullPolicy: Always readinessProbe: quay.io/mongodb versionUpgradeHook: quay.io/mongodb search: community: - name: mongodb-search-community - repo: quay.io/mongodb - version: 1.47.0 -subresourceEnabled: true + name: mongodb-search + repo: quay.io/mongodb/staging + version: latest versionUpgradeHook: name: mongodb-kubernetes-operator-version-upgrade-post-start-hook version: 1.0.9 @@ -143,6 +142,8 @@ kind: ServiceAccount metadata: name: mongodb-kubernetes-appdb namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret --- # Source: mongodb-kubernetes/templates/database-roles.yaml apiVersion: v1 @@ -150,6 +151,8 @@ kind: ServiceAccount metadata: name: mongodb-kubernetes-database-pods namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret --- # Source: mongodb-kubernetes/templates/database-roles.yaml apiVersion: v1 @@ -157,6 +160,8 @@ kind: ServiceAccount metadata: name: mongodb-kubernetes-ops-manager namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret --- # Source: mongodb-kubernetes/templates/operator-sa.yaml apiVersion: v1 @@ -164,8 +169,10 @@ kind: ServiceAccount metadata: name: mongodb-kubernetes-operator namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml +# Source: mongodb-kubernetes/templates/operator-roles-clustermongodbroles.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -178,34 +185,7 @@ rules: resources: - clustermongodbroles --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: mongodb-kubernetes-operator-mongodb-webhook -rules: - - apiGroups: - - "admissionregistration.k8s.io" - resources: - - validatingwebhookconfigurations - verbs: - - get - - create - - update - - delete - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# Source: mongodb-kubernetes/templates/operator-roles.yaml +# Source: mongodb-kubernetes/templates/operator-roles-telemetry.yaml # Additional ClusterRole for clusterVersionDetection kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -233,7 +213,34 @@ rules: verbs: - list --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml +# Source: mongodb-kubernetes/templates/operator-roles-webhook.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Source: mongodb-kubernetes/templates/operator-roles-clustermongodbroles.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -247,30 +254,30 @@ subjects: name: mongodb-kubernetes-operator namespace: mongodb --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml +# Source: mongodb-kubernetes/templates/operator-roles-telemetry.yaml +# ClusterRoleBinding for clusterVersionDetection kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: mongodb-kubernetes-operator-mongodb-webhook-binding + name: mongodb-kubernetes-operator-mongodb-cluster-telemetry-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: mongodb-kubernetes-operator-mongodb-webhook + name: mongodb-kubernetes-operator-cluster-telemetry subjects: - kind: ServiceAccount name: mongodb-kubernetes-operator namespace: mongodb --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml -# ClusterRoleBinding for clusterVersionDetection +# Source: mongodb-kubernetes/templates/operator-roles-webhook.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: mongodb-kubernetes-operator-mongodb-cluster-telemetry-binding + name: mongodb-kubernetes-operator-mongodb-webhook-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: mongodb-kubernetes-operator-cluster-telemetry + name: mongodb-kubernetes-operator-mongodb-webhook subjects: - kind: ServiceAccount name: mongodb-kubernetes-operator @@ -298,7 +305,7 @@ rules: - delete - get --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml +# Source: mongodb-kubernetes/templates/operator-roles-base.yaml kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -378,7 +385,14 @@ rules: - opsmanagers/status - mongodbmulticluster/status - mongodbsearch/status - +--- +# Source: mongodb-kubernetes/templates/operator-roles-pvc-resize.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-pvc-resize + namespace: mongodb +rules: - apiGroups: - '' resources: @@ -406,7 +420,7 @@ subjects: name: mongodb-kubernetes-appdb namespace: mongodb --- -# Source: mongodb-kubernetes/templates/operator-roles.yaml +# Source: mongodb-kubernetes/templates/operator-roles-base.yaml kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -421,6 +435,21 @@ subjects: name: mongodb-kubernetes-operator namespace: mongodb --- +# Source: mongodb-kubernetes/templates/operator-roles-pvc-resize.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-pvc-resize-binding + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-operator-pvc-resize +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- # Source: mongodb-kubernetes/templates/operator.yaml apiVersion: apps/v1 kind: Deployment @@ -445,9 +474,11 @@ spec: securityContext: runAsNonRoot: true runAsUser: 2000 + imagePullSecrets: + - name: prerelease-image-pullsecret containers: - name: mongodb-kubernetes-operator - image: "quay.io/mongodb/mongodb-kubernetes:1.2.0" + image: "quay.io/mongodb/staging/mongodb-kubernetes:1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2" imagePullPolicy: Always args: - -watch-resource=mongodb @@ -488,29 +519,29 @@ spec: value: Always # Database - name: MONGODB_ENTERPRISE_DATABASE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-database + value: quay.io/mongodb/staging/mongodb-kubernetes-database - name: INIT_DATABASE_IMAGE_REPOSITORY - value: quay.io/mongodb/mongodb-kubernetes-init-database + value: quay.io/mongodb/staging/mongodb-kubernetes-init-database - name: INIT_DATABASE_VERSION - value: 1.2.0 + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 - name: DATABASE_VERSION - value: 1.2.0 + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 # Ops Manager - name: OPS_MANAGER_IMAGE_REPOSITORY value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY - value: quay.io/mongodb/mongodb-kubernetes-init-ops-manager + value: quay.io/mongodb/staging/mongodb-kubernetes-init-ops-manager - name: INIT_OPS_MANAGER_VERSION - value: 1.2.0 + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 # AppDB - name: INIT_APPDB_IMAGE_REPOSITORY - value: quay.io/mongodb/mongodb-kubernetes-init-appdb + value: quay.io/mongodb/staging/mongodb-kubernetes-init-appdb - name: INIT_APPDB_VERSION - value: 1.2.0 + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 - name: OPS_MANAGER_IMAGE_PULL_POLICY value: Always - name: AGENT_IMAGE - value: "quay.io/mongodb/mongodb-agent:108.0.2.8729-1" + value: "quay.io/mongodb/mongodb-agent:108.0.12.8846-1" - name: MDB_AGENT_IMAGE_REPOSITORY value: "quay.io/mongodb/mongodb-agent" - name: MONGODB_IMAGE @@ -521,6 +552,8 @@ spec: value: ubi8 - name: PERFORM_FAILOVER value: 'true' + - name: IMAGE_PULL_SECRETS + value: prerelease-image-pullsecret - name: MDB_MAX_CONCURRENT_RECONCILES value: "1" - name: POD_NAME @@ -544,9 +577,9 @@ spec: value: "ubi8" # Community Env Vars End - name: MDB_SEARCH_COMMUNITY_REPO_URL - value: "quay.io/mongodb" + value: "quay.io/mongodb/staging" - name: MDB_SEARCH_COMMUNITY_NAME - value: "mongodb-search-community" + value: "mongodb-search" - name: MDB_SEARCH_COMMUNITY_VERSION - value: "1.47.0" + value: "latest" diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0110_wait_for_operator_deployment.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0110_wait_for_operator_deployment.out new file mode 100644 index 000000000..7c8a8b287 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0110_wait_for_operator_deployment.out @@ -0,0 +1,9 @@ +Waiting for deployment "mongodb-kubernetes-operator" rollout to finish: 0 of 1 updated replicas are available... +deployment "mongodb-kubernetes-operator" successfully rolled out +Operator deployment in mongodb namespace +NAME READY UP-TO-DATE AVAILABLE AGE +mongodb-kubernetes-operator 1/1 1 1 2s + +Operator pod in mongodb namespace +NAME READY STATUS RESTARTS AGE +mongodb-kubernetes-operator-5b44cd54d4-rjxrj 1/1 Running 0 3s diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0315_wait_for_community_resource.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0315_wait_for_community_resource.out new file mode 100644 index 000000000..3110199cb --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0315_wait_for_community_resource.out @@ -0,0 +1,16 @@ +Waiting for MongoDBCommunity resource to reach Running phase... +mongodbcommunity.mongodbcommunity.mongodb.com/mdbc-rs condition met + +MongoDBCommunity resource +NAME PHASE VERSION +mdbc-rs Running 8.0.10 + +Pods running in cluster kind-kind +NAME READY STATUS RESTARTS AGE +mdb-debug-mdbc-rs-0-0 1/1 Running 0 118s +mdb-debug-mdbc-rs-1-0 1/1 Running 0 118s +mdb-debug-mdbc-rs-2-0 1/1 Running 0 117s +mdbc-rs-0 2/2 Running 0 2m11s +mdbc-rs-1 2/2 Running 0 83s +mdbc-rs-2 2/2 Running 0 34s +mongodb-kubernetes-operator-5b44cd54d4-rjxrj 1/1 Running 0 2m16s diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0330_wait_for_community_resource.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0330_wait_for_community_resource.out new file mode 100644 index 000000000..6971e0146 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0330_wait_for_community_resource.out @@ -0,0 +1,2 @@ +Waiting for MongoDBCommunity resource to reach Running phase... +mongodbcommunity.mongodbcommunity.mongodb.com/mdbc-rs condition met diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0335_show_running_pods.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0335_show_running_pods.out new file mode 100644 index 000000000..144cefd50 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0335_show_running_pods.out @@ -0,0 +1,20 @@ + +MongoDBCommunity resource +NAME PHASE VERSION +mdbc-rs Running 8.0.10 + +MongoDBSearch resource +NAME PHASE AGE +mdbc-rs Running 5m33s + +Pods running in cluster kind-kind +NAME READY STATUS RESTARTS AGE +mdb-debug-mdbc-rs-0-0 1/1 Running 0 7m32s +mdb-debug-mdbc-rs-1-0 1/1 Running 0 7m32s +mdb-debug-mdbc-rs-2-0 1/1 Running 0 7m31s +mdb-debug-mdbc-rs-search-0-0 1/1 Running 0 5m32s +mdbc-rs-0 2/2 Running 1 (33s ago) 7m45s +mdbc-rs-1 2/2 Running 1 (3m11s ago) 6m57s +mdbc-rs-2 2/2 Running 1 (113s ago) 6m8s +mdbc-rs-search-0 1/1 Running 4 (4m20s ago) 5m33s +mongodb-kubernetes-operator-5b44cd54d4-rjxrj 1/1 Running 0 7m50s diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0420_import_movies_mflix_database.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0420_import_movies_mflix_database.out new file mode 100644 index 000000000..a7432faf0 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0420_import_movies_mflix_database.out @@ -0,0 +1,2 @@ +Downloading sample database archive... +Restoring sample database diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0440_wait_for_search_index_ready.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0440_wait_for_search_index_ready.out new file mode 100644 index 000000000..b264580af --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0440_wait_for_search_index_ready.out @@ -0,0 +1 @@ +Sleeping to wait for search indexes to be created diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0444_list_search_indexes.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0444_list_search_indexes.out new file mode 100644 index 000000000..66bd52f34 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0444_list_search_indexes.out @@ -0,0 +1,32 @@ +{ + cursor: { + id: 0, + ns: 'sample_mflix.movies', + firstBatch: [ + { + id: '68b5d833862bdd46756f23fe', + name: 'default', + type: 'search', + latestDefinition: { + indexID: ObjectId('68b5d833862bdd46756f23fe'), + name: 'default', + database: 'sample_mflix', + lastObservedCollectionName: 'movies', + collectionUUID: UUID('eb5bdd9e-fe14-4a0e-a012-5af2d7324d7a'), + numPartitions: 1, + mappings: { dynamic: true, fields: {} }, + indexFeatureVersion: 3 + } + } + ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1756747889, i: 1 }), + signature: { + hash: Binary.createFromBase64('BJNOThO1gxmbkTscNSABaUstAy0=', 0), + keyId: Long('7545172961045512197') + } + }, + operationTime: Timestamp({ t: 1756747889, i: 1 }) +} diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0445_list_vector_search_indexes.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0445_list_vector_search_indexes.out new file mode 100644 index 000000000..afac79fb5 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0445_list_vector_search_indexes.out @@ -0,0 +1,40 @@ +{ + cursor: { + id: 0, + ns: 'sample_mflix.embedded_movies', + firstBatch: [ + { + id: '68b5d834862bdd46756f23ff', + name: 'vector_index', + type: 'vectorSearch', + latestDefinition: { + type: 'vectorSearch', + indexID: ObjectId('68b5d834862bdd46756f23ff'), + name: 'vector_index', + database: 'sample_mflix', + lastObservedCollectionName: 'embedded_movies', + collectionUUID: UUID('98d05ae8-9fa5-4813-9833-54355c75a016'), + numPartitions: 1, + fields: [ + { + type: 'vector', + path: 'plot_embedding_voyage_3_large', + numDimensions: 2048, + similarity: 'dotProduct', + quantization: 'scalar' + } + ] + } + } + ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1756747889, i: 1 }), + signature: { + hash: Binary.createFromBase64('BJNOThO1gxmbkTscNSABaUstAy0=', 0), + keyId: Long('7545172961045512197') + } + }, + operationTime: Timestamp({ t: 1756747889, i: 1 }) +} diff --git a/docs/community-search/quick-start/output/0450_execute_search_query.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0450_execute_search_query.out similarity index 100% rename from docs/community-search/quick-start/output/0450_execute_search_query.out rename to scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0450_execute_search_query.out diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0455_execute_vector_search_query.out b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0455_execute_vector_search_query.out new file mode 100644 index 000000000..fb9f3de41 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/03_0455_execute_vector_search_query.out @@ -0,0 +1,54 @@ +mdbc-rs [primary] test> switched to db sample_mflix +mdbc-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'At the age of 21, Tim discovers he can travel in time and change what happens and has happened in his own life. His decision to make his world a better place by getting a girlfriend turns out not to be as easy as you might think.', + title: 'About Time', + score: 0.7704131603240967 + }, + { + plot: 'A psychiatrist makes multiple trips through time to save a woman that was murdered by her brutal husband.', + title: 'Retroactive', + score: 0.7597770690917969 + }, + { + plot: 'An officer for a security agency that regulates time travel, must fend for his life against a shady politician who has a tie to his past.', + title: 'Timecop', + score: 0.7574796676635742 + }, + { + plot: 'A time-travel experiment in which a robot probe is sent from the year 2073 to the year 1973 goes terribly wrong thrusting one of the project scientists, a man named Nicholas Sinclair into a...', + title: 'A.P.E.X.', + score: 0.7573235034942627 + }, + { + plot: 'After visiting 2015, Marty McFly must repeat his visit to 1955 to prevent disastrous changes to 1985... without interfering with his first trip.', + title: 'Back to the Future Part II', + score: 0.751945972442627 + }, + { + plot: 'A reporter, learning of time travelers visiting 20th century disasters, tries to change the history they know by averting upcoming disasters.', + title: 'Thrill Seekers', + score: 0.7503504753112793 + }, + { + plot: 'Hoping to alter the events of the past, a 19th century inventor instead travels 800,000 years into the future, where he finds humankind divided into two warring races.', + title: 'The Time Machine', + score: 0.750007152557373 + }, + { + plot: 'Lyle, a motorcycle champion is traveling the Mexican desert, when he find himself in the action radius of a time machine. So he find himself one century back in the past between rapists, ...', + title: 'Timerider: The Adventure of Lyle Swann', + score: 0.7499568462371826 + }, + { + plot: 'A romantic drama about a Chicago librarian with a gene that causes him to involuntarily time travel, and the complications it creates for his marriage.', + title: "The Time Traveler's Wife", + score: 0.7492842674255371 + }, + { + plot: 'A modern aircraft carrier is thrown back in time to 1941 near Hawaii, just hours before the Japanese attack on Pearl Harbor.', + title: 'The Final Countdown', + score: 0.7472751140594482 + } +] +mdbc-rs [primary] sample_mflix> \ No newline at end of file diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0090_helm_add_mogodb_repo.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0090_helm_add_mogodb_repo.out new file mode 100644 index 000000000..b1576c236 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0090_helm_add_mogodb_repo.out @@ -0,0 +1,6 @@ +"mongodb" already exists with the same configuration, skipping +Hang tight while we grab the latest from your chart repositories... +...Successfully got an update from the "mongodb" chart repository +Update Complete. ⎈Happy Helming!⎈ +NAME CHART VERSION APP VERSION DESCRIPTION +mongodb/mongodb-kubernetes 1.2.0 MongoDB Controllers for Kubernetes translate th... diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0100_install_operator.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0100_install_operator.out new file mode 100644 index 000000000..0f016af81 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0100_install_operator.out @@ -0,0 +1,585 @@ +Release "mongodb-kubernetes" does not exist. Installing it now. +NAME: mongodb-kubernetes +LAST DEPLOYED: Mon Sep 1 18:24:58 2025 +NAMESPACE: mongodb +STATUS: deployed +REVISION: 1 +TEST SUITE: None +USER-SUPPLIED VALUES: +registry: + imagePullSecrets: prerelease-image-pullsecret + +COMPUTED VALUES: +agent: + name: mongodb-agent + version: 108.0.12.8846-1 +community: + agent: + name: mongodb-agent + version: 108.0.2.8729-1 + mongodb: + imageType: ubi8 + name: mongodb-community-server + repo: quay.io/mongodb + name: mongodb-database + registry: + agent: quay.io/mongodb + resource: + members: 3 + name: mongodb-replica-set + tls: + caCertificateSecretRef: tls-ca-key-pair + certManager: + certDuration: 8760h + renewCertBefore: 720h + certificateKeySecretRef: tls-certificate + enabled: false + sampleX509User: false + useCertManager: true + useX509: false + version: 4.4.0 +database: + name: mongodb-kubernetes-database + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +initAppDb: + name: mongodb-kubernetes-init-appdb + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +initDatabase: + name: mongodb-kubernetes-init-database + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +initOpsManager: + name: mongodb-kubernetes-init-ops-manager + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +managedSecurityContext: false +mongodb: + appdbAssumeOldFormat: false + imageType: ubi8 + name: mongodb-enterprise-server + repo: quay.io/mongodb +multiCluster: + clusterClientTimeout: 10 + clusters: [] + kubeConfigSecretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + performFailOver: true +operator: + additionalArguments: [] + affinity: {} + baseName: mongodb-kubernetes + createOperatorServiceAccount: true + createResourcesServiceAccountsAndRoles: true + deployment_name: mongodb-kubernetes-operator + enableClusterMongoDBRoles: true + enablePVCResize: true + env: prod + maxConcurrentReconciles: 1 + mdbDefaultArchitecture: non-static + name: mongodb-kubernetes-operator + nodeSelector: {} + operator_image_name: mongodb-kubernetes + replicas: 1 + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + telemetry: + collection: + clusters: {} + deployments: {} + frequency: 1h + operators: {} + send: + frequency: 168h + tolerations: [] + vaultSecretBackend: + enabled: false + tlsSecretRef: "" + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + watchedResources: + - mongodb + - opsmanagers + - mongodbusers + - mongodbcommunity + - mongodbsearch + webhook: + installClusterRole: true + registerConfiguration: true +opsManager: + name: mongodb-enterprise-ops-manager-ubi +readinessProbe: + name: mongodb-kubernetes-readinessprobe + version: 1.0.22 +registry: + agent: quay.io/mongodb + appDb: quay.io/mongodb/staging + database: quay.io/mongodb/staging + imagePullSecrets: prerelease-image-pullsecret + initAppDb: quay.io/mongodb/staging + initDatabase: quay.io/mongodb/staging + initOpsManager: quay.io/mongodb/staging + operator: quay.io/mongodb/staging + opsManager: quay.io/mongodb + pullPolicy: Always + readinessProbe: quay.io/mongodb + versionUpgradeHook: quay.io/mongodb +search: + community: + name: mongodb-search + repo: quay.io/mongodb/staging + version: latest +versionUpgradeHook: + name: mongodb-kubernetes-operator-version-upgrade-post-start-hook + version: 1.0.9 + +HOOKS: +MANIFEST: +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-appdb + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-database-pods + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-ops-manager + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/operator-sa.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/operator-roles-clustermongodbroles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-cluster-mongodb-role +rules: + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - clustermongodbroles +--- +# Source: mongodb-kubernetes/templates/operator-roles-telemetry.yaml +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: mongodb-kubernetes/templates/operator-roles-webhook.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Source: mongodb-kubernetes/templates/operator-roles-clustermongodbroles.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-cluster-mongodb-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-kubernetes-operator-mongodb-cluster-mongodb-role +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-telemetry.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-kubernetes-operator-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-webhook.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-kubernetes-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-appdb + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get +--- +# Source: mongodb-kubernetes/templates/operator-roles-base.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodbcommunity.mongodb.com + resources: + - mongodbcommunity + - mongodbcommunity/status + - mongodbcommunity/spec + - mongodbcommunity/finalizers + verbs: + - '*' + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodbsearch + - mongodbsearch/finalizers + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + - mongodbsearch/status +--- +# Source: mongodb-kubernetes/templates/operator-roles-pvc-resize.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-pvc-resize + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-appdb + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-appdb +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-appdb + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-base.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-operator +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-pvc-resize.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-pvc-resize-binding + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-operator-pvc-resize +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-kubernetes-operator + app.kubernetes.io/instance: mongodb-kubernetes-operator + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-kubernetes-operator + app.kubernetes.io/instance: mongodb-kubernetes-operator + spec: + serviceAccountName: mongodb-kubernetes-operator + securityContext: + runAsNonRoot: true + runAsUser: 2000 + imagePullSecrets: + - name: prerelease-image-pullsecret + containers: + - name: mongodb-kubernetes-operator + image: "quay.io/mongodb/staging/mongodb-kubernetes:1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2" + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + - -watch-resource=mongodbcommunity + - -watch-resource=mongodbsearch + - -watch-resource=clustermongodbroles + command: + - /usr/local/bin/mongodb-kubernetes-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: prod + - name: MDB_DEFAULT_ARCHITECTURE + value: non-static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1h" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/staging/mongodb-kubernetes-database + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/staging/mongodb-kubernetes-init-database + - name: INIT_DATABASE_VERSION + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + - name: DATABASE_VERSION + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/staging/mongodb-kubernetes-init-ops-manager + - name: INIT_OPS_MANAGER_VERSION + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/staging/mongodb-kubernetes-init-appdb + - name: INIT_APPDB_VERSION + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent:108.0.12.8846-1" + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: ubi8 + - name: PERFORM_FAILOVER + value: 'true' + - name: IMAGE_PULL_SECRETS + value: prerelease-image-pullsecret + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "1" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: mongodb-kubernetes-operator + # Community Env Vars Start + - name: MDB_COMMUNITY_AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent:108.0.2.8729-1" + - name: VERSION_UPGRADE_HOOK_IMAGE + value: "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9" + - name: READINESS_PROBE_IMAGE + value: "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22" + - name: MDB_COMMUNITY_IMAGE + value: "mongodb-community-server" + - name: MDB_COMMUNITY_REPO_URL + value: "quay.io/mongodb" + - name: MDB_COMMUNITY_IMAGE_TYPE + value: "ubi8" + # Community Env Vars End + - name: MDB_SEARCH_COMMUNITY_REPO_URL + value: "quay.io/mongodb/staging" + - name: MDB_SEARCH_COMMUNITY_NAME + value: "mongodb-search" + - name: MDB_SEARCH_COMMUNITY_VERSION + value: "latest" + diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0310_wait_for_database_resource.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0310_wait_for_database_resource.out new file mode 100644 index 000000000..9a5c191d1 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0310_wait_for_database_resource.out @@ -0,0 +1,16 @@ +Waiting for MongoDB resource to reach Running phase... +mongodb.mongodb.com/mdb-rs condition met + +MongoDB resource +NAME PHASE VERSION TYPE AGE +mdb-rs Running 8.0.10 ReplicaSet 2m46s + +Pods running in cluster kind-kind +NAME READY STATUS RESTARTS AGE +mdb-debug-mdb-rs-0-0 1/1 Running 0 2m26s +mdb-debug-mdb-rs-1-0 1/1 Running 0 2m26s +mdb-debug-mdb-rs-2-0 1/1 Running 0 2m25s +mdb-rs-0 1/1 Running 0 2m29s +mdb-rs-1 1/1 Running 0 89s +mdb-rs-2 1/1 Running 0 46s +mongodb-kubernetes-operator-5b44cd54d4-kvrpv 1/1 Running 0 2m47s diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0335_show_running_pods.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0335_show_running_pods.out new file mode 100644 index 000000000..97734a89d --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/02_0335_show_running_pods.out @@ -0,0 +1,20 @@ + +MongoDB resource +NAME PHASE VERSION TYPE AGE +mdb-rs Running 8.0.10 ReplicaSet 4m26s + +MongoDBSearch resource +NAME PHASE AGE +mdb-rs Running 97s + +Pods running in cluster kind-kind +NAME READY STATUS RESTARTS AGE +mdb-debug-mdb-rs-0-0 1/1 Running 0 4m7s +mdb-debug-mdb-rs-1-0 1/1 Running 0 4m7s +mdb-debug-mdb-rs-2-0 1/1 Running 0 4m6s +mdb-debug-mdb-rs-search-0-0 1/1 Running 0 97s +mdb-rs-0 1/1 Running 0 4m10s +mdb-rs-1 1/1 Running 0 3m10s +mdb-rs-2 1/1 Running 0 2m27s +mdb-rs-search-0 1/1 Running 0 29s +mongodb-kubernetes-operator-5b44cd54d4-kvrpv 1/1 Running 0 4m28s diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0420_import_movies_mflix_database.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0420_import_movies_mflix_database.out new file mode 100644 index 000000000..a7432faf0 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0420_import_movies_mflix_database.out @@ -0,0 +1,2 @@ +Downloading sample database archive... +Restoring sample database diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0440_wait_for_search_index_ready.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0440_wait_for_search_index_ready.out new file mode 100644 index 000000000..b264580af --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0440_wait_for_search_index_ready.out @@ -0,0 +1 @@ +Sleeping to wait for search indexes to be created diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0444_list_search_indexes.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0444_list_search_indexes.out new file mode 100644 index 000000000..09342c66e --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0444_list_search_indexes.out @@ -0,0 +1,32 @@ +{ + cursor: { + id: 0, + ns: 'sample_mflix.movies', + firstBatch: [ + { + id: '68b5c9f984d938552792ba0c', + name: 'default', + type: 'search', + latestDefinition: { + indexID: ObjectId('68b5c9f984d938552792ba0c'), + name: 'default', + database: 'sample_mflix', + lastObservedCollectionName: 'movies', + collectionUUID: UUID('b4127db1-78ed-4f2a-b8e8-54a826075622'), + numPartitions: 1, + mappings: { dynamic: true, fields: {} }, + indexFeatureVersion: 3 + } + } + ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1756744247, i: 1 }), + signature: { + hash: Binary.createFromBase64('7Ylj1qg2orgtBnElSVeObPTpTjw=', 0), + keyId: Long('7545158272257359877') + } + }, + operationTime: Timestamp({ t: 1756744247, i: 1 }) +} diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0445_list_vector_search_indexes.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0445_list_vector_search_indexes.out new file mode 100644 index 000000000..a8d0a7470 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0445_list_vector_search_indexes.out @@ -0,0 +1,40 @@ +{ + cursor: { + id: 0, + ns: 'sample_mflix.embedded_movies', + firstBatch: [ + { + id: '68b5c9fa84d938552792ba0d', + name: 'vector_index', + type: 'vectorSearch', + latestDefinition: { + type: 'vectorSearch', + indexID: ObjectId('68b5c9fa84d938552792ba0d'), + name: 'vector_index', + database: 'sample_mflix', + lastObservedCollectionName: 'embedded_movies', + collectionUUID: UUID('c9a6ef19-2993-4edc-904e-31964b889467'), + numPartitions: 1, + fields: [ + { + type: 'vector', + path: 'plot_embedding_voyage_3_large', + numDimensions: 2048, + similarity: 'dotProduct', + quantization: 'scalar' + } + ] + } + } + ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1756744247, i: 1 }), + signature: { + hash: Binary.createFromBase64('7Ylj1qg2orgtBnElSVeObPTpTjw=', 0), + keyId: Long('7545158272257359877') + } + }, + operationTime: Timestamp({ t: 1756744247, i: 1 }) +} diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0450_execute_search_query.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0450_execute_search_query.out new file mode 100644 index 000000000..f425a96d8 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0450_execute_search_query.out @@ -0,0 +1,22 @@ +mdb-rs [primary] test> switched to db sample_mflix +mdb-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'A sports agent stages an unconventional recruitment strategy to get talented Indian cricket players to play Major League Baseball.', + genres: [ 'Biography', 'Drama', 'Sport' ], + title: 'Million Dollar Arm', + released: ISODate('2014-05-16T00:00:00.000Z') + }, + { + plot: 'A Taiwanese high school baseball team travels to Japan in 1931 to compete in a national tournament.', + genres: [ 'Biography', 'Drama', 'History' ], + title: 'Kano', + released: ISODate('2014-02-27T00:00:00.000Z') + }, + { + plot: "12-year-old Josh is a mixed race boy and a promising baseball player. He is abused by his mother's boyfriend Byrd, and neglected by his mother Debbie. He forges his own path in life when ...", + genres: [ 'Drama' ], + title: 'Calloused Hands', + released: ISODate('2013-03-03T00:00:00.000Z') + } +] +mdb-rs [primary] sample_mflix> \ No newline at end of file diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0455_execute_vector_search_query.out b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0455_execute_vector_search_query.out new file mode 100644 index 000000000..1623a6fdc --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_enterprise_snippets/03_0455_execute_vector_search_query.out @@ -0,0 +1,54 @@ +mdb-rs [primary] test> switched to db sample_mflix +mdb-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'At the age of 21, Tim discovers he can travel in time and change what happens and has happened in his own life. His decision to make his world a better place by getting a girlfriend turns out not to be as easy as you might think.', + title: 'About Time', + score: 0.7704131603240967 + }, + { + plot: 'A psychiatrist makes multiple trips through time to save a woman that was murdered by her brutal husband.', + title: 'Retroactive', + score: 0.7597770690917969 + }, + { + plot: 'An officer for a security agency that regulates time travel, must fend for his life against a shady politician who has a tie to his past.', + title: 'Timecop', + score: 0.7574796676635742 + }, + { + plot: 'A time-travel experiment in which a robot probe is sent from the year 2073 to the year 1973 goes terribly wrong thrusting one of the project scientists, a man named Nicholas Sinclair into a...', + title: 'A.P.E.X.', + score: 0.7573235034942627 + }, + { + plot: 'After visiting 2015, Marty McFly must repeat his visit to 1955 to prevent disastrous changes to 1985... without interfering with his first trip.', + title: 'Back to the Future Part II', + score: 0.751945972442627 + }, + { + plot: 'A reporter, learning of time travelers visiting 20th century disasters, tries to change the history they know by averting upcoming disasters.', + title: 'Thrill Seekers', + score: 0.7503504753112793 + }, + { + plot: 'Hoping to alter the events of the past, a 19th century inventor instead travels 800,000 years into the future, where he finds humankind divided into two warring races.', + title: 'The Time Machine', + score: 0.750007152557373 + }, + { + plot: 'Lyle, a motorcycle champion is traveling the Mexican desert, when he find himself in the action radius of a time machine. So he find himself one century back in the past between rapists, ...', + title: 'Timerider: The Adventure of Lyle Swann', + score: 0.7499568462371826 + }, + { + plot: 'A romantic drama about a Chicago librarian with a gene that causes him to involuntarily time travel, and the complications it creates for his marriage.', + title: "The Time Traveler's Wife", + score: 0.7492842674255371 + }, + { + plot: 'A modern aircraft carrier is thrown back in time to 1941 near Hawaii, just hours before the Japanese attack on Pearl Harbor.', + title: 'The Final Countdown', + score: 0.7472751140594482 + } +] +mdb-rs [primary] sample_mflix> \ No newline at end of file diff --git a/scripts/code_snippets/tests/test_kind_search_community_snippets.sh b/scripts/code_snippets/tests/test_kind_search_community_snippets.sh new file mode 100755 index 000000000..522433a7a --- /dev/null +++ b/scripts/code_snippets/tests/test_kind_search_community_snippets.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eou pipefail +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + +source scripts/dev/set_env_context.sh + +script_name=$(readlink -f "${BASH_SOURCE[0]}") + +_SNIPPETS_OUTPUT_DIR="$(dirname "${script_name}")/outputs/$(basename "${script_name%.*}")" +export _SNIPPETS_OUTPUT_DIR +mkdir -p "${_SNIPPETS_OUTPUT_DIR}" + +dump_logs() { + if [[ "${SKIP_DUMP:-"false"}" != "true" ]]; then + scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh "${K8S_CTX}" + fi +} +trap dump_logs EXIT + +test_dir="./docs/search/01-search-community-deploy" +source "${test_dir}/env_variables.sh" +echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" +# shellcheck disable=SC1090 +test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" +${test_dir}/test.sh + +test_dir="./docs/search/03-search-query-usage" +echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" +# shellcheck disable=SC1090 +test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" + +export MDB_RESOURCE_NAME="mdbc-rs" +export MDB_CONNECTION_STRING="mongodb://mdb-user:${MDB_USER_PASSWORD}@${MDB_RESOURCE_NAME}-0.${MDB_RESOURCE_NAME}-svc.${MDB_NS}.svc.cluster.local:27017/?replicaSet=${MDB_RESOURCE_NAME}" + +${test_dir}/test.sh + diff --git a/scripts/code_snippets/tests/test_kind_search_enterprise_snippets.sh b/scripts/code_snippets/tests/test_kind_search_enterprise_snippets.sh new file mode 100755 index 000000000..945053844 --- /dev/null +++ b/scripts/code_snippets/tests/test_kind_search_enterprise_snippets.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eou pipefail +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + +source scripts/dev/set_env_context.sh + +script_name=$(readlink -f "${BASH_SOURCE[0]}") + +_SNIPPETS_OUTPUT_DIR="$(dirname "${script_name}")/outputs/$(basename "${script_name%.*}")" +export _SNIPPETS_OUTPUT_DIR +mkdir -p "${_SNIPPETS_OUTPUT_DIR}" + +dump_logs() { + if [[ "${SKIP_DUMP:-"false"}" != "true" ]]; then + scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh "${K8S_CTX}" + fi +} +trap dump_logs EXIT + +test_dir="./docs/search/02-search-enterprise-deploy" +source "${test_dir}/env_variables.sh" +echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" +# shellcheck disable=SC1090 +test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" +${test_dir}/test.sh + +test_dir="./docs/search/03-search-query-usage" +echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" +# shellcheck disable=SC1090 +test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" + +export MDB_CONNECTION_STRING="mongodb://mdb-user:${MDB_USER_PASSWORD}@${MDB_RESOURCE_NAME}-0.${MDB_RESOURCE_NAME}-svc.${MDB_NS}.svc.cluster.local:27017/?replicaSet=${MDB_RESOURCE_NAME}" + +${test_dir}/test.sh diff --git a/scripts/code_snippets/validate_snippets.py b/scripts/code_snippets/validate_snippets.py new file mode 100755 index 000000000..be797add9 --- /dev/null +++ b/scripts/code_snippets/validate_snippets.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import os +import sys +from collections import defaultdict + + +def find_snippet_directories(): + """Find all directories containing both test.sh and code_snippets subdirectory.""" + snippet_dirs = [] + + # Traverse current directory recursively to find test.sh files + for root, dirs, files in os.walk("."): + if "test.sh" in files: + # Check if this directory also has a code_snippets subdirectory + code_snippets_path = os.path.join(root, "code_snippets") + if os.path.isdir(code_snippets_path): + snippet_dirs.append(root) + + return snippet_dirs + + +def verify_snippets_files_are_unique(): + """Check if files in snippet directories have unique names across all directories.""" + dirs = find_snippet_directories() + + if not dirs: + print("No snippet directories found (no test.sh files).") + return True + + file_map = defaultdict(list) + + print(f"Checking for duplicate file names across code snippet directories:\n {"\t\n".join(dirs)}") + + # Scan all files in code_snippets subdirectories only + for snippet_dir in dirs: + code_snippets_dir = os.path.join(snippet_dir, "code_snippets") + if os.path.exists(code_snippets_dir): + for file in os.listdir(code_snippets_dir): + file_path = os.path.join(code_snippets_dir, file) + if os.path.isfile(file_path): + file_map[file].append(file_path) + + # Check for duplicates + duplicates_found = False + for filename, paths in file_map.items(): + if len(paths) > 1: + if not duplicates_found: + print("ERROR: Duplicate file names found:") + duplicates_found = True + print(f" File '{filename}' appears in multiple locations:") + for path in sorted(paths): + print(f" {path}") + print() + + if duplicates_found: + print("Failure - please rename duplicate files to ensure uniqueness across all snippet directories.") + return False + else: + print("OK - all snippets have unique names across directories.") + return True + + +if __name__ == "__main__": + checks = [verify_snippets_files_are_unique()] + + # Exit 0 if all checks pass, 1 if any fail + sys.exit(0 if all(checks) else 1) diff --git a/scripts/dev/configure_container_auth.sh b/scripts/dev/configure_container_auth.sh index 1afd8429d..c221603a2 100755 --- a/scripts/dev/configure_container_auth.sh +++ b/scripts/dev/configure_container_auth.sh @@ -158,12 +158,12 @@ fi aws ecr get-login-password --region "eu-west-1" | registry_login "AWS" "268558157000.dkr.ecr.eu-west-1.amazonaws.com" -if [[ -n "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then +if [[ -n "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then # log in to quay.io for the mongodb/mongodb-search-community private repo # TODO remove once we switch to the official repo in Public Preview quay_io_auth_file=$(mktemp) config_tmp=$(mktemp) - echo "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}" + echo "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}" exec_cmd jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}" exec_cmd mv "${config_tmp}" "${CONFIG_PATH}" rm "${quay_io_auth_file}" diff --git a/scripts/dev/contexts/prerelease_kind_code_snippets b/scripts/dev/contexts/prerelease_kind_code_snippets index d6fea1d84..4248bb9ee 100644 --- a/scripts/dev/contexts/prerelease_kind_code_snippets +++ b/scripts/dev/contexts/prerelease_kind_code_snippets @@ -12,3 +12,5 @@ source "${script_dir}/root-context" export NAMESPACE=mongodb export CODE_SNIPPETS_FLAVOR=e2e_prerelease export CODE_SNIPPETS_COMMIT_OUTPUT=true + +export PRERELEASE_IMAGE_PULLSECRET="${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" diff --git a/scripts/dev/contexts/private-context-template b/scripts/dev/contexts/private-context-template index f1809728e..2a95eaf42 100644 --- a/scripts/dev/contexts/private-context-template +++ b/scripts/dev/contexts/private-context-template @@ -97,8 +97,7 @@ export e2e_cloud_qa_orgid_owner_static_2="${OM_ORGID}" export e2e_cloud_qa_apikey_owner_static_2="${OM_API_KEY}" export e2e_cloud_qa_user_owner_static_2="${OM_USER}" -# TODO to be removed at public preview stage of community-search -export COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON="" +export PRERELEASE_PULLSECRET_DOCKERCONFIGJSON="" # uncomment to enable license update with pre-commit script # export MDB_UPDATE_LICENSES=true diff --git a/scripts/dev/contexts/private_kind_code_snippets b/scripts/dev/contexts/private_kind_code_snippets index 704957a6f..6ae3751d8 100644 --- a/scripts/dev/contexts/private_kind_code_snippets +++ b/scripts/dev/contexts/private_kind_code_snippets @@ -8,6 +8,9 @@ script_name=$(readlink -f "${BASH_SOURCE[0]}") script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" +source "${script_dir}/e2e_mdb_community" export NAMESPACE=mongodb export CODE_SNIPPETS_FLAVOR=e2e_private + +export ops_manager_version="cloud_qa" diff --git a/scripts/dev/update_docs_snippets.sh b/scripts/dev/update_docs_snippets.sh index b9a45b378..bdec61db2 100755 --- a/scripts/dev/update_docs_snippets.sh +++ b/scripts/dev/update_docs_snippets.sh @@ -46,17 +46,16 @@ function prepare_repositories() { } function copy_files() { - samples_dir=$1 - dst_dir="${DOCS_DIR}/source/includes/code-examples/reference-architectures/${samples_dir}" - src_dir="${MCK_DIR}/public/architectures/${samples_dir}" + local src_dir="$1" + local dst_dir="$2" rm -rf "${dst_dir}" mkdir -p "${dst_dir}" - cp -r "${src_dir}/code_snippets" "${dst_dir}" - cp -r "${src_dir}/output" "${dst_dir}" || true - cp "${src_dir}/env_variables.sh" "${dst_dir}" || true - cp -r "${src_dir}/yamls" "${dst_dir}" || true + cp -r "${src_dir}/code_snippets" "${dst_dir}" 2>/dev/null || true + cp -r "${src_dir}/output" "${dst_dir}" 2>/dev/null || true + cp "${src_dir}/env_variables.sh" "${dst_dir}" 2>/dev/null || true + cp -r "${src_dir}/yamls" "${dst_dir}" 2>/dev/null || true } function prepare_docs_pr() { @@ -74,17 +73,26 @@ function prepare_docs_pr() { pushd ../ prepare_repositories -copy_files "ops-manager-multi-cluster" -copy_files "ops-manager-mc-no-mesh" -copy_files "mongodb-sharded-multi-cluster" -copy_files "mongodb-sharded-mc-no-mesh" -copy_files "mongodb-replicaset-multi-cluster" -copy_files "mongodb-replicaset-mc-no-mesh" -copy_files "setup-multi-cluster/verify-connectivity" -copy_files "setup-multi-cluster/setup-gke" -copy_files "setup-multi-cluster/setup-istio" -copy_files "setup-multi-cluster/setup-operator" -copy_files "setup-multi-cluster/setup-cert-manager" -copy_files "setup-multi-cluster/setup-externaldns" + +REF_ARCH_SRC_DIR="${MCK_DIR}/public/architectures" +REF_ARCH_DST_DIR="${DOCS_DIR}/source/includes/code-examples/reference-architectures" + +copy_files "${REF_ARCH_SRC_DIR}/ops-manager-multi-cluster" "${REF_ARCH_DST_DIR}/ops-manager-multi-cluster" +copy_files "${REF_ARCH_SRC_DIR}/ops-manager-mc-no-mesh" "${REF_ARCH_DST_DIR}/ops-manager-mc-no-mesh" +copy_files "${REF_ARCH_SRC_DIR}/mongodb-sharded-multi-cluster" "${REF_ARCH_DST_DIR}/mongodb-sharded-multi-cluster" +copy_files "${REF_ARCH_SRC_DIR}/mongodb-sharded-mc-no-mesh" "${REF_ARCH_DST_DIR}/mongodb-sharded-mc-no-mesh" +copy_files "${REF_ARCH_SRC_DIR}/mongodb-replicaset-multi-cluster" "${REF_ARCH_DST_DIR}/mongodb-replicaset-multi-cluster" +copy_files "${REF_ARCH_SRC_DIR}/mongodb-replicaset-mc-no-mesh" "${REF_ARCH_DST_DIR}/mongodb-replicaset-mc-no-mesh" +copy_files "${REF_ARCH_SRC_DIR}/setup-multi-cluster/verify-connectivity" "${REF_ARCH_DST_DIR}/setup-multi-cluster/verify-connectivity" +copy_files "${REF_ARCH_SRC_DIR}/setup-multi-cluster/setup-gke" "${REF_ARCH_DST_DIR}/setup-multi-cluster/setup-gke" +copy_files "${REF_ARCH_SRC_DIR}/setup-multi-cluster/setup-istio" "${REF_ARCH_DST_DIR}/setup-multi-cluster/setup-istio" +copy_files "${REF_ARCH_SRC_DIR}/setup-multi-cluster/setup-operator" "${REF_ARCH_DST_DIR}/setup-multi-cluster/setup-operator" +copy_files "${REF_ARCH_SRC_DIR}/setup-multi-cluster/setup-cert-manager" "${REF_ARCH_DST_DIR}/setup-multi-cluster/setup-cert-manager" +copy_files "${REF_ARCH_SRC_DIR}/setup-multi-cluster/setup-externaldns" "${REF_ARCH_DST_DIR}/setup-multi-cluster/setup-externaldns" + +DOCS_SNIPPETS_SRC_DIR="${MCK_DIR}/docs" +DOCS_SNIPPEES_DST_DIR="${DOCS_DIR}/source/includes/code-examples" +copy_files "${DOCS_SNIPPETS_SRC_DIR}/community-search/quick-start" "${DOCS_SNIPPEES_DST_DIR}/community-search/quick-start" + prepare_docs_pr popd From 88c129077d61c3c3ff4d2d2f331b4fdef1c5cd16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 8 Sep 2025 22:36:19 +0200 Subject: [PATCH 04/17] CLOUDP-312564: MongoDB Search - Public Preview Co-authored-by: Anand Singh Co-authored-by: Yavor Georgiev --- .evergreen-tasks.yml | 26 ++ .evergreen.yml | 6 + api/v1/search/mongodbsearch_types.go | 99 ++++- api/v1/search/zz_generated.deepcopy.go | 98 +++++ .../mongodb.com_clustermongodbroles.yaml | 2 +- config/crd/bases/mongodb.com_mongodb.yaml | 48 +-- .../mongodb.com_mongodbmulticluster.yaml | 48 +-- .../crd/bases/mongodb.com_mongodbsearch.yaml | 130 ++++--- .../crd/bases/mongodb.com_mongodbusers.yaml | 2 +- config/crd/bases/mongodb.com_opsmanagers.yaml | 112 +----- ...ommunity.mongodb.com_mongodbcommunity.yaml | 54 +-- .../operator/mongodbreplicaset_controller.go | 100 ++++- .../operator/mongodbsearch_controller.go | 80 +++- .../operator/mongodbsearch_controller_test.go | 153 ++++++-- .../operator/watch/config_change_handler.go | 12 +- .../community_search_source.go | 84 +++++ .../community_search_source_test.go | 208 +++++++++++ .../enterprise_search_source.go | 88 +++++ .../enterprise_search_source_test.go | 290 +++++++++++++++ .../external_search_source.go | 49 +++ .../mongodbsearch_reconcile_helper.go | 347 +++++++++++++++--- .../mongodbsearch_reconcile_helper_test.go | 62 +--- .../search_controller/search_construction.go | 136 +++---- .../edit_mms_configuration.go | 4 +- .../kubetester/helm.py | 3 + .../kubetester/mongodb.py | 6 +- .../common/search/movies_search_helper.py | 42 ++- .../tests/common/search/search_tester.py | 14 +- ...nity-replicaset-sample-mflix-external.yaml | 112 ++++++ .../community-replicaset-sample-mflix.yaml | 69 ++-- .../enterprise-replicaset-sample-mflix.yaml | 34 ++ .../fixtures/mongodbuser-mdb-admin.yaml | 16 + .../search/fixtures/mongodbuser-mdb-user.yaml | 16 + .../mongodbuser-search-sync-source-user.yaml | 18 + .../fixtures/search-with-user-password.yaml | 5 + .../tests/search/search_community_basic.py | 42 ++- .../search_community_external_mongod_basic.py | 144 ++++++++ .../search_community_external_mongod_tls.py | 190 ++++++++++ .../tests/search/search_community_tls.py | 173 +++++++++ .../tests/search/search_enterprise_basic.py | 183 +++++++++ .../tests/search/search_enterprise_tls.py | 239 ++++++++++++ .../crds/mongodb.com_mongodbsearch.yaml | 107 +++++- .../controllers/replica_set_controller.go | 76 +++- .../pkg/mongot/mongot_config.go | 89 +++-- mongodb-community-operator/pkg/tls/tls.go | 113 ++++++ pkg/telemetry/collector.go | 52 ++- pkg/telemetry/collector_test.go | 113 ++++-- public/crds.yaml | 107 +++++- scripts/dev/contexts/e2e_mdb_community | 2 + scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa | 2 + .../contexts/e2e_static_mdb_kind_ubi_cloudqa | 2 + scripts/dev/contexts/evg-private-context | 4 +- scripts/dev/contexts/root-context | 8 +- .../dev/contexts/variables/mongodb_search_dev | 8 + scripts/funcs/operator_deployment | 3 + 55 files changed, 3538 insertions(+), 692 deletions(-) create mode 100644 controllers/search_controller/community_search_source.go create mode 100644 controllers/search_controller/community_search_source_test.go create mode 100644 controllers/search_controller/enterprise_search_source.go create mode 100644 controllers/search_controller/enterprise_search_source_test.go create mode 100644 controllers/search_controller/external_search_source.go create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py create mode 100644 mongodb-community-operator/pkg/tls/tls.go create mode 100644 scripts/dev/contexts/variables/mongodb_search_dev diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml index 9b6cfd893..6ec158a41 100644 --- a/.evergreen-tasks.yml +++ b/.evergreen-tasks.yml @@ -1288,3 +1288,29 @@ tasks: tags: ["patch-run"] commands: - func: "e2e_test" + + - name: e2e_search_community_tls + tags: ["patch-run"] + commands: + - func: "e2e_test" + + - name: e2e_search_external_basic + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_search_external_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_search_enterprise_basic + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_search_enterprise_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + diff --git a/.evergreen.yml b/.evergreen.yml index ddf1ad3e8..dd3d3eaed 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -567,6 +567,9 @@ task_groups: tasks: - e2e_community_replicaset_scale - e2e_search_community_basic + - e2e_search_community_tls + - e2e_search_external_basic + - e2e_search_external_tls # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant - name: e2e_mdb_kind_cloudqa_task_group @@ -691,6 +694,9 @@ task_groups: - e2e_replica_set_oidc_workforce - e2e_sharded_cluster_oidc_m2m_group - e2e_sharded_cluster_oidc_m2m_user + # MongoDBSearch test group + - e2e_search_enterprise_basic + - e2e_search_enterprise_tls <<: *teardown_group # this task group contains just a one task, which is smoke testing whether the operator diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index 43e9be1be..5a8037b51 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -1,6 +1,8 @@ package search import ( + "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -14,8 +16,10 @@ import ( ) const ( - MongotDefaultPort = 27027 - MongotDefaultMetricsPort = 9946 + MongotDefaultPort = 27027 + MongotDefaultMetricsPort = 9946 + MongotDefautHealthCheckPort = 8080 + MongotDefaultSyncSourceUsername = "search-sync-source" ) func init() { @@ -23,21 +27,66 @@ func init() { } type MongoDBSearchSpec struct { + // Optional version of MongoDB Search component (mongot). If not set, then the operator will set the most appropriate version of MongoDB Search. // +optional Version string `json:"version"` + // MongoDB database connection details from which MongoDB Search will synchronize data to build indexes. // +optional Source *MongoDBSource `json:"source"` + // StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + // which aren't exposed as fields in the MongoDBSearch.spec. // +optional StatefulSetConfiguration *common.StatefulSetConfiguration `json:"statefulSet,omitempty"` + // Configure MongoDB Search's persistent volume. If not defined, the operator will request 10GB of storage. // +optional Persistence *common.Persistence `json:"persistence,omitempty"` + // Configure resource requests and limits for the MongoDB Search pods. // +optional ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"` + // Configure security settings of the MongoDB Search server that MongoDB database is connecting to when performing search queries. + // +optional + Security Security `json:"security"` } type MongoDBSource struct { // +optional MongoDBResourceRef *userv1.MongoDBResourceRef `json:"mongodbResourceRef,omitempty"` + // +optional + ExternalMongoDBSource *ExternalMongoDBSource `json:"external,omitempty"` + // +optional + PasswordSecretRef *userv1.SecretKeyRef `json:"passwordSecretRef,omitempty"` + // +optional + Username *string `json:"username,omitempty"` +} + +type ExternalMongoDBSource struct { + HostAndPorts []string `json:"hostAndPorts,omitempty"` + // mongod keyfile used to connect to the external MongoDB deployment + KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyFileSecretRef,omitempty"` + // +optional + TLS *ExternalMongodTLS `json:"tls,omitempty"` // TLS configuration for the external MongoDB deployment +} + +type ExternalMongodTLS struct { + Enabled bool `json:"enabled"` + // +optional + CA *corev1.LocalObjectReference `json:"ca,omitempty"` +} + +type Security struct { + // +optional + TLS TLS `json:"tls"` +} + +type TLS struct { + Enabled bool `json:"enabled"` + // CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + // The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + // This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + // Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + // If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + // +optional + CertificateKeySecret corev1.LocalObjectReference `json:"certificateKeySecretRef"` } type MongoDBSearchStatus struct { @@ -105,6 +154,25 @@ func (s *MongoDBSearch) MongotConfigConfigMapNamespacedName() types.NamespacedNa return types.NamespacedName{Name: s.Name + "-search-config", Namespace: s.Namespace} } +func (s *MongoDBSearch) SourceUserPasswordSecretRef() *userv1.SecretKeyRef { + if s.Spec.Source != nil && s.Spec.Source.PasswordSecretRef != nil { + return s.Spec.Source.PasswordSecretRef + } + + return &userv1.SecretKeyRef{ + Name: fmt.Sprintf("%s-%s-password", s.Name, MongotDefaultSyncSourceUsername), + Key: "password", + } +} + +func (s *MongoDBSearch) SourceUsername() string { + if s.Spec.Source != nil && s.Spec.Source.Username != nil { + return *s.Spec.Source.Username + } + + return MongotDefaultSyncSourceUsername +} + func (s *MongoDBSearch) StatefulSetNamespacedName() types.NamespacedName { return types.NamespacedName{Name: s.Name + "-search", Namespace: s.Namespace} } @@ -118,13 +186,17 @@ func (s *MongoDBSearch) GetOwnerReferences() []metav1.OwnerReference { return []metav1.OwnerReference{ownerReference} } -func (s *MongoDBSearch) GetMongoDBResourceRef() userv1.MongoDBResourceRef { +func (s *MongoDBSearch) GetMongoDBResourceRef() *userv1.MongoDBResourceRef { + if s.IsExternalMongoDBSource() { + return nil + } + mdbResourceRef := userv1.MongoDBResourceRef{Namespace: s.Namespace, Name: s.Name} if s.Spec.Source != nil && s.Spec.Source.MongoDBResourceRef != nil && s.Spec.Source.MongoDBResourceRef.Name != "" { mdbResourceRef.Name = s.Spec.Source.MongoDBResourceRef.Name } - return mdbResourceRef + return &mdbResourceRef } func (s *MongoDBSearch) GetMongotPort() int32 { @@ -134,3 +206,22 @@ func (s *MongoDBSearch) GetMongotPort() int32 { func (s *MongoDBSearch) GetMongotMetricsPort() int32 { return MongotDefaultMetricsPort } + +// TLSSecretNamespacedName will get the namespaced name of the Secret containing the server certificate and key +func (s *MongoDBSearch) TLSSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: s.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: s.Namespace} +} + +// TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator +// containing the combined certificate and key. +func (s *MongoDBSearch) TLSOperatorSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: s.Name + "-search-certificate-key", Namespace: s.Namespace} +} + +func (s *MongoDBSearch) GetMongotHealthCheckPort() int32 { + return MongotDefautHealthCheckPort +} + +func (s *MongoDBSearch) IsExternalMongoDBSource() bool { + return s.Spec.Source != nil && s.Spec.Source.ExternalMongoDBSource != nil +} diff --git a/api/v1/search/zz_generated.deepcopy.go b/api/v1/search/zz_generated.deepcopy.go index e9384e3de..c66322146 100644 --- a/api/v1/search/zz_generated.deepcopy.go +++ b/api/v1/search/zz_generated.deepcopy.go @@ -28,6 +28,56 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalMongoDBSource) DeepCopyInto(out *ExternalMongoDBSource) { + *out = *in + if in.HostAndPorts != nil { + in, out := &in.HostAndPorts, &out.HostAndPorts + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.KeyFileSecretKeyRef != nil { + in, out := &in.KeyFileSecretKeyRef, &out.KeyFileSecretKeyRef + *out = new(user.SecretKeyRef) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(ExternalMongodTLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalMongoDBSource. +func (in *ExternalMongoDBSource) DeepCopy() *ExternalMongoDBSource { + if in == nil { + return nil + } + out := new(ExternalMongoDBSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalMongodTLS) DeepCopyInto(out *ExternalMongodTLS) { + *out = *in + if in.CA != nil { + in, out := &in.CA, &out.CA + *out = new(v1.LocalObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalMongodTLS. +func (in *ExternalMongodTLS) DeepCopy() *ExternalMongodTLS { + if in == nil { + return nil + } + out := new(ExternalMongodTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MongoDBSearch) DeepCopyInto(out *MongoDBSearch) { *out = *in @@ -109,6 +159,7 @@ func (in *MongoDBSearchSpec) DeepCopyInto(out *MongoDBSearchSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + out.Security = in.Security } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBSearchSpec. @@ -150,6 +201,21 @@ func (in *MongoDBSource) DeepCopyInto(out *MongoDBSource) { *out = new(user.MongoDBResourceRef) **out = **in } + if in.ExternalMongoDBSource != nil { + in, out := &in.ExternalMongoDBSource, &out.ExternalMongoDBSource + *out = new(ExternalMongoDBSource) + (*in).DeepCopyInto(*out) + } + if in.PasswordSecretRef != nil { + in, out := &in.PasswordSecretRef, &out.PasswordSecretRef + *out = new(user.SecretKeyRef) + **out = **in + } + if in.Username != nil { + in, out := &in.Username, &out.Username + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBSource. @@ -161,3 +227,35 @@ func (in *MongoDBSource) DeepCopy() *MongoDBSource { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Security) DeepCopyInto(out *Security) { + *out = *in + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Security. +func (in *Security) DeepCopy() *Security { + if in == nil { + return nil + } + out := new(Security) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLS) DeepCopyInto(out *TLS) { + *out = *in + out.CertificateKeySecret = in.CertificateKeySecret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLS. +func (in *TLS) DeepCopy() *TLS { + if in == nil { + return nil + } + out := new(TLS) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/mongodb.com_clustermongodbroles.yaml b/config/crd/bases/mongodb.com_clustermongodbroles.yaml index 3d583bcfd..9241b7dad 100644 --- a/config/crd/bases/mongodb.com_clustermongodbroles.yaml +++ b/config/crd/bases/mongodb.com_clustermongodbroles.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: clustermongodbroles.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_mongodb.yaml b/config/crd/bases/mongodb.com_mongodb.yaml index d421d8837..2a7076877 100644 --- a/config/crd/bases/mongodb.com_mongodb.yaml +++ b/config/crd/bases/mongodb.com_mongodb.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodb.mongodb.com spec: group: mongodb.com @@ -1410,29 +1410,7 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' automationUserName: type: string clientCertificateSecretRef: @@ -1467,29 +1445,9 @@ spec: bindQueryUser: type: string caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml index 3f1fa05c9..01fe3f2e6 100644 --- a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml +++ b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodbmulticluster.mongodb.com spec: group: mongodb.com @@ -670,29 +670,7 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' automationUserName: type: string clientCertificateSecretRef: @@ -727,29 +705,9 @@ spec: bindQueryUser: type: string caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbsearch.yaml b/config/crd/bases/mongodb.com_mongodbsearch.yaml index 72ea0e50e..7c53c195c 100644 --- a/config/crd/bases/mongodb.com_mongodbsearch.yaml +++ b/config/crd/bases/mongodb.com_mongodbsearch.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodbsearch.mongodb.com spec: group: mongodb.com @@ -49,6 +49,8 @@ spec: spec: properties: persistence: + description: Configure MongoDB Search's persistent volume. If not + defined, the operator will request 10GB of storage. properties: multiple: properties: @@ -95,66 +97,60 @@ spec: type: object type: object resourceRequirements: - description: ResourceRequirements describes the compute resource requirements. + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ResourceRequirements' + description: Configure resource requests and limits for the MongoDB + Search pods. + security: + description: Configure security settings of the MongoDB Search server + that MongoDB database is connecting to when performing search queries. properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + tls: + properties: + certificateKeySecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + enabled: + type: boolean + required: + - enabled type: object type: object source: + description: MongoDB database connection details from which MongoDB + Search will synchronize data to build indexes. properties: + external: + properties: + hostAndPorts: + items: + type: string + type: array + keyFileSecretRef: + description: mongod keyfile used to connect to the external + MongoDB deployment + properties: + key: + type: string + name: + type: string + required: + - name + type: object + tls: + properties: + ca: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' + enabled: + type: boolean + required: + - enabled + type: object + type: object mongodbResourceRef: properties: name: @@ -164,11 +160,26 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. + StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + which aren't exposed as fields in the MongoDBSearch.spec. properties: metadata: description: StatefulSetMetadataWrapper is a wrapper around Labels @@ -190,6 +201,9 @@ spec: - spec type: object version: + description: Optional version of MongoDB Search component (mongot). + If not set, then the operator will set the most appropriate version + of MongoDB Search. type: string type: object status: diff --git a/config/crd/bases/mongodb.com_mongodbusers.yaml b/config/crd/bases/mongodb.com_mongodbusers.yaml index 89713ce7f..a81f0d449 100644 --- a/config/crd/bases/mongodb.com_mongodbusers.yaml +++ b/config/crd/bases/mongodb.com_mongodbusers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: mongodbusers.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_opsmanagers.yaml b/config/crd/bases/mongodb.com_opsmanagers.yaml index 3ace001da..c830b9a24 100644 --- a/config/crd/bases/mongodb.com_opsmanagers.yaml +++ b/config/crd/bases/mongodb.com_opsmanagers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: opsmanagers.mongodb.com spec: group: mongodb.com @@ -730,30 +730,7 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - description: SecretKeySelector selects a key of a - Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' automationUserName: type: string clientCertificateSecretRef: @@ -789,29 +766,9 @@ spec: bindQueryUser: type: string caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic servers: items: type: string @@ -1236,29 +1193,7 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' type: array irsaEnabled: description: |- @@ -1328,29 +1263,7 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - description: SecretKeySelector selects a key of a Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' type: array irsaEnabled: description: |- @@ -1495,8 +1408,6 @@ spec: required: - spec type: object - required: - - members type: object clusterDomain: description: Cluster domain to override the default *.svc.cluster.local @@ -1536,13 +1447,13 @@ spec: Service when creating a ClusterIP type Service type: string externalTrafficPolicy: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local - type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. @@ -1553,12 +1464,12 @@ spec: format: int32 type: integer type: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP - type: string required: - type type: object @@ -1600,7 +1511,6 @@ spec: - spec type: object required: - - clusterName - members type: object type: array @@ -1626,13 +1536,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local - type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1642,12 +1552,12 @@ spec: format: int32 type: integer type: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP - type: string required: - type type: object @@ -1667,13 +1577,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local - type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1683,12 +1593,12 @@ spec: format: int32 type: integer type: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP - type: string required: - type type: object diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 36d5c892d..a0004e22e 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.15.0 service.binding: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret service.binding/connectionString: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv service.binding/password: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password @@ -330,24 +330,13 @@ spec: authentication: properties: agentCertificateSecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- AgentCertificateSecret is a reference to a Secret containing the certificate and the key for the automation agent The secret needs to have available: - certificate under key: "tls.crt" - private key under key: "tls.key" If additionally, tls.pem is present, then it needs to be equal to the concatenation of tls.crt and tls.key - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic agentMode: description: AgentMode contains the authentication mode used by the automation agent. @@ -466,57 +455,24 @@ spec: communication properties: caCertificateSecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic caConfigMapRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" This field is ignored when CaCertificateSecretRef is configured - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic certificateKeySecretRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic enabled: type: boolean optional: diff --git a/controllers/operator/mongodbreplicaset_controller.go b/controllers/operator/mongodbreplicaset_controller.go index bf6b2d2b7..fad1ec6b4 100644 --- a/controllers/operator/mongodbreplicaset_controller.go +++ b/controllers/operator/mongodbreplicaset_controller.go @@ -4,12 +4,16 @@ import ( "context" "fmt" + "github.com/blang/semver" "go.uber.org/zap" "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -22,6 +26,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" rolev1 "github.com/mongodb/mongodb-kubernetes/api/v1/role" + searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" mdbstatus "github.com/mongodb/mongodb-kubernetes/api/v1/status" "github.com/mongodb/mongodb-kubernetes/controllers/om" "github.com/mongodb/mongodb-kubernetes/controllers/om/backup" @@ -39,6 +44,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/controllers/operator/recovery" "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" + "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" mcoConstruct "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/annotations" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/configmap" @@ -52,6 +58,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/pkg/util/architectures" "github.com/mongodb/mongodb-kubernetes/pkg/util/env" util_int "github.com/mongodb/mongodb-kubernetes/pkg/util/int" + "github.com/mongodb/mongodb-kubernetes/pkg/util/maputil" "github.com/mongodb/mongodb-kubernetes/pkg/vault" "github.com/mongodb/mongodb-kubernetes/pkg/vault/vaultwatcher" ) @@ -222,6 +229,8 @@ func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reco return r.updateStatus(ctx, rs, workflow.Failed(xerrors.Errorf("Failed to reconcileHostnameOverrideConfigMap: %w", err)), log) } + shouldMirrorKeyfile := r.applySearchOverrides(ctx, rs, log) + sts := construct.DatabaseStatefulSet(*rs, rsConfig, log) if status := r.ensureRoles(ctx, rs.Spec.DbCommonSpec, r.enableClusterMongoDBRoles, conn, kube.ObjectKeyFromApiObject(rs), log); !status.IsOK() { return r.updateStatus(ctx, rs, status, log) @@ -251,7 +260,7 @@ func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reco // See CLOUDP-189433 and CLOUDP-229222 for more details. if recovery.ShouldTriggerRecovery(rs.Status.Phase != mdbstatus.PhaseRunning, rs.Status.LastTransition) { log.Warnf("Triggering Automatic Recovery. The MongoDB resource %s/%s is in %s state since %s", rs.Namespace, rs.Name, rs.Status.Phase, rs.Status.LastTransition) - automationConfigStatus := r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, true).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + automationConfigStatus := r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, true, shouldMirrorKeyfile).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") deploymentError := create.DatabaseInKubernetes(ctx, r.client, *rs, sts, rsConfig, log) if deploymentError != nil { log.Errorf("Recovery failed because of deployment errors, %w", deploymentError) @@ -267,7 +276,7 @@ func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reco } status = workflow.RunInGivenOrder(publishAutomationConfigFirst(ctx, r.client, *rs, lastSpec, rsConfig, log), func() workflow.Status { - return r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, false).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + return r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, tlsCertPath, internalClusterCertPath, agentCertSecretSelector, prometheusCertHash, false, shouldMirrorKeyfile).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") }, func() workflow.Status { workflowStatus := create.HandlePVCResize(ctx, r.client, &sts, log) @@ -421,6 +430,19 @@ func AddReplicaSetController(ctx context.Context, mgr manager.Manager, imageUrls zap.S().Errorf("Failed to watch for vault secret changes: %w", err) } } + + err = c.Watch(source.Kind(mgr.GetCache(), &searchv1.MongoDBSearch{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, search *searchv1.MongoDBSearch) []reconcile.Request { + source := search.GetMongoDBResourceRef() + if source == nil { + return []reconcile.Request{} + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: source.Namespace, Name: source.Name}}} + }))) + if err != nil { + return err + } + zap.S().Infof("Registered controller %s", util.MongoDbReplicaSetController) return nil @@ -428,7 +450,7 @@ func AddReplicaSetController(ctx context.Context, mgr manager.Manager, imageUrls // updateOmDeploymentRs performs OM registration operation for the replicaset. So the changes will be finally propagated // to automation agents in containers -func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, conn om.Connection, membersNumberBefore int, rs *mdbv1.MongoDB, set appsv1.StatefulSet, log *zap.SugaredLogger, caFilePath, tlsCertPath, internalClusterCertPath string, agentCertSecretSelector corev1.SecretKeySelector, prometheusCertHash string, isRecovering bool) workflow.Status { +func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, conn om.Connection, membersNumberBefore int, rs *mdbv1.MongoDB, set appsv1.StatefulSet, log *zap.SugaredLogger, caFilePath, tlsCertPath, internalClusterCertPath string, agentCertSecretSelector corev1.SecretKeySelector, prometheusCertHash string, isRecovering bool, shouldMirrorKeyfileForMongot bool) workflow.Status { log.Debug("Entering UpdateOMDeployments") // Only "concrete" RS members should be observed // - if scaling down, let's observe only members that will remain after scale-down operation @@ -477,6 +499,11 @@ func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, c err = conn.ReadUpdateDeployment( func(d om.Deployment) error { + if shouldMirrorKeyfileForMongot { + if err := r.mirrorKeyfileIntoSecretForMongot(ctx, d, rs, log); err != nil { + return err + } + } return ReconcileReplicaSetAC(ctx, d, rs.Spec.DbCommonSpec, lastRsConfig.ToMap(), rs.Name, replicaSet, caFilePath, internalClusterCertPath, &p, log) }, log, @@ -617,3 +644,70 @@ func getAllHostsRs(set appsv1.StatefulSet, clusterName string, membersCount int, hostnames, _ := dns.GetDnsForStatefulSetReplicasSpecified(set, clusterName, membersCount, externalDomain) return hostnames } + +func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, rs *mdbv1.MongoDB, log *zap.SugaredLogger) bool { + search := r.lookupCorrespondingSearchResource(ctx, rs, log) + if search == nil { + log.Debugf("No MongoDBSearch resource found, skipping search overrides") + return false + } + + log.Infof("Applying search overrides from MongoDBSearch %s", search.NamespacedName()) + + if rs.Spec.AdditionalMongodConfig == nil { + rs.Spec.AdditionalMongodConfig = mdbv1.NewEmptyAdditionalMongodConfig() + } + searchMongodConfig := search_controller.GetMongodConfigParameters(search) + rs.Spec.AdditionalMongodConfig.AddOption("setParameter", searchMongodConfig["setParameter"]) + + mdbVersion, err := semver.ParseTolerant(rs.Spec.Version) + if err != nil { + log.Warnf("Failed to parse MongoDB version %q: %w. Proceeding without the automatic creation of the searchCoordinator role that's necessary for MongoDB <8.2", rs.Spec.Version, err) + } else if semver.MustParse("8.2.0").GT(mdbVersion) { + log.Infof("Polyfilling the searchCoordinator role for MongoDB %s", rs.Spec.Version) + + if rs.Spec.Security == nil { + rs.Spec.Security = &mdbv1.Security{} + } + rs.Spec.Security.Roles = append(rs.Spec.Security.Roles, search_controller.SearchCoordinatorRole()) + } + + return true +} + +func (r *ReconcileMongoDbReplicaSet) mirrorKeyfileIntoSecretForMongot(ctx context.Context, d om.Deployment, rs *mdbv1.MongoDB, log *zap.SugaredLogger) error { + keyfileContents := maputil.ReadMapValueAsString(d, "auth", "key") + keyfileSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-keyfile", rs.Name), Namespace: rs.Namespace}} + + log.Infof("Mirroring the replicaset %s's keyfile into the secret %s", rs.ObjectKey(), kube.ObjectKeyFromApiObject(keyfileSecret)) + + _, err := controllerutil.CreateOrUpdate(ctx, r.client, keyfileSecret, func() error { + keyfileSecret.StringData = map[string]string{"keyfile": keyfileContents} + return controllerutil.SetOwnerReference(rs, keyfileSecret, r.client.Scheme()) + }) + if err != nil { + return xerrors.Errorf("Failed to mirror the replicaset's keyfile into a secret: %w", err) + } else { + return nil + } +} + +func (r *ReconcileMongoDbReplicaSet) lookupCorrespondingSearchResource(ctx context.Context, rs *mdbv1.MongoDB, log *zap.SugaredLogger) *searchv1.MongoDBSearch { + var search *searchv1.MongoDBSearch + searchList := &searchv1.MongoDBSearchList{} + if err := r.client.List(ctx, searchList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(search_controller.MongoDBSearchIndexFieldName, rs.GetNamespace()+"/"+rs.GetName()), + }); err != nil { + log.Debugf("Failed to list MongoDBSearch resources: %v", err) + } + // this validates that there is exactly one MongoDBSearch pointing to this resource, + // and that this resource passes search validations. If either fails, proceed without a search target + // for the mongod automation config. + if len(searchList.Items) == 1 { + searchSource := search_controller.NewEnterpriseResourceSearchSource(rs) + if searchSource.Validate() == nil { + search = &searchList.Items[0] + } + } + return search +} diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index 63ed00a5f..a25efed3c 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -13,12 +13,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" - "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/watch" kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" "github.com/mongodb/mongodb-kubernetes/pkg/kube/commoncontroller" "github.com/mongodb/mongodb-kubernetes/pkg/util" @@ -27,15 +30,14 @@ import ( type MongoDBSearchReconciler struct { kubeClient kubernetesClient.Client - mdbcWatcher *watch.ResourceWatcher + watch *watch.ResourceWatcher operatorSearchConfig search_controller.OperatorSearchConfig } func newMongoDBSearchReconciler(client client.Client, operatorSearchConfig search_controller.OperatorSearchConfig) *MongoDBSearchReconciler { - mdbcWatcher := watch.New() return &MongoDBSearchReconciler{ kubeClient: kubernetesClient.NewClient(client), - mdbcWatcher: &mdbcWatcher, + watch: watch.NewResourceWatcher(), operatorSearchConfig: operatorSearchConfig, } } @@ -50,31 +52,77 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci return result, err } - sourceResource, err := getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch) + searchSource, err := r.getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch, log) if err != nil { return reconcile.Result{RequeueAfter: time.Second * util.RetryTimeSec}, err } - r.mdbcWatcher.Watch(ctx, sourceResource.NamespacedName(), request.NamespacedName) + r.watch.AddWatchedResourceIfNotAdded(searchSource.KeyfileSecretName(), mdbSearch.Namespace, watch.Secret, mdbSearch.NamespacedName()) - reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, sourceResource, r.operatorSearchConfig) + // Watch for changes in database source CA certificate secrets or configmaps + tlsSourceConfig := searchSource.TLSConfig() + if tlsSourceConfig != nil { + for wType, resources := range tlsSourceConfig.ResourcesToWatch { + for _, resource := range resources { + r.watch.AddWatchedResourceIfNotAdded(resource.Name, resource.Namespace, wType, mdbSearch.NamespacedName()) + } + } + } + + // Watch our own TLS certificate secret for changes + if mdbSearch.Spec.Security.TLS.Enabled { + r.watch.AddWatchedResourceIfNotAdded(mdbSearch.Spec.Security.TLS.CertificateKeySecret.Name, mdbSearch.Namespace, watch.Secret, mdbSearch.NamespacedName()) + } + + reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, searchSource, r.operatorSearchConfig) return reconcileHelper.Reconcile(ctx, log).ReconcileResult() } -func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, error) { +func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch, log *zap.SugaredLogger) (search_controller.SearchSourceDBResource, error) { + if search.IsExternalMongoDBSource() { + return search_controller.NewExternalSearchSource(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil + } + sourceMongoDBResourceRef := search.GetMongoDBResourceRef() - mdbcName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name} + if sourceMongoDBResourceRef == nil { + return nil, xerrors.New("MongoDBSearch source MongoDB resource reference is not set") + } + + sourceName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name} + log.Infof("Looking up Search source %s", sourceName) + + mdb := &mdbv1.MongoDB{} + if err := kubeClient.Get(ctx, sourceName, mdb); err != nil { + if !apierrors.IsNotFound(err) { + return nil, xerrors.Errorf("error getting MongoDB %s: %w", sourceName, err) + } + } else { + r.watch.AddWatchedResourceIfNotAdded(sourceMongoDBResourceRef.Name, sourceMongoDBResourceRef.Namespace, watch.MongoDB, search.NamespacedName()) + return search_controller.NewEnterpriseResourceSearchSource(mdb), nil + } + mdbc := &mdbcv1.MongoDBCommunity{} - if err := kubeClient.Get(ctx, mdbcName, mdbc); err != nil { - return nil, xerrors.Errorf("error getting MongoDBCommunity %s", mdbcName) + if err := kubeClient.Get(ctx, sourceName, mdbc); err != nil { + if !apierrors.IsNotFound(err) { + return nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", sourceName, err) + } + } else { + r.watch.AddWatchedResourceIfNotAdded(sourceMongoDBResourceRef.Name, sourceMongoDBResourceRef.Namespace, "MongoDBCommunity", search.NamespacedName()) + return search_controller.NewCommunityResourceSearchSource(mdbc), nil } - return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), nil + + return nil, xerrors.Errorf("No database resource named %s found", sourceName) } func mdbcSearchIndexBuilder(rawObj client.Object) []string { mdbSearch := rawObj.(*searchv1.MongoDBSearch) - return []string{mdbSearch.GetMongoDBResourceRef().Namespace + "/" + mdbSearch.GetMongoDBResourceRef().Name} + resourceRef := mdbSearch.GetMongoDBResourceRef() + if resourceRef == nil { + return []string{} + } + + return []string{resourceRef.Namespace + "/" + resourceRef.Name} } func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operatorSearchConfig search_controller.OperatorSearchConfig) error { @@ -87,7 +135,11 @@ func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operat return ctrl.NewControllerManagedBy(mgr). WithOptions(controller.Options{MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}). // nolint:forbidigo For(&searchv1.MongoDBSearch{}). - Watches(&mdbcv1.MongoDBCommunity{}, r.mdbcWatcher). + Watches(&mdbv1.MongoDB{}, &watch.ResourcesHandler{ResourceType: watch.MongoDB, ResourceWatcher: r.watch}). + Watches(&mdbcv1.MongoDBCommunity{}, &watch.ResourcesHandler{ResourceType: "MongoDBCommunity", ResourceWatcher: r.watch}). + Watches(&corev1.Secret{}, &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: r.watch}). + Watches(&corev1.ConfigMap{}, &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: r.watch}). Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Secret{}). Complete(r) } diff --git a/controllers/operator/mongodbsearch_controller_test.go b/controllers/operator/mongodbsearch_controller_test.go index 0c7ddcce5..28f966ff8 100644 --- a/controllers/operator/mongodbsearch_controller_test.go +++ b/controllers/operator/mongodbsearch_controller_test.go @@ -8,9 +8,8 @@ import ( "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1 "k8s.io/api/apps/v1" @@ -25,7 +24,9 @@ import ( "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1/common" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/mongot" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/util/constants" ) func newMongoDBCommunity(name, namespace string) *mdbcv1.MongoDBCommunity { @@ -34,7 +35,7 @@ func newMongoDBCommunity(name, namespace string) *mdbcv1.MongoDBCommunity { Spec: mdbcv1.MongoDBCommunitySpec{ Type: mdbcv1.ReplicaSet, Members: 1, - Version: "8.0", + Version: "8.0.10", }, } } @@ -50,15 +51,25 @@ func newMongoDBSearch(name, namespace, mdbcName string) *searchv1.MongoDBSearch } } -func newSearchReconciler( +func newSearchReconcilerWithOperatorConfig( mdbc *mdbcv1.MongoDBCommunity, + operatorConfig search_controller.OperatorSearchConfig, searches ...*searchv1.MongoDBSearch, ) (*MongoDBSearchReconciler, client.Client) { builder := mock.NewEmptyFakeClientBuilder() builder.WithIndex(&searchv1.MongoDBSearch{}, search_controller.MongoDBSearchIndexFieldName, mdbcSearchIndexBuilder) if mdbc != nil { - builder.WithObjects(mdbc) + keyfileSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: mdbc.GetAgentKeyfileSecretNamespacedName().Name, + Namespace: mdbc.Namespace, + }, + StringData: map[string]string{ + constants.AgentKeyfileKey: "keyfile", + }, + } + builder.WithObjects(mdbc, keyfileSecret) } for _, search := range searches { @@ -68,25 +79,58 @@ func newSearchReconciler( } fakeClient := builder.Build() - return newMongoDBSearchReconciler(fakeClient, search_controller.OperatorSearchConfig{}), fakeClient + + return newMongoDBSearchReconciler(fakeClient, operatorConfig), fakeClient +} + +func newSearchReconciler( + mdbc *mdbcv1.MongoDBCommunity, + searches ...*searchv1.MongoDBSearch, +) (*MongoDBSearchReconciler, client.Client) { + return newSearchReconcilerWithOperatorConfig(mdbc, search_controller.OperatorSearchConfig{}, searches...) } func buildExpectedMongotConfig(search *searchv1.MongoDBSearch, mdbc *mdbcv1.MongoDBCommunity) mongot.Config { - return mongot.Config{CommunityPrivatePreview: mongot.CommunityPrivatePreview{ - MongodHostAndPort: fmt.Sprintf( - "%s.%s.svc.cluster.local:%d", - mdbc.ServiceName(), mdbc.Namespace, - mdbc.GetMongodConfiguration().GetDBPort(), - ), - QueryServerAddress: fmt.Sprintf("localhost:%d", search.GetMongotPort()), - KeyFilePath: "/mongot/keyfile/keyfile", - DataPath: "/mongot/data/config.yml", - Metrics: mongot.Metrics{ + var hostAndPorts []string + for i := range mdbc.Spec.Members { + hostAndPorts = append(hostAndPorts, fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", mdbc.Name, i, mdbc.Name+"-svc", search.Namespace, 27017)) + } + return mongot.Config{ + SyncSource: mongot.ConfigSyncSource{ + ReplicaSet: mongot.ConfigReplicaSet{ + HostAndPort: hostAndPorts, + Username: searchv1.MongotDefaultSyncSourceUsername, + PasswordFile: "/tmp/sourceUserPassword", + TLS: ptr.To(false), + ReadPreference: ptr.To("secondaryPreferred"), + AuthSource: ptr.To("admin"), + }, + }, + Storage: mongot.ConfigStorage{ + DataPath: "/mongot/data/config.yml", + }, + Server: mongot.ConfigServer{ + Wireproto: &mongot.ConfigWireproto{ + Address: "0.0.0.0:27027", + Authentication: &mongot.ConfigAuthentication{ + Mode: "keyfile", + KeyFile: "/tmp/keyfile", + }, + TLS: mongot.ConfigTLS{Mode: mongot.ConfigTLSModeDisabled}, + }, + }, + Metrics: mongot.ConfigMetrics{ Enabled: true, - Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()), + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotMetricsPort()), + }, + HealthCheck: mongot.ConfigHealthCheck{ + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotHealthCheckPort()), + }, + Logging: mongot.ConfigLogging{ + Verbosity: "TRACE", + LogPath: nil, }, - Logging: mongot.Logging{Verbosity: "DEBUG"}, - }} + } } func TestMongoDBSearchReconcile_NotFound(t *testing.T) { @@ -146,10 +190,6 @@ func TestMongoDBSearchReconcile_Success(t *testing.T) { sts := &appsv1.StatefulSet{} err = c.Get(ctx, search.StatefulSetNamespacedName(), sts) assert.NoError(t, err) - - queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request]()) - reconciler.mdbcWatcher.Create(ctx, event.CreateEvent{Object: mdbc}, queue) - assert.Equal(t, 1, queue.Len()) } func checkSearchReconcileFailed( @@ -183,16 +223,6 @@ func TestMongoDBSearchReconcile_InvalidVersion(t *testing.T) { checkSearchReconcileFailed(ctx, t, reconciler, c, search, "MongoDB version") } -func TestMongoDBSearchReconcile_TLSNotSupported(t *testing.T) { - ctx := context.Background() - search := newMongoDBSearch("search", mock.TestNamespace, "mdb") - mdbc := newMongoDBCommunity("mdb", mock.TestNamespace) - mdbc.Spec.Security.TLS.Enabled = true - reconciler, c := newSearchReconciler(mdbc, search) - - checkSearchReconcileFailed(ctx, t, reconciler, c, search, "TLS-enabled") -} - func TestMongoDBSearchReconcile_MultipleSearchResources(t *testing.T) { ctx := context.Background() search1 := newMongoDBSearch("search1", mock.TestNamespace, "mdb") @@ -202,3 +232,60 @@ func TestMongoDBSearchReconcile_MultipleSearchResources(t *testing.T) { checkSearchReconcileFailed(ctx, t, reconciler, c, search1, "multiple MongoDBSearch") } + +func TestMongoDBSearchReconcile_InvalidSearchImageVersion(t *testing.T) { + ctx := context.Background() + expectedMsg := "MongoDBSearch version 1.47.0 is not supported because of breaking changes. The operator will ignore this resource: it will not reconcile or reconfigure the workload. Existing deployments will continue to run, but cannot be managed by the operator. To regain operator management, you must delete and recreate the MongoDBSearch resource." + + tests := []struct { + name string + specVersion string + operatorVersion string + statefulSetConfig *common.StatefulSetConfiguration + }{ + { + name: "unsupported version in Spec.Version", + specVersion: "1.47.0", + }, + { + name: "unsupported version in operator config", + operatorVersion: "1.47.0", + }, + { + name: "unsupported version in StatefulSetConfiguration", + statefulSetConfig: &common.StatefulSetConfiguration{ + SpecWrapper: common.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: search_controller.MongotContainerName, + Image: "testrepo/mongot:1.47.0", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + search := newMongoDBSearch("search", mock.TestNamespace, "mdb") + mdbc := newMongoDBCommunity("mdb", mock.TestNamespace) + + search.Spec.Version = tc.specVersion + search.Spec.StatefulSetConfiguration = tc.statefulSetConfig + + operatorConfig := search_controller.OperatorSearchConfig{ + SearchVersion: tc.operatorVersion, + } + reconciler, _ := newSearchReconcilerWithOperatorConfig(mdbc, operatorConfig, search) + + checkSearchReconcileFailed(ctx, t, reconciler, reconciler.kubeClient, search, expectedMsg) + }) + } +} diff --git a/controllers/operator/watch/config_change_handler.go b/controllers/operator/watch/config_change_handler.go index 7b53ee4fe..77f725f20 100644 --- a/controllers/operator/watch/config_change_handler.go +++ b/controllers/operator/watch/config_change_handler.go @@ -14,8 +14,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" corev1 "k8s.io/api/core/v1" - - rolev1 "github.com/mongodb/mongodb-kubernetes/api/v1/role" ) // Type is an enum for all kubernetes types watched by controller for changes for configuration @@ -87,10 +85,14 @@ func (c *ResourcesHandler) doHandle(namespace, name string, q workqueue.TypedRat // Seems we don't need to react on config map/secret removal.. func (c *ResourcesHandler) Delete(ctx context.Context, e event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - switch v := e.Object.(type) { - case *rolev1.ClusterMongoDBRole: - c.doHandle(v.GetNamespace(), v.GetName(), q) + switch e.Object.(type) { + case *corev1.ConfigMap: + return + case *corev1.Secret: + return } + + c.doHandle(e.Object.GetNamespace(), e.Object.GetName(), q) } func (c *ResourcesHandler) Generic(context.Context, event.TypedGenericEvent[client.Object], workqueue.TypedRateLimitingInterface[reconcile.Request]) { diff --git a/controllers/search_controller/community_search_source.go b/controllers/search_controller/community_search_source.go new file mode 100644 index 000000000..8b10a1cbf --- /dev/null +++ b/controllers/search_controller/community_search_source.go @@ -0,0 +1,84 @@ +package search_controller + +import ( + "fmt" + "strings" + + "github.com/blang/semver" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + corev1 "k8s.io/api/core/v1" + + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" + mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" + "github.com/mongodb/mongodb-kubernetes/pkg/util" +) + +func NewCommunityResourceSearchSource(mdbc *mdbcv1.MongoDBCommunity) SearchSourceDBResource { + return &CommunitySearchSource{MongoDBCommunity: mdbc} +} + +type CommunitySearchSource struct { + *mdbcv1.MongoDBCommunity +} + +func (r *CommunitySearchSource) HostSeeds() []string { + seeds := make([]string, r.Spec.Members) + for i := range seeds { + seeds[i] = fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", r.Name, i, r.ServiceName(), r.Namespace, r.GetMongodConfiguration().GetDBPort()) + } + return seeds +} + +func (r *CommunitySearchSource) KeyfileSecretName() string { + return r.MongoDBCommunity.GetAgentKeyfileSecretNamespacedName().Name +} + +func (r *CommunitySearchSource) TLSConfig() *TLSSourceConfig { + if !r.Spec.Security.TLS.Enabled { + return nil + } + + var volume corev1.Volume + watchedResources := make(map[watch.Type][]types.NamespacedName) + + if r.Spec.Security.TLS.CaCertificateSecret != nil { + volume = statefulset.CreateVolumeFromSecret("ca", r.Spec.Security.TLS.CaCertificateSecret.Name) + watchedResources[watch.Secret] = []types.NamespacedName{r.TLSCaCertificateSecretNamespacedName()} + } else { + volume = statefulset.CreateVolumeFromConfigMap("ca", r.Spec.Security.TLS.CaConfigMap.Name) + watchedResources[watch.ConfigMap] = []types.NamespacedName{r.TLSConfigMapNamespacedName()} + } + + return &TLSSourceConfig{ + CAFileName: "ca.crt", + CAVolume: volume, + ResourcesToWatch: watchedResources, + } +} + +func (r *CommunitySearchSource) Validate() error { + version, err := semver.ParseTolerant(r.GetMongoDBVersion()) + if err != nil { + return xerrors.Errorf("error parsing MongoDB version '%s': %w", r.Spec.Version, err) + } else if version.LT(semver.MustParse("8.0.10")) { + return xerrors.New("MongoDB version must be 8.0.10 or higher") + } + + foundScram := false + for _, authMode := range r.Spec.Security.Authentication.Modes { + // Check for SCRAM, SCRAM-SHA-1, or SCRAM-SHA-256 + if strings.HasPrefix(strings.ToUpper(string(authMode)), util.SCRAM) { + foundScram = true + break + } + } + + if !foundScram && len(r.Spec.Security.Authentication.Modes) > 0 { + return xerrors.New("MongoDBSearch requires SCRAM authentication to be enabled") + } + + return nil +} diff --git a/controllers/search_controller/community_search_source_test.go b/controllers/search_controller/community_search_source_test.go new file mode 100644 index 000000000..90bf9967e --- /dev/null +++ b/controllers/search_controller/community_search_source_test.go @@ -0,0 +1,208 @@ +package search_controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" +) + +func newCommunitySearchSource(version string, authModes []mdbcv1.AuthMode) *CommunitySearchSource { + return &CommunitySearchSource{ + MongoDBCommunity: &mdbcv1.MongoDBCommunity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mongodb", + Namespace: "test-namespace", + }, + Spec: mdbcv1.MongoDBCommunitySpec{ + Version: version, + Security: mdbcv1.Security{ + Authentication: mdbcv1.Authentication{ + Modes: authModes, + }, + }, + }, + }, + } +} + +func TestCommunitySearchSource_Validate(t *testing.T) { + cases := []struct { + name string + version string + authModes []mdbcv1.AuthMode + expectError bool + expectedErrMsg string + }{ + // Version validation tests + { + name: "Invalid version", + version: "invalid.version", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "error parsing MongoDB version", + }, + { + name: "Version too old", + version: "7.0.0", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Version just below minimum", + version: "8.0.9", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid minimum version", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Version above minimum", + version: "8.1.0", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Version with build number", + version: "8.1.0-rc1", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + // Authentication mode tests - empty/nil cases + { + name: "Empty auth modes", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{}, + expectError: false, + }, + { + name: "Nil auth modes", + version: "8.0.10", + authModes: nil, + expectError: false, + }, + { + name: "X509 mode only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + { + name: "X509 and SCRAM", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"X509", "SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Multiple auth modes with SCRAM first", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-1", "X509"}, + expectError: false, + }, + { + name: "Multiple auth modes with SCRAM last", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"PLAIN", "X509", "SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Multiple non-SCRAM modes", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"PLAIN", "X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + // SCRAM variant tests + { + name: "SCRAM only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM"}, + expectError: false, + }, + { + name: "SCRAM-SHA-1 only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-1"}, + expectError: false, + }, + { + name: "SCRAM-SHA-256 only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "All SCRAM variants", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"SCRAM", "SCRAM-SHA-1", "SCRAM-SHA-256"}, + expectError: false, + }, + // Case-insensitive tests (now supported with ToUpper) + { + name: "Lowercase SCRAM", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"scram-sha-256"}, + expectError: false, + }, + { + name: "Mixed case SCRAM", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"Scram-Sha-256"}, + expectError: false, + }, + // Edge case tests + { + name: "PLAIN only", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"PLAIN"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + // Combined validation tests + { + name: "Invalid version with valid auth", + version: "7.0.0", + authModes: []mdbcv1.AuthMode{"SCRAM-SHA-256"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid version with invalid auth", + version: "8.0.10", + authModes: []mdbcv1.AuthMode{"X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + { + name: "Invalid version with invalid auth", + version: "7.0.0", + authModes: []mdbcv1.AuthMode{"X509"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", // Should fail on version first + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + src := newCommunitySearchSource(c.version, c.authModes) + err := src.Validate() + + if c.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), c.expectedErrMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/controllers/search_controller/enterprise_search_source.go b/controllers/search_controller/enterprise_search_source.go new file mode 100644 index 000000000..c1256fbbb --- /dev/null +++ b/controllers/search_controller/enterprise_search_source.go @@ -0,0 +1,88 @@ +package search_controller + +import ( + "fmt" + "strings" + + "github.com/blang/semver" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" + "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" + "github.com/mongodb/mongodb-kubernetes/pkg/util" +) + +type EnterpriseResourceSearchSource struct { + *mdbv1.MongoDB +} + +func NewEnterpriseResourceSearchSource(mdb *mdbv1.MongoDB) SearchSourceDBResource { + return EnterpriseResourceSearchSource{mdb} +} + +func (r EnterpriseResourceSearchSource) HostSeeds() []string { + seeds := make([]string, r.Spec.Members) + for i := range seeds { + seeds[i] = fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", r.Name, i, r.ServiceName(), r.Namespace, r.Spec.GetAdditionalMongodConfig().GetPortOrDefault()) + } + return seeds +} + +func (r EnterpriseResourceSearchSource) TLSConfig() *TLSSourceConfig { + if !r.Spec.Security.IsTLSEnabled() { + return nil + } + + return &TLSSourceConfig{ + CAFileName: "ca-pem", + CAVolume: statefulset.CreateVolumeFromConfigMap("ca", r.Spec.Security.TLSConfig.CA), + ResourcesToWatch: map[watch.Type][]types.NamespacedName{ + watch.ConfigMap: { + {Namespace: r.Namespace, Name: r.Spec.Security.TLSConfig.CA}, + }, + }, + } +} + +func (r EnterpriseResourceSearchSource) KeyfileSecretName() string { + return fmt.Sprintf("%s-keyfile", r.Name) +} + +func (r EnterpriseResourceSearchSource) Validate() error { + version, err := semver.ParseTolerant(r.Spec.GetMongoDBVersion()) + if err != nil { + return xerrors.Errorf("error parsing MongoDB version '%s': %w", r.Spec.GetMongoDBVersion(), err) + } else if version.LT(semver.MustParse("8.0.10")) { + return xerrors.New("MongoDB version must be 8.0.10 or higher") + } + + if r.Spec.GetTopology() != mdbv1.ClusterTopologySingleCluster { + return xerrors.Errorf("MongoDBSearch is only supported for %s topology", mdbv1.ClusterTopologySingleCluster) + } + + if r.GetResourceType() != mdbv1.ReplicaSet { + return xerrors.Errorf("MongoDBSearch is only supported for %s resources", mdbv1.ReplicaSet) + } + + authModes := r.Spec.GetSecurityAuthenticationModes() + foundScram := false + for _, authMode := range authModes { + // Check for SCRAM, SCRAM-SHA-1, or SCRAM-SHA-256 + if strings.HasPrefix(strings.ToUpper(authMode), util.SCRAM) { + foundScram = true + break + } + } + + if !foundScram && len(authModes) > 0 { + return xerrors.New("MongoDBSearch requires SCRAM authentication to be enabled") + } + + if r.Spec.Security.GetInternalClusterAuthenticationMode() == util.X509 { + return xerrors.New("MongoDBSearch does not support X.509 internal cluster authentication") + } + + return nil +} diff --git a/controllers/search_controller/enterprise_search_source_test.go b/controllers/search_controller/enterprise_search_source_test.go new file mode 100644 index 000000000..1362ab605 --- /dev/null +++ b/controllers/search_controller/enterprise_search_source_test.go @@ -0,0 +1,290 @@ +package search_controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" +) + +func newEnterpriseSearchSource(version string, topology string, resourceType mdbv1.ResourceType, authModes []string, internalClusterAuth string) EnterpriseResourceSearchSource { + authModesList := make([]mdbv1.AuthMode, len(authModes)) + for i, mode := range authModes { + authModesList[i] = mdbv1.AuthMode(mode) + } + + // Create security with authentication if needed + var security *mdbv1.Security + if len(authModes) > 0 || internalClusterAuth != "" { + security = &mdbv1.Security{ + Authentication: &mdbv1.Authentication{ + Enabled: len(authModes) > 0, + Modes: authModesList, + InternalCluster: internalClusterAuth, + }, + } + } + + src := EnterpriseResourceSearchSource{ + MongoDB: &mdbv1.MongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mongodb", + Namespace: "test-namespace", + }, + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + Version: version, + ResourceType: resourceType, + Security: security, + }, + }, + }, + } + + // Set topology directly since it's inlined from DbCommonSpec + src.Spec.Topology = topology + return src +} + +func TestEnterpriseResourceSearchSource_Validate(t *testing.T) { + cases := []struct { + name string + version string + topology string + resourceType mdbv1.ResourceType + authModes []string + internalClusterAuth string + expectError bool + expectedErrMsg string + }{ + // Version validation tests + { + name: "Invalid version", + version: "invalid.version", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "error parsing MongoDB version", + }, + { + name: "Version too old", + version: "7.0.0", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Version just below minimum", + version: "8.0.9", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid minimum version", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + { + name: "Version above minimum", + version: "8.1.0", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + // Topology validation tests + { + name: "Invalid topology - MultiCluster", + version: "8.0.10", + topology: mdbv1.ClusterTopologyMultiCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for SingleCluster topology", + }, + { + name: "Valid topology - SingleCluster", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + { + name: "Empty topology defaults to SingleCluster", + version: "8.0.10", + topology: "", + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + // Resource type validation tests + { + name: "Invalid resource type - Standalone", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.Standalone, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for ReplicaSet resources", + }, + { + name: "Invalid resource type - ShardedCluster", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ShardedCluster, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for ReplicaSet resources", + }, + { + name: "Valid resource type - ReplicaSet", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + // Authentication mode tests + { + name: "No SCRAM authentication", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"X509"}, + expectError: true, + expectedErrMsg: "MongoDBSearch requires SCRAM authentication to be enabled", + }, + { + name: "Empty authentication modes", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: false, + }, + { + name: "Nil authentication modes", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: nil, + expectError: false, + }, + { + name: "Valid SCRAM authentication", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Mixed auth modes with SCRAM", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"X509", "SCRAM-SHA-256"}, + expectError: false, + }, + { + name: "Case insensitive SCRAM", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"scram-sha-256"}, + expectError: false, + }, + { + name: "SCRAM variants", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM", "SCRAM-SHA-1", "SCRAM-SHA-256"}, + expectError: false, + }, + // Internal cluster authentication tests + { + name: "X509 internal cluster auth not supported", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + internalClusterAuth: "X509", + expectError: true, + expectedErrMsg: "MongoDBSearch does not support X.509 internal cluster authentication", + }, + { + name: "Valid internal cluster auth - empty", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + internalClusterAuth: "", + expectError: false, + }, + { + name: "Valid internal cluster auth - SCRAM", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{"SCRAM-SHA-256"}, + internalClusterAuth: "SCRAM", + expectError: false, + }, + // Combined validation tests + { + name: "Multiple validation failures - version takes precedence", + version: "7.0.0", + topology: mdbv1.ClusterTopologyMultiCluster, + resourceType: mdbv1.Standalone, + authModes: []string{"X509"}, + expectError: true, + expectedErrMsg: "MongoDB version must be 8.0.10 or higher", + }, + { + name: "Valid version, invalid topology", + version: "8.0.10", + topology: mdbv1.ClusterTopologyMultiCluster, + resourceType: mdbv1.ReplicaSet, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for SingleCluster topology", + }, + { + name: "Valid version and topology, invalid resource type", + version: "8.0.10", + topology: mdbv1.ClusterTopologySingleCluster, + resourceType: mdbv1.Standalone, + authModes: []string{}, + expectError: true, + expectedErrMsg: "MongoDBSearch is only supported for ReplicaSet resources", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + src := newEnterpriseSearchSource(c.version, c.topology, c.resourceType, c.authModes, c.internalClusterAuth) + err := src.Validate() + + if c.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), c.expectedErrMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/controllers/search_controller/external_search_source.go b/controllers/search_controller/external_search_source.go new file mode 100644 index 000000000..5a408246e --- /dev/null +++ b/controllers/search_controller/external_search_source.go @@ -0,0 +1,49 @@ +package search_controller + +import ( + "k8s.io/apimachinery/pkg/types" + + searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" + "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" +) + +func NewExternalSearchSource(namespace string, spec *searchv1.ExternalMongoDBSource) SearchSourceDBResource { + return &externalSearchResource{namespace: namespace, spec: spec} +} + +// externalSearchResource implements SearchSourceDBResource for deployments managed outside the operator. +type externalSearchResource struct { + namespace string + spec *searchv1.ExternalMongoDBSource +} + +func (r *externalSearchResource) Validate() error { + return nil +} + +func (r *externalSearchResource) TLSConfig() *TLSSourceConfig { + if r.spec.TLS == nil || !r.spec.TLS.Enabled { + return nil + } + + return &TLSSourceConfig{ + CAFileName: "ca.crt", + CAVolume: statefulset.CreateVolumeFromSecret("ca", r.spec.TLS.CA.Name), + ResourcesToWatch: map[watch.Type][]types.NamespacedName{ + watch.Secret: { + {Namespace: r.namespace, Name: r.spec.TLS.CA.Name}, + }, + }, + } +} + +func (r *externalSearchResource) KeyfileSecretName() string { + if r.spec.KeyFileSecretKeyRef != nil { + return r.spec.KeyFileSecretKeyRef.Name + } + + return "" +} + +func (r *externalSearchResource) HostSeeds() []string { return r.spec.HostAndPorts } diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/search_controller/mongodbsearch_reconcile_helper.go index 59523ce8e..4b384a707 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper.go @@ -7,30 +7,42 @@ import ( "fmt" "strings" - "github.com/blang/semver" "github.com/ghodss/yaml" "go.uber.org/zap" "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/automationconfig" kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/podtemplatespec" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/service" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/mongot" + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/tls" + "github.com/mongodb/mongodb-kubernetes/pkg/kube" "github.com/mongodb/mongodb-kubernetes/pkg/kube/commoncontroller" "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" ) const ( - MongoDBSearchIndexFieldName = "mdbsearch-for-mongodbresourceref-index" + MongoDBSearchIndexFieldName = "mdbsearch-for-mongodbresourceref-index" + unsupportedSearchVersion = "1.47.0" + unsupportedSearchVersionErrorFmt = "MongoDBSearch version %s is not supported because of breaking changes. " + + "The operator will ignore this resource: it will not reconcile or reconfigure the workload. " + + "Existing deployments will continue to run, but cannot be managed by the operator. " + + "To regain operator management, you must delete and recreate the MongoDBSearch resource." ) type OperatorSearchConfig struct { @@ -72,7 +84,11 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S log = log.With("MongoDBSearch", r.mdbSearch.NamespacedName()) log.Infof("Reconciling MongoDBSearch") - if err := ValidateSearchSource(r.db); err != nil { + if err := r.db.Validate(); err != nil { + return workflow.Failed(err) + } + + if err := r.ValidateSearchImageVersion(); err != nil { return workflow.Failed(err) } @@ -80,39 +96,79 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S return workflow.Failed(err) } + keyfileStsModification, err := r.ensureSourceKeyfile(ctx, log) + if apierrors.IsNotFound(err) { + return workflow.Pending("Waiting for keyfile secret to be created") + } else if err != nil { + return workflow.Failed(err) + } + if err := r.ensureSearchService(ctx, r.mdbSearch); err != nil { return workflow.Failed(err) } - mongotConfig := createMongotConfig(r.mdbSearch, r.db) - configHash, err := r.ensureMongotConfig(ctx, mongotConfig) + ingressTlsMongotModification, ingressTlsStsModification, err := r.ensureIngressTlsConfig(ctx) if err != nil { return workflow.Failed(err) } - if err := r.createOrUpdateStatefulSet(ctx, log, configHash); err != nil { + egressTlsMongotModification, egressTlsStsModification, err := r.ensureEgressTlsConfig(ctx) + if err != nil { + return workflow.Failed(err) + } + + configHash, err := r.ensureMongotConfig(ctx, log, createMongotConfig(r.mdbSearch, r.db), ingressTlsMongotModification, egressTlsMongotModification) + if err != nil { return workflow.Failed(err) } - if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.db.NamespacedName().Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() { + configHashModification := statefulset.WithPodSpecTemplate(podtemplatespec.WithAnnotations( + map[string]string{ + "mongotConfigHash": configHash, + }, + )) + + if err := r.createOrUpdateStatefulSet(ctx, log, CreateSearchStatefulSetFunc(r.mdbSearch, r.db, r.buildImageString()), configHashModification, keyfileStsModification, ingressTlsStsModification, egressTlsStsModification); err != nil { + return workflow.Failed(err) + } + + if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.mdbSearch.Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() { return statefulSetStatus } return workflow.OK() } -func (r *MongoDBSearchReconcileHelper) createOrUpdateStatefulSet(ctx context.Context, log *zap.SugaredLogger, mongotConfigHash string) error { +func (r *MongoDBSearchReconcileHelper) ensureSourceKeyfile(ctx context.Context, log *zap.SugaredLogger) (statefulset.Modification, error) { + keyfileSecretName := kube.ObjectKey(r.mdbSearch.GetNamespace(), r.db.KeyfileSecretName()) + keyfileSecret := &corev1.Secret{} + if err := r.client.Get(ctx, keyfileSecretName, keyfileSecret); err != nil { + return nil, err + } + + return statefulset.Apply( + // make sure mongot pods get restarted if the keyfile changes + statefulset.WithPodSpecTemplate(podtemplatespec.WithAnnotations( + map[string]string{ + "keyfileHash": hashBytes(keyfileSecret.Data["keyfile"]), + }, + )), + ), nil +} + +func (r *MongoDBSearchReconcileHelper) buildImageString() string { imageVersion := r.mdbSearch.Spec.Version if imageVersion == "" { imageVersion = r.operatorSearchConfig.SearchVersion } - searchImage := fmt.Sprintf("%s/%s:%s", r.operatorSearchConfig.SearchRepo, r.operatorSearchConfig.SearchName, imageVersion) + return fmt.Sprintf("%s/%s:%s", r.operatorSearchConfig.SearchRepo, r.operatorSearchConfig.SearchName, imageVersion) +} +func (r *MongoDBSearchReconcileHelper) createOrUpdateStatefulSet(ctx context.Context, log *zap.SugaredLogger, modifications ...statefulset.Modification) error { stsName := r.mdbSearch.StatefulSetNamespacedName() sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: stsName.Name, Namespace: stsName.Namespace}} op, err := controllerutil.CreateOrUpdate(ctx, r.client, sts, func() error { - stsModification := CreateSearchStatefulSetFunc(r.mdbSearch, r.db, searchImage, mongotConfigHash) - stsModification(sts) + statefulset.Apply(modifications...)(sts) return nil }) if err != nil { @@ -142,7 +198,9 @@ func (r *MongoDBSearchReconcileHelper) ensureSearchService(ctx context.Context, return nil } -func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, mongotConfig mongot.Config) (string, error) { +func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, log *zap.SugaredLogger, modifications ...mongot.Modification) (string, error) { + mongotConfig := mongot.Config{} + mongot.Apply(modifications...)(&mongotConfig) configData, err := yaml.Marshal(mongotConfig) if err != nil { return "", err @@ -163,13 +221,89 @@ func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, m return "", err } - zap.S().Debugf("Updated mongot config yaml config map: %v (%s) with the following configuration: %s", cmName, op, string(configData)) + log.Debugf("Updated mongot config yaml config map: %v (%s) with the following configuration: %s", cmName, op, string(configData)) - return hashMongotConfig(configData), nil + return hashBytes(configData), nil } -func hashMongotConfig(mongotConfigYaml []byte) string { - hashBytes := sha256.Sum256(mongotConfigYaml) +func (r *MongoDBSearchReconcileHelper) ensureIngressTlsConfig(ctx context.Context) (mongot.Modification, statefulset.Modification, error) { + if !r.mdbSearch.Spec.Security.TLS.Enabled { + mongotModification := func(config *mongot.Config) { + config.Server.Wireproto.TLS.Mode = mongot.ConfigTLSModeDisabled + } + return mongotModification, statefulset.NOOP(), nil + } + + // TODO: validate that the certificate in the user-provided Secret in .spec.security.tls.certificateKeySecret is issued by the CA in the operator's CA Secret + + certFileName, err := tls.EnsureTLSSecret(ctx, r.client, r.mdbSearch) + if err != nil { + return nil, nil, err + } + + mongotModification := func(config *mongot.Config) { + certPath := tls.OperatorSecretMountPath + certFileName + config.Server.Wireproto.TLS.Mode = mongot.ConfigTLSModeTLS + config.Server.Wireproto.TLS.CertificateKeyFile = ptr.To(certPath) + } + + tlsSecret := r.mdbSearch.TLSOperatorSecretNamespacedName() + tlsVolume := statefulset.CreateVolumeFromSecret("tls", tlsSecret.Name) + tlsVolumeMount := statefulset.CreateVolumeMount("tls", tls.OperatorSecretMountPath, statefulset.WithReadOnly(true)) + statefulsetModification := statefulset.WithPodSpecTemplate(podtemplatespec.Apply( + podtemplatespec.WithVolume(tlsVolume), + podtemplatespec.WithContainer(MongotContainerName, container.Apply( + container.WithVolumeMounts([]corev1.VolumeMount{tlsVolumeMount}), + )), + )) + + return mongotModification, statefulsetModification, nil +} + +func (r *MongoDBSearchReconcileHelper) ensureEgressTlsConfig(ctx context.Context) (mongot.Modification, statefulset.Modification, error) { + tlsSourceConfig := r.db.TLSConfig() + if tlsSourceConfig == nil { + return mongot.NOOP(), statefulset.NOOP(), nil + } + + mongotModification := func(config *mongot.Config) { + config.SyncSource.ReplicaSet.TLS = ptr.To(true) + } + + _, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + caVolume := tlsSourceConfig.CAVolume + trustStoreVolume := statefulset.CreateVolumeFromEmptyDir("cacerts") + statefulsetModification := statefulset.WithPodSpecTemplate(podtemplatespec.Apply( + podtemplatespec.WithVolume(caVolume), + podtemplatespec.WithVolume(trustStoreVolume), + podtemplatespec.WithInitContainer("init-cacerts", container.Apply( + container.WithImage(r.buildImageString()), + containerSecurityContext, + container.WithVolumeMounts([]corev1.VolumeMount{ + statefulset.CreateVolumeMount(caVolume.Name, tls.CAMountPath, statefulset.WithReadOnly(true)), + statefulset.CreateVolumeMount(trustStoreVolume.Name, "/java/trust-store", statefulset.WithReadOnly(false)), + }), + container.WithCommand([]string{"sh"}), + container.WithArgs([]string{ + "-c", + fmt.Sprintf(` +cp /mongot-community/bin/jdk/lib/security/cacerts /java/trust-store/cacerts +/mongot-community/bin/jdk/bin/keytool -keystore /java/trust-store/cacerts -storepass changeit -noprompt -trustcacerts -importcert -alias mongodcert -file %s/%s + `, tls.CAMountPath, tlsSourceConfig.CAFileName), + }), + )), + podtemplatespec.WithContainer(MongotContainerName, container.Apply( + container.WithVolumeMounts([]corev1.VolumeMount{ + statefulset.CreateVolumeMount(trustStoreVolume.Name, "/mongot-community/bin/jdk/lib/security/cacerts", statefulset.WithReadOnly(true), statefulset.WithSubPath("cacerts")), + }), + )), + )) + + return mongotModification, statefulsetModification, nil +} + +func hashBytes(bytes []byte) string { + hashBytes := sha256.Sum256(bytes) return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hashBytes[:]) } @@ -203,30 +337,67 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service { TargetPort: intstr.FromInt32(search.GetMongotMetricsPort()), }) + serviceBuilder.AddPort(&corev1.ServicePort{ + Name: "healthcheck", + Protocol: corev1.ProtocolTCP, + Port: search.GetMongotHealthCheckPort(), + TargetPort: intstr.FromInt32(search.GetMongotHealthCheckPort()), + }) + return serviceBuilder.Build() } -func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Config { - return mongot.Config{CommunityPrivatePreview: mongot.CommunityPrivatePreview{ - MongodHostAndPort: fmt.Sprintf("%s.%s.svc.cluster.local:%d", db.DatabaseServiceName(), db.GetNamespace(), db.DatabasePort()), - QueryServerAddress: fmt.Sprintf("localhost:%d", search.GetMongotPort()), - KeyFilePath: "/mongot/keyfile/keyfile", - DataPath: "/mongot/data/config.yml", - Metrics: mongot.Metrics{ +func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Modification { + return func(config *mongot.Config) { + hostAndPorts := db.HostSeeds() + + config.SyncSource = mongot.ConfigSyncSource{ + ReplicaSet: mongot.ConfigReplicaSet{ + HostAndPort: hostAndPorts, + Username: search.SourceUsername(), + PasswordFile: "/tmp/sourceUserPassword", + TLS: ptr.To(false), + ReadPreference: ptr.To("secondaryPreferred"), + AuthSource: ptr.To("admin"), + }, + } + config.Storage = mongot.ConfigStorage{ + DataPath: "/mongot/data/config.yml", + } + config.Server = mongot.ConfigServer{ + Wireproto: &mongot.ConfigWireproto{ + Address: "0.0.0.0:27027", + Authentication: &mongot.ConfigAuthentication{ + Mode: "keyfile", + KeyFile: "/tmp/keyfile", + }, + }, + } + config.Metrics = mongot.ConfigMetrics{ Enabled: true, - Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()), - }, - Logging: mongot.Logging{ - Verbosity: "DEBUG", - }, - }} + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotMetricsPort()), + } + config.HealthCheck = mongot.ConfigHealthCheck{ + Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotHealthCheckPort()), + } + config.Logging = mongot.ConfigLogging{ + Verbosity: "TRACE", + LogPath: nil, + } + } } -func GetMongodConfigParameters(search *searchv1.MongoDBSearch) map[string]interface{} { - return map[string]interface{}{ - "setParameter": map[string]interface{}{ - "mongotHost": mongotHostAndPort(search), - "searchIndexManagementHostAndPort": mongotHostAndPort(search), +func GetMongodConfigParameters(search *searchv1.MongoDBSearch) map[string]any { + searchTLSMode := automationconfig.TLSModeDisabled + if search.Spec.Security.TLS.Enabled { + searchTLSMode = automationconfig.TLSModeRequired + } + return map[string]any{ + "setParameter": map[string]any{ + "mongotHost": mongotHostAndPort(search), + "searchIndexManagementHostAndPort": mongotHostAndPort(search), + "skipAuthenticationToSearchIndexManagementServer": false, + "searchTLSMode": string(searchTLSMode), }, } } @@ -236,27 +407,17 @@ func mongotHostAndPort(search *searchv1.MongoDBSearch) string { return fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName.Name, svcName.Namespace, search.GetMongotPort()) } -func ValidateSearchSource(db SearchSourceDBResource) error { - version, err := semver.ParseTolerant(db.GetMongoDBVersion()) - if err != nil { - return xerrors.Errorf("error parsing MongoDB version '%s': %w", db.GetMongoDBVersion(), err) - } else if version.Major < 8 { - return xerrors.New("MongoDB version must be 8.0 or higher") - } - - if db.IsSecurityTLSConfigEnabled() { - return xerrors.New("MongoDBSearch does not support TLS-enabled sources") +func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error { + if r.mdbSearch.Spec.Source != nil && r.mdbSearch.Spec.Source.ExternalMongoDBSource != nil { + return nil } - return nil -} - -func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error { + ref := r.mdbSearch.GetMongoDBResourceRef() searchList := &searchv1.MongoDBSearchList{} if err := r.client.List(ctx, searchList, &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, r.db.GetNamespace()+"/"+r.db.Name()), + FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, ref.Namespace+"/"+ref.Name), }); err != nil { - return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", r.db.Name(), err) + return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", ref.Name, err) } if len(searchList.Items) > 1 { @@ -264,8 +425,92 @@ func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSourc for i, search := range searchList.Items { resourceNames[i] = search.Name } - return xerrors.Errorf("Found multiple MongoDBSearch resources for search source '%s': %s", r.db.Name(), strings.Join(resourceNames, ", ")) + return xerrors.Errorf( + "Found multiple MongoDBSearch resources for search source '%s': %s", ref.Name, + strings.Join(resourceNames, ", "), + ) } return nil } + +func (r *MongoDBSearchReconcileHelper) ValidateSearchImageVersion() error { + version := r.getMongotImage() + + if strings.Contains(version, unsupportedSearchVersion) { + return xerrors.Errorf(unsupportedSearchVersionErrorFmt, unsupportedSearchVersion) + } + + return nil +} + +func (r *MongoDBSearchReconcileHelper) getMongotImage() string { + version := strings.TrimSpace(r.mdbSearch.Spec.Version) + if version != "" { + return version + } + + if r.operatorSearchConfig.SearchVersion != "" { + return r.operatorSearchConfig.SearchVersion + } + + if r.mdbSearch.Spec.StatefulSetConfiguration == nil { + return "" + } + + for _, container := range r.mdbSearch.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers { + if container.Name == MongotContainerName { + return container.Image + } + } + + return "" +} + +func SearchCoordinatorRole() mdbv1.MongoDBRole { + // direct translation of https://github.com/10gen/mongo/blob/6f8d95a513eea8f91ea9f5d895dd8a288dfcf725/src/mongo/db/auth/builtin_roles.yml#L652 + return mdbv1.MongoDBRole{ + Role: "searchCoordinator", + Db: "admin", + Roles: []mdbv1.InheritedRole{ + { + Role: "clusterMonitor", + Db: "admin", + }, + { + Role: "directShardOperations", + Db: "admin", + }, + { + Role: "readAnyDatabase", + Db: "admin", + }, + }, + Privileges: []mdbv1.Privilege{ + { + Resource: mdbv1.Resource{ + Db: "__mdb_internal_search", + }, + Actions: []string{ + "changeStream", "collStats", "dbHash", "dbStats", "find", + "killCursors", "listCollections", "listIndexes", "listSearchIndexes", + // performRawDataOperations is available only on mongod master + // "performRawDataOperations", + "planCacheRead", "cleanupStructuredEncryptionData", + "compactStructuredEncryptionData", "convertToCapped", "createCollection", + "createIndex", "createSearchIndexes", "dropCollection", "dropIndex", + "dropSearchIndex", "insert", "remove", "renameCollectionSameDB", + "update", "updateSearchIndex", + }, + }, + // TODO: this causes the error "(BadValue) resource: {cluster: true} conflicts with resource type 'db'" + // { + // Resource: mdbv1.Resource{ + // Cluster: ptr.To(true), + // }, + // Actions: []string{"bypassDefaultMaxTimeMS"}, + // }, + }, + AuthenticationRestrictions: nil, + } +} diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go b/controllers/search_controller/mongodbsearch_reconcile_helper_test.go index fe03ceb70..5a2a757ce 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper_test.go @@ -15,66 +15,6 @@ import ( kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" ) -func TestMongoDBSearchReconcileHelper_ValidateSearchSource(t *testing.T) { - mdbcMeta := metav1.ObjectMeta{ - Name: "test-mongodb", - Namespace: "test", - } - - cases := []struct { - name string - mdbc mdbcv1.MongoDBCommunity - expectedError string - }{ - { - name: "Invalid version", - mdbc: mdbcv1.MongoDBCommunity{ - ObjectMeta: mdbcMeta, - Spec: mdbcv1.MongoDBCommunitySpec{ - Version: "4.4.0", - }, - }, - expectedError: "MongoDB version must be 8.0 or higher", - }, - { - name: "Valid version", - mdbc: mdbcv1.MongoDBCommunity{ - ObjectMeta: mdbcMeta, - Spec: mdbcv1.MongoDBCommunitySpec{ - Version: "8.0", - }, - }, - }, - { - name: "TLS enabled", - mdbc: mdbcv1.MongoDBCommunity{ - ObjectMeta: mdbcMeta, - Spec: mdbcv1.MongoDBCommunitySpec{ - Version: "8.0", - Security: mdbcv1.Security{ - TLS: mdbcv1.TLS{ - Enabled: true, - }, - }, - }, - }, - expectedError: "MongoDBSearch does not support TLS-enabled sources", - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - db := NewSearchSourceDBResourceFromMongoDBCommunity(&c.mdbc) - err := ValidateSearchSource(db) - if c.expectedError == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, c.expectedError) - } - }) - } -} - func TestMongoDBSearchReconcileHelper_ValidateSingleMongoDBSearchForSearchSource(t *testing.T) { mdbSearchSpec := searchv1.MongoDBSearchSpec{ Source: &searchv1.MongoDBSource{ @@ -145,7 +85,7 @@ func TestMongoDBSearchReconcileHelper_ValidateSingleMongoDBSearchForSearchSource clientBuilder.WithObjects(v) } - helper := NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(clientBuilder.Build()), mdbSearch, NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), OperatorSearchConfig{}) + helper := NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(clientBuilder.Build()), mdbSearch, NewCommunityResourceSearchSource(mdbc), OperatorSearchConfig{}) err := helper.ValidateSingleMongoDBSearchForSearchSource(t.Context()) if c.expectedError == "" { assert.NoError(t, err) diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index ea169339d..ffaa72358 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -9,7 +9,7 @@ import ( searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/construct" - mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1/common" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/podtemplatespec" @@ -19,73 +19,30 @@ import ( ) const ( - MongotContainerName = "mongot" + MongotContainerName = "mongot" + SearchLivenessProbePath = "/health" + SearchReadinessProbePath = "/health" // Todo: Update this when search GA is available ) // SearchSourceDBResource is an object wrapping a MongoDBCommunity object // Its purpose is to: // - isolate and identify all the data we need to get from the CR in order to reconcile search resources // - implement search reconcile logic in a generic way that is working for any types of MongoDB databases (all database CRs). -// -// TODO check if we could use already existing interface (DbCommon, MongoDBStatefulSetOwner, etc.) type SearchSourceDBResource interface { - Name() string - NamespacedName() types.NamespacedName KeyfileSecretName() string - GetNamespace() string - HasSeparateDataAndLogsVolumes() bool - DatabaseServiceName() string - DatabasePort() int - GetMongoDBVersion() string - IsSecurityTLSConfigEnabled() bool + TLSConfig() *TLSSourceConfig + HostSeeds() []string + Validate() error } -func NewSearchSourceDBResourceFromMongoDBCommunity(mdbc *mdbcv1.MongoDBCommunity) SearchSourceDBResource { - return &mdbcSearchResource{db: mdbc} -} - -type mdbcSearchResource struct { - db *mdbcv1.MongoDBCommunity -} - -func (r *mdbcSearchResource) Name() string { - return r.db.Name -} - -func (r *mdbcSearchResource) NamespacedName() types.NamespacedName { - return r.db.NamespacedName() -} - -func (r *mdbcSearchResource) KeyfileSecretName() string { - return r.db.GetAgentKeyfileSecretNamespacedName().Name -} - -func (r *mdbcSearchResource) GetNamespace() string { - return r.db.Namespace -} - -func (r *mdbcSearchResource) HasSeparateDataAndLogsVolumes() bool { - return r.db.HasSeparateDataAndLogsVolumes() -} - -func (r *mdbcSearchResource) DatabaseServiceName() string { - return r.db.ServiceName() -} - -func (r *mdbcSearchResource) GetMongoDBVersion() string { - return r.db.Spec.Version -} - -func (r *mdbcSearchResource) IsSecurityTLSConfigEnabled() bool { - return r.db.Spec.Security.TLS.Enabled -} - -func (r *mdbcSearchResource) DatabasePort() int { - return r.db.GetMongodConfiguration().GetDBPort() +type TLSSourceConfig struct { + CAFileName string + CAVolume corev1.Volume + ResourcesToWatch map[watch.Type][]types.NamespacedName } // ReplicaSetOptions returns a set of options which will configure a ReplicaSet StatefulSet -func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBResource SearchSourceDBResource, searchImage string, mongotConfigHash string) statefulset.Modification { +func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBResource SearchSourceDBResource, searchImage string) statefulset.Modification { labels := map[string]string{ "app": mdbSearch.SearchServiceNamespacedName().Name, } @@ -95,14 +52,18 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso dataVolumeName := "data" keyfileVolumeName := "keyfile" + sourceUserPasswordVolumeName := "password" mongotConfigVolumeName := "config" pvcVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/mongot/data", statefulset.WithSubPath("data")) - keyfileVolume := statefulset.CreateVolumeFromSecret("keyfile", sourceDBResource.KeyfileSecretName()) + keyfileVolume := statefulset.CreateVolumeFromSecret(keyfileVolumeName, sourceDBResource.KeyfileSecretName()) keyfileVolumeMount := statefulset.CreateVolumeMount(keyfileVolumeName, "/mongot/keyfile", statefulset.WithReadOnly(true)) - mongotConfigVolume := statefulset.CreateVolumeFromConfigMap("config", mdbSearch.MongotConfigConfigMapNamespacedName().Name) + sourceUserPasswordVolume := statefulset.CreateVolumeFromSecret(sourceUserPasswordVolumeName, mdbSearch.SourceUserPasswordSecretRef().Name) + sourceUserPasswordVolumeMount := statefulset.CreateVolumeMount(sourceUserPasswordVolumeName, "/mongot/sourceUserPassword", statefulset.WithReadOnly(true)) + + mongotConfigVolume := statefulset.CreateVolumeFromConfigMap(mongotConfigVolumeName, mdbSearch.MongotConfigConfigMapNamespacedName().Name) mongotConfigVolumeMount := statefulset.CreateVolumeMount(mongotConfigVolumeName, "/mongot/config", statefulset.WithReadOnly(true)) var persistenceConfig *common.PersistenceConfig @@ -120,12 +81,14 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso keyfileVolumeMount, tmpVolumeMount, mongotConfigVolumeMount, + sourceUserPasswordVolumeMount, } volumes := []corev1.Volume{ tmpVolume, keyfileVolume, mongotConfigVolume, + sourceUserPasswordVolume, } stsModifications := []statefulset.Modification{ @@ -142,11 +105,7 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso podtemplatespec.Apply( podSecurityContext, podtemplatespec.WithPodLabels(labels), - podtemplatespec.WithAnnotations(map[string]string{ - "mongotConfigHash": mongotConfigHash, - }), podtemplatespec.WithVolumes(volumes), - podtemplatespec.WithServiceAccount(sourceDBResource.DatabaseServiceName()), podtemplatespec.WithServiceAccount(util.MongoDBServiceAccount), podtemplatespec.WithContainer(MongotContainerName, mongodbSearchContainer(mdbSearch, volumeMounts, searchImage)), ), @@ -170,22 +129,65 @@ func mongodbSearchContainer(mdbSearch *searchv1.MongoDBSearch, volumeMounts []co container.WithName(MongotContainerName), container.WithImage(searchImage), container.WithImagePullPolicy(corev1.PullAlways), - container.WithReadinessProbe(probes.Apply( - probes.WithTCPSocket("", intstr.FromInt32(mdbSearch.GetMongotPort())), - probes.WithInitialDelaySeconds(20), - probes.WithPeriodSeconds(10), - )), + container.WithLivenessProbe(mongotLivenessProbe(mdbSearch)), + container.WithReadinessProbe(mongotReadinessProbe(mdbSearch)), container.WithResourceRequirements(createSearchResourceRequirements(mdbSearch.Spec.ResourceRequirements)), container.WithVolumeMounts(volumeMounts), container.WithCommand([]string{"sh"}), container.WithArgs([]string{ "-c", - "/mongot-community/mongot --config /mongot/config/config.yml", + ` +cp /mongot/keyfile/keyfile /tmp/keyfile +chown 2000:2000 /tmp/keyfile +chmod 0600 /tmp/keyfile + +cp /mongot/sourceUserPassword/password /tmp/sourceUserPassword +chown 2000:2000 /tmp/sourceUserPassword +chmod 0600 /tmp/sourceUserPassword + +/mongot-community/mongot --config /mongot/config/config.yml +`, }), containerSecurityContext, ) } +func mongotLivenessProbe(search *searchv1.MongoDBSearch) func(*corev1.Probe) { + return probes.Apply( + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Port: intstr.FromInt32(search.GetMongotHealthCheckPort()), + Path: SearchLivenessProbePath, + }, + }), + probes.WithInitialDelaySeconds(10), + probes.WithPeriodSeconds(10), + probes.WithTimeoutSeconds(5), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(10), + ) +} + +// mongotReadinessProbe just uses the endpoint intended for liveness checks; +// readiness check endpoint may be available in search GA. +func mongotReadinessProbe(search *searchv1.MongoDBSearch) func(*corev1.Probe) { + return probes.Apply( + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Port: intstr.FromInt32(search.GetMongotHealthCheckPort()), + Path: SearchReadinessProbePath, + }, + }), + probes.WithInitialDelaySeconds(20), + probes.WithPeriodSeconds(10), + probes.WithTimeoutSeconds(5), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(3), + ) +} + func createSearchResourceRequirements(requirements *corev1.ResourceRequirements) corev1.ResourceRequirements { if requirements != nil { return *requirements diff --git a/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go b/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go index 793914d3a..ca7b9a3b0 100755 --- a/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go +++ b/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go @@ -145,7 +145,7 @@ func readLinesFromFile(name string) ([]string, error) { func writeLinesToFile(name string, lines []string) error { output := strings.Join(lines, lineBreak) - err := os.WriteFile(name, []byte(output), 0o775) + err := os.WriteFile(name, []byte(output), 0o644) if err != nil { return xerrors.Errorf("error writing to file %s: %w", name, err) } @@ -168,7 +168,7 @@ func appendLinesToFile(name string, lines string) error { func getOmPropertiesFromEnvVars() map[string]string { props := map[string]string{} - for _, pair := range os.Environ() { + for _, pair := range os.Environ() { // nolint:forbidigo if !strings.HasPrefix(pair, omPropertyPrefix) { continue } diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index da6887b81..9e0cca8fe 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -120,6 +120,9 @@ def helm_repo_add(repo_name: str, url: str): def process_run_and_check(args, **kwargs): + if "check" not in kwargs: + kwargs["check"] = True + try: logger.debug(f"subprocess.run: {args}") completed_process = subprocess.run(args, **kwargs) diff --git a/docker/mongodb-kubernetes-tests/kubetester/mongodb.py b/docker/mongodb-kubernetes-tests/kubetester/mongodb.py index c9970361a..b3f38239b 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/mongodb.py +++ b/docker/mongodb-kubernetes-tests/kubetester/mongodb.py @@ -278,10 +278,8 @@ def configure_custom_tls( tls_cert_secret_name: str, ): ensure_nested_objects(self, ["spec", "security", "tls"]) - self["spec"]["security"] = { - "certsSecretPrefix": tls_cert_secret_name, - "tls": {"enabled": True, "ca": issuer_ca_configmap_name}, - } + self["spec"]["security"]["certsSecretPrefix"] = tls_cert_secret_name + self["spec"]["security"]["tls"].update({"enabled": True, "ca": issuer_ca_configmap_name}) def build_list_of_hosts(self): """Returns the list of full_fqdn:27017 for every member of the mongodb resource""" diff --git a/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py b/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py index e5629bda0..be2151158 100644 --- a/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py +++ b/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py @@ -1,3 +1,7 @@ +import logging + +import pymongo.errors +from kubetester import kubetester from tests import test_logger from tests.common.search.search_tester import SearchTester @@ -26,9 +30,33 @@ def create_search_index(self): def wait_for_search_indexes(self): self.search_tester.wait_for_search_indexes_ready(self.db_name, self.col_name) - def assert_search_query(self): - # sample taken from: https://www.mongodb.com/docs/atlas/atlas-search/tutorial/run-query/#run-a-complex-query-on-the-movies-collection-7 - result = self.search_tester.client[self.db_name][self.col_name].aggregate( + def assert_search_query(self, retry_timeout: int = 1): + def wait_for_search_results(): + # sample taken from: https://www.mongodb.com/docs/atlas/atlas-search/tutorial/run-query/#run-a-complex-query-on-the-movies-collection-7 + count = 0 + status_msg = "" + try: + result = self.execute_example_search_query() + status_msg = f"{self.db_name}/{self.col_name}: search query results:\n" + for r in result: + status_msg += f"{r}\n" + count += 1 + status_msg += f"Count: {count}" + logger.debug(status_msg) + except pymongo.errors.PyMongoError as e: + logger.debug(f"error: {e}") + + return count == 4, status_msg + + kubetester.run_periodically( + fn=wait_for_search_results, + timeout=retry_timeout, + sleep_time=1, + msg="Search query to return correct data", + ) + + def execute_example_search_query(self): + return self.search_tester.client[self.db_name][self.col_name].aggregate( [ { "$search": { @@ -47,11 +75,3 @@ def assert_search_query(self): {"$project": {"title": 1, "plot": 1, "genres": 1, "_id": 0}}, ] ) - - logger.debug(f"{self.db_name}/{self.col_name}: search query results:") - count = 0 - for r in result: - logger.debug(r) - count += 1 - - assert count == 4 diff --git a/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py b/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py index bbca34b3b..6dccfb5ae 100644 --- a/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py +++ b/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py @@ -15,8 +15,10 @@ class SearchTester(MongoTester): def __init__( self, connection_string: str, + use_ssl: bool = False, + ca_path: str | None = None, ): - super().__init__(connection_string, False) + super().__init__(connection_string, use_ssl, ca_path) def mongorestore_from_url(self, archive_url: str, ns_include: str, mongodb_tools_dir: str = ""): logger.debug(f"running mongorestore from {archive_url}") @@ -26,7 +28,11 @@ def mongorestore_from_url(self, archive_url: str, ns_include: str, mongodb_tools logger.debug(f"Downloaded sample file from {archive_url} to {sample_file.name} (size: {size})") mongorestore_path = os.path.join(mongodb_tools_dir, "mongorestore") mongorestore_cmd = f"{mongorestore_path} --archive={sample_file.name} --verbose=1 --drop --nsInclude {ns_include} --uri={self.cnx_string}" - process_run_and_check(mongorestore_cmd.split()) + if self.default_opts.get("tls", False): + mongorestore_cmd += " --ssl" + if ca_path := self.default_opts.get("tlsCAFile"): + mongorestore_cmd += " --sslCAFile=" + ca_path + process_run_and_check(mongorestore_cmd.split(), capture_output=True) def create_search_index(self, database_name: str, collection_name: str): database = self.client[database_name] @@ -49,6 +55,10 @@ def wait_for_search_indexes_ready(self, database_name: str, collection_name: str def search_indexes_ready(self, database_name: str, collection_name: str): search_indexes = self.get_search_indexes(database_name, collection_name) + if len(search_indexes) == 0: + logger.error(f"There are no search indexes available in {database_name}.{collection_name}") + return False + for idx in search_indexes: if idx.get("status") != "READY": logger.debug(f"{database_name}/{collection_name}: search index {idx} is not ready") diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml new file mode 100644 index 000000000..45e332fd3 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix-external.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: mdbc-rs +spec: + version: 8.0.10 + type: ReplicaSet + members: 3 + security: + authentication: + ignoreUnknownUsers: true + modes: + - SCRAM + roles: + - role: searchCoordinator + db: admin + roles: + - name: clusterMonitor + db: admin + - name: directShardOperations + db: admin + - name: readAnyDatabase + db: admin + privileges: + - resource: + db: "__mdb_internal_search" + collection: "" + actions: + - "changeStream" + - "collStats" + - "dbHash" + - "dbStats" + - "find" + - "killCursors" + - "listCollections" + - "listIndexes" + - "listSearchIndexes" + - "planCacheRead" + - "cleanupStructuredEncryptionData" + - "compactStructuredEncryptionData" + - "convertToCapped" + - "createCollection" + - "createIndex" + - "createSearchIndexes" + - "dropCollection" + - "dropIndex" + - "dropSearchIndex" + - "insert" + - "remove" + - "renameCollectionSameDB" + - "update" + - "updateSearchIndex" + - resource: + cluster: true + actions: + - "bypassDefaultMaxTimeMS" + agent: + logLevel: DEBUG + statefulSet: + spec: + template: + spec: + containers: + - name: mongod + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi + - name: mongodb-agent + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "0.5" + memory: 1Gi + users: + # admin user with root role + - name: mdb-admin + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-admin-user-password + scramCredentialsSecretName: mdb-admin-user + roles: + - name: root + db: admin + # user performing search queries + - name: mdb-user + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-user-password + scramCredentialsSecretName: mdb-user-scram + roles: + - name: restore + db: sample_mflix + - name: readWrite + db: sample_mflix + # user used by MongoDB Search to connect to MongoDB database to synchronize data from + # For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically + # From MongoDB 8.2, searchCoordinator role will be a built-in role. + - name: search-sync-source + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: mdbc-rs-search-sync-source-password + scramCredentialsSecretName: mdbc-rs-search-sync-source + roles: + - name: searchCoordinator + db: admin diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml index 4ba3d40b6..fe54d614d 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml @@ -4,27 +4,16 @@ kind: MongoDBCommunity metadata: name: mdbc-rs spec: - members: 3 + version: 8.0.10 type: ReplicaSet - version: "8.0.5" + members: 3 security: authentication: - modes: ["SCRAM"] + ignoreUnknownUsers: true + modes: + - SCRAM agent: logLevel: DEBUG - users: - - name: my-user - db: admin - passwordSecretRef: # a reference to the secret that will be used to generate the user's password - name: my-user-password - roles: - - name: clusterAdmin - db: admin - - name: userAdminAnyDatabase - db: admin - - name: readWrite - db: sample_mflix - scramCredentialsSecretName: my-scram statefulSet: spec: template: @@ -33,16 +22,48 @@ spec: - name: mongod resources: limits: - cpu: "3" - memory: 5Gi - requests: cpu: "2" - memory: 5Gi + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi - name: mongodb-agent resources: limits: - cpu: "3" - memory: 5Gi + cpu: "1" + memory: 2Gi requests: - cpu: "2" - memory: 5Gi + cpu: "0.5" + memory: 1Gi + users: + # admin user with root role + - name: mdb-admin + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-admin-user-password + scramCredentialsSecretName: mdb-admin-user + roles: + - name: root + db: admin + # user performing search queries + - name: mdb-user + db: admin + passwordSecretRef: # a reference to the secret containing user password + name: mdb-user-password + scramCredentialsSecretName: mdb-user-scram + roles: + - name: restore + db: sample_mflix + - name: readWrite + db: sample_mflix + # user used by MongoDB Search to connect to MongoDB database to synchronize data from + # For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically + # From MongoDB 8.2, searchCoordinator role will be a built-in role. + - name: search-sync-source + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: mdbc-rs-search-sync-source-password + scramCredentialsSecretName: mdbc-rs-search-sync-source + roles: + - name: searchCoordinator + db: admin diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml new file mode 100644 index 000000000..29d455b51 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-rs +spec: + members: 3 + version: 8.0.10 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + security: + authentication: + enabled: true + ignoreUnknownUsers: true + modes: + - SCRAM + agent: + logLevel: DEBUG + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml new file mode 100644 index 000000000..0b3fe4c77 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml @@ -0,0 +1,16 @@ +# admin user with root role +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mdb-admin +spec: + username: mdb-admin + db: admin + mongodbResourceRef: + name: mdb-rs + passwordSecretKeyRef: + name: mdb-admin-user-password + key: password + roles: + - name: root + db: admin \ No newline at end of file diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml new file mode 100644 index 000000000..579cc58d0 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-user.yaml @@ -0,0 +1,16 @@ +# user performing search queries +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mdb-user +spec: + username: mdb-user + db: admin + mongodbResourceRef: + name: mdb-rs + passwordSecretKeyRef: + name: mdb-user-password + key: password + roles: + - name: readWrite + db: sample_mflix diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml new file mode 100644 index 000000000..cd1eab1a5 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml @@ -0,0 +1,18 @@ +# user used by MongoDB Search to connect to MongoDB database to synchronize data from +# For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically +# From MongoDB 8.2, searchCoordinator role will be a built-in role. +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: search-sync-source-user +spec: + username: search-sync-source + db: admin + mongodbResourceRef: + name: mdb-rs + passwordSecretKeyRef: + name: mdb-rs-search-sync-source-password + key: password + roles: + - name: searchCoordinator + db: admin diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml new file mode 100644 index 000000000..33648e04a --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml @@ -0,0 +1,5 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBSearch +metadata: + name: mdbc-rs +spec: {} diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py index 69cbca895..97e9c1af7 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py @@ -12,21 +12,26 @@ logger = test_logger.get_test_logger(__name__) -USER_PASSWORD = "Passw0rd." -USER_NAME = "my-user" +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + MDBC_RESOURCE_NAME = "mdbc-rs" @fixture(scope="function") -def mdbc(namespace: str, custom_mdb_version: str) -> MongoDBCommunity: +def mdbc(namespace: str) -> MongoDBCommunity: resource = MongoDBCommunity.from_yaml( yaml_fixture("community-replicaset-sample-mflix.yaml"), name=MDBC_RESOURCE_NAME, namespace=namespace, ) - resource["spec"]["version"] = custom_mdb_version - if try_load(resource): return resource @@ -53,8 +58,14 @@ def test_install_operator(namespace: str, operator_installation_config: dict[str @mark.e2e_search_community_basic -def test_install_secret(namespace: str): - create_or_update_secret(namespace=namespace, name="my-user-password", data={"password": USER_PASSWORD}) +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) @mark.e2e_search_community_basic @@ -64,7 +75,7 @@ def test_create_database_resource(mdbc: MongoDBCommunity): @mark.e2e_search_community_basic -def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): +def test_create_search_resource(mdbs: MongoDBSearch): mdbs.update() mdbs.assert_reaches_phase(Phase.Running, timeout=300) @@ -76,7 +87,9 @@ def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): @fixture(scope="function") def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper: - return movies_search_helper.SampleMoviesSearchHelper(SearchTester(get_connection_string(mdbc))) + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)) + ) @mark.e2e_search_community_basic @@ -89,15 +102,10 @@ def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelp sample_movies_helper.create_search_index() -@mark.e2e_search_community_basic -def test_search_wait_for_search_indexes(sample_movies_helper: SampleMoviesSearchHelper): - sample_movies_helper.wait_for_search_indexes() - - @mark.e2e_search_community_basic def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): - sample_movies_helper.assert_search_query() + sample_movies_helper.assert_search_query(retry_timeout=60) -def get_connection_string(mdbc: MongoDBCommunity) -> str: - return f"mongodb://{USER_NAME}:{USER_PASSWORD}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py new file mode 100644 index 000000000..110e134dc --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -0,0 +1,144 @@ +from kubetester import create_or_update_secret, try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" +MDBS_RESOURCE_NAME = "mdbs" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix-external.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + mongot_host = f"{MDBS_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" + if "additionalMongodConfig" not in resource["spec"]: + resource["spec"]["additionalMongodConfig"] = {} + if "setParameter" not in resource["spec"]["additionalMongodConfig"]: + resource["spec"]["additionalMongodConfig"]["setParameter"] = {} + + resource["spec"]["additionalMongodConfig"]["setParameter"].update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "disabled", + } + ) + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + name=MDBS_RESOURCE_NAME, + namespace=namespace, + ) + + seeds = [ + f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" for i in range(mdbc["spec"]["members"]) + ] + + resource["spec"] = { + "source": { + "external": { + "hostAndPorts": seeds, + "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile", "key": "keyfile"}, + "tls": {"enabled": False}, + }, + "passwordSecretRef": {"name": f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", "key": "password"}, + "username": MONGOT_USER_NAME, + } + } + + return resource + + +@mark.e2e_search_external_basic +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_external_basic +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + + create_or_update_secret( + namespace=namespace, + name=f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", + data={"password": MONGOT_USER_PASSWORD}, + ) + + +@mark.e2e_search_external_basic +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_basic +def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_basic +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)) + ) + + +@mark.e2e_search_external_basic +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_external_basic +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_external_basic +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py new file mode 100644 index 000000000..f6a270a48 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -0,0 +1,190 @@ +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" +MDBS_RESOURCE_NAME = "mdbs" +TLS_SECRET_NAME = "tls-secret" +TLS_CA_SECRET_NAME = "tls-ca-secret" +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix-external.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + mongot_host = f"{MDBS_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" + if "additionalMongodConfig" not in resource["spec"]: + resource["spec"]["additionalMongodConfig"] = {} + if "setParameter" not in resource["spec"]["additionalMongodConfig"]: + resource["spec"]["additionalMongodConfig"]["setParameter"] = {} + + resource["spec"]["additionalMongodConfig"]["setParameter"].update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "requireTLS", + } + ) + + resource["spec"]["security"]["tls"] = { + "enabled": True, + "certificateKeySecretRef": {"name": TLS_SECRET_NAME}, + "caCertificateSecretRef": {"name": TLS_SECRET_NAME}, + } + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + name=MDBS_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + return resource + + +@mark.e2e_search_external_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_external_tls +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, + name=f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", + data={"password": MONGOT_USER_PASSWORD}, + ) + + +@mark.e2e_search_external_tls +def test_install_tls_secrets_and_configmaps( + namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str, issuer_ca_filepath: str +): + create_tls_certs(issuer, namespace, mdbc.name, mdbc["spec"]["members"], secret_name=TLS_SECRET_NAME) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + ca = open(issuer_ca_filepath).read() + + ca_secret_name = f"{mdbc.name}-ca" + create_or_update_secret(namespace=namespace, name=ca_secret_name, data={"ca.crt": ca}) + + +@mark.e2e_search_external_tls +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_tls +def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): + seeds = [ + f"{mdbc.name}-{i}.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017" + for i in range(mdbc["spec"]["members"]) + ] + + mdbs["spec"]["source"] = { + "external": { + "hostAndPorts": seeds, + "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, + "tls": { + "enabled": True, + "ca": {"name": f"{mdbc.name}-ca"}, + }, + }, + "passwordSecretRef": {"name": f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", "key": "password"}, + "username": MONGOT_USER_NAME, + } + + mdbs["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_external_tls +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester( + get_connection_string(mdbc, USER_NAME, USER_PASSWORD), + use_ssl=True, + ca_path=issuer_ca_filepath, + ) + ) + + +@mark.e2e_search_external_tls +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_external_tls +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_external_tls +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return ( + f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/" + f"?replicaSet={mdbc.name}" + ) diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py new file mode 100644 index 000000000..44418b793 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py @@ -0,0 +1,173 @@ +import pymongo +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" + +TLS_SECRET_NAME = "tls-secret" + +# MongoDBSearch TLS configuration +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + # Add TLS configuration + resource["spec"]["security"]["tls"] = { + "enabled": True, + "certificateKeySecretRef": {"name": TLS_SECRET_NAME}, + "caCertificateSecretRef": {"name": TLS_SECRET_NAME}, + } + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + # Add TLS configuration to MongoDBSearch + if "spec" not in resource: + resource["spec"] = {} + + resource["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + return resource + + +@mark.e2e_search_community_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_community_tls +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + # Create user password secrets + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) + + +@mark.e2e_search_community_tls +def test_install_tls_secrets_and_configmaps(namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str): + create_tls_certs(issuer, namespace, mdbc.name, mdbc["spec"]["members"], secret_name=TLS_SECRET_NAME) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + +@mark.e2e_search_community_tls +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_community_tls +def test_create_search_resource(mdbs: MongoDBSearch): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_community_tls +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@mark.e2e_search_community_tls +def test_validate_tls_connections(mdbc: MongoDBCommunity, mdbs: MongoDBSearch, namespace: str, issuer_ca_filepath: str): + with pymongo.MongoClient( + f"mongodb://{mdbc.name}-0.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=30000, + connectTimeoutMS=20000, + ) as mongodb_client: + mongodb_info = mongodb_client.admin.command("hello") + assert mongodb_info.get("ok") == 1, "MongoDBCommunity connection failed" + + with pymongo.MongoClient( + f"mongodb://{mdbs.name}-search-svc.{namespace}.svc.cluster.local:27027", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=10000, + connectTimeoutMS=10000, + ) as search_client: + search_info = search_client.admin.command("hello") + assert search_info.get("ok") == 1, "MongoDBSearch connection failed" + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath), + ) + + +@mark.e2e_search_community_tls +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_community_tls +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_community_tls +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py new file mode 100644 index 000000000..a3de8dcf8 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py @@ -0,0 +1,183 @@ +import yaml +from kubetester import create_or_update_secret, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB +from kubetester.mongodb_search import MongoDBSearch +from kubetester.mongodb_user import MongoDBUser +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = f"{ADMIN_USER_NAME}-password" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = f"{MONGOT_USER_NAME}-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = f"{USER_NAME}-password" + +MDB_RESOURCE_NAME = "mdb-rs" + + +@fixture(scope="function") +def mdb(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("enterprise-replicaset-sample-mflix.yaml"), + name=MDB_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml(yaml_fixture("search-minimal.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME) + + if try_load(resource): + return resource + + return resource + + +@fixture(scope="function") +def admin_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-mdb-admin.yaml"), namespace=namespace, name=ADMIN_USER_NAME + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodbuser-mdb-user.yaml"), namespace=namespace, name=USER_NAME) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def mongot_user(namespace: str, mdbs: MongoDBSearch) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-search-sync-source-user.yaml"), + namespace=namespace, + name=f"{mdbs.name}-{MONGOT_USER_NAME}", + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = MONGOT_USER_NAME + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@mark.e2e_search_enterprise_basic +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_enterprise_basic +def test_create_database_resource(mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_enterprise_basic +def test_create_users( + namespace: str, admin_user: MongoDBUser, user: MongoDBUser, mongot_user: MongoDBUser, mdb: MongoDB +): + create_or_update_secret( + namespace, name=admin_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": ADMIN_USER_PASSWORD} + ) + admin_user.create() + admin_user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=user["spec"]["passwordSecretKeyRef"]["name"], data={"password": USER_PASSWORD} + ) + user.create() + user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=mongot_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": MONGOT_USER_PASSWORD} + ) + mongot_user.create() + # we deliberately don't wait for this user to be ready, because to be reconciled successfully it needs the searchCoordinator role + # which the ReplicaSet reconciler will only define in the automation config after the MongoDBSearch resource is created. + + +@mark.e2e_search_enterprise_basic +def test_create_search_resource(mdbs: MongoDBSearch): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_enterprise_basic +def test_wait_for_database_resource_ready(mdb: MongoDB): + mdb.assert_abandons_phase(Phase.Running, timeout=1800) + mdb.assert_reaches_phase(Phase.Running, timeout=1800) + + for idx in range(mdb.get_members()): + mongod_config = yaml.safe_load( + KubernetesTester.run_command_in_pod_container( + f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"] + ) + ) + setParameter = mongod_config.get("setParameter", {}) + assert ( + "mongotHost" in setParameter and "searchIndexManagementHostAndPort" in setParameter + ), "mongot parameters not found in mongod config" + + +@mark.e2e_search_enterprise_basic +def test_search_restore_sample_database(mdb: MongoDB): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD)) + ) + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_enterprise_basic +def test_search_create_search_index(mdb: MongoDB): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD)) + ) + sample_movies_helper.create_search_index() + + +@mark.e2e_search_enterprise_basic +def test_search_assert_search_query(mdb: MongoDB): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD)) + ) + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py new file mode 100644 index 000000000..f4c0bc96f --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py @@ -0,0 +1,239 @@ +import pymongo +import yaml +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_mongodb_tls_certs, create_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB +from kubetester.mongodb_search import MongoDBSearch +from kubetester.mongodb_user import MongoDBUser +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = f"{ADMIN_USER_NAME}-password" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = f"{MONGOT_USER_NAME}-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = f"{USER_NAME}-password" + +MDB_RESOURCE_NAME = "mdb-rs" + +# MongoDBSearch TLS configuration +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdb(namespace: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("enterprise-replicaset-sample-mflix.yaml"), + name=MDB_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.configure_custom_tls(issuer_ca_configmap, "certs") + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml(yaml_fixture("search-minimal.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME) + + if try_load(resource): + return resource + + # Add TLS configuration to MongoDBSearch + if "spec" not in resource: + resource["spec"] = {} + + resource["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + return resource + + +@fixture(scope="function") +def admin_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-mdb-admin.yaml"), namespace=namespace, name=ADMIN_USER_NAME + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodbuser-mdb-user.yaml"), namespace=namespace, name=USER_NAME) + + if try_load(resource): + return resource + + resource["spec"]["username"] = resource.name + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@fixture(scope="function") +def mongot_user(namespace: str, mdbs: MongoDBSearch) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodbuser-search-sync-source-user.yaml"), + namespace=namespace, + name=f"{mdbs.name}-{MONGOT_USER_NAME}", + ) + + if try_load(resource): + return resource + + resource["spec"]["username"] = MONGOT_USER_NAME + resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password" + + return resource + + +@mark.e2e_search_enterprise_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_enterprise_tls +def test_install_tls_secrets_and_configmaps(namespace: str, mdb: MongoDB, mdbs: MongoDBSearch, issuer: str): + create_mongodb_tls_certs(issuer, namespace, mdb.name, f"certs-{mdb.name}-cert", mdb.get_members()) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + +@mark.e2e_search_enterprise_tls +def test_create_database_resource(mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_enterprise_tls +def test_create_users( + namespace: str, admin_user: MongoDBUser, user: MongoDBUser, mongot_user: MongoDBUser, mdb: MongoDB +): + create_or_update_secret( + namespace, name=admin_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": ADMIN_USER_PASSWORD} + ) + admin_user.create() + admin_user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=user["spec"]["passwordSecretKeyRef"]["name"], data={"password": USER_PASSWORD} + ) + user.create() + user.assert_reaches_phase(Phase.Updated, timeout=300) + + create_or_update_secret( + namespace, name=mongot_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": MONGOT_USER_PASSWORD} + ) + mongot_user.create() + # we deliberately don't wait for this user to be ready, because to be reconciled successfully it needs the searchCoordinator role + # which the ReplicaSet reconciler will only define in the automation config after the MongoDBSearch resource is created. + + +@mark.e2e_search_enterprise_tls +def test_create_search_resource(mdbs: MongoDBSearch): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_enterprise_tls +def test_wait_for_database_resource_ready(mdb: MongoDB): + mdb.assert_abandons_phase(Phase.Running, timeout=1800) + mdb.assert_reaches_phase(Phase.Running, timeout=1800) + + for idx in range(mdb.get_members()): + mongod_config = yaml.safe_load( + KubernetesTester.run_command_in_pod_container( + f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"] + ) + ) + setParameter = mongod_config.get("setParameter", {}) + assert ( + "mongotHost" in setParameter and "searchIndexManagementHostAndPort" in setParameter + ), "mongot parameters not found in mongod config" + + +@mark.e2e_search_enterprise_tls +def test_validate_tls_connections(mdb: MongoDB, mdbs: MongoDBSearch, namespace: str, issuer_ca_filepath: str): + with pymongo.MongoClient( + f"mongodb://{mdb.name}-0.{mdb.name}-svc.{namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=30000, + connectTimeoutMS=20000, + ) as mongodb_client: + mongodb_info = mongodb_client.admin.command("hello") + assert mongodb_info.get("ok") == 1, "MongoDB connection failed" + + with pymongo.MongoClient( + f"mongodb://{mdbs.name}-search-svc.{namespace}.svc.cluster.local:27027", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=10000, + connectTimeoutMS=10000, + ) as search_client: + search_info = search_client.admin.command("hello") + assert search_info.get("ok") == 1, "MongoDBSearch connection failed" + + +@mark.e2e_search_enterprise_tls +def test_search_restore_sample_database(mdb: MongoDB, issuer_ca_filepath: str): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester( + get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath + ) + ) + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_enterprise_tls +def test_search_create_search_index(mdb: MongoDB, issuer_ca_filepath: str): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath) + ) + sample_movies_helper.create_search_index() + + +@mark.e2e_search_enterprise_tls +def test_search_assert_search_query(mdb: MongoDB, issuer_ca_filepath: str): + sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath) + ) + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}" diff --git a/helm_chart/crds/mongodb.com_mongodbsearch.yaml b/helm_chart/crds/mongodb.com_mongodbsearch.yaml index 72ea0e50e..38dc38f2a 100644 --- a/helm_chart/crds/mongodb.com_mongodbsearch.yaml +++ b/helm_chart/crds/mongodb.com_mongodbsearch.yaml @@ -49,6 +49,8 @@ spec: spec: properties: persistence: + description: Configure MongoDB Search's persistent volume. If not + defined, the operator will request 10GB of storage. properties: multiple: properties: @@ -95,7 +97,8 @@ spec: type: object type: object resourceRequirements: - description: ResourceRequirements describes the compute resource requirements. + description: Configure resource requests and limits for the MongoDB + Search pods. properties: claims: description: |- @@ -153,8 +156,88 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + security: + description: Configure security settings of the MongoDB Search server + that MongoDB database is connecting to when performing search queries. + properties: + tls: + properties: + certificateKeySecretRef: + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object source: + description: MongoDB database connection details from which MongoDB + Search will synchronize data to build indexes. properties: + external: + properties: + hostAndPorts: + items: + type: string + type: array + keyFileSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + tls: + properties: + ca: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object mongodbResourceRef: properties: name: @@ -164,11 +247,26 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. + StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + which aren't exposed as fields in the MongoDBSearch.spec. properties: metadata: description: StatefulSetMetadataWrapper is a wrapper around Labels @@ -190,6 +288,9 @@ spec: - spec type: object version: + description: Optional version of MongoDB Search component (mongot). + If not set, then the operator will set the most appropriate version + of MongoDB Search. type: string type: object status: diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index ba3e5b168..67e062648 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -88,6 +89,9 @@ func NewReconciler(mgr manager.Manager, mongodbRepoUrl, mongodbImage, mongodbIma func findMdbcForSearch(ctx context.Context, rawObj k8sClient.Object) []reconcile.Request { mdbSearch := rawObj.(*searchv1.MongoDBSearch) + if mdbSearch.GetMongoDBResourceRef() == nil { + return nil + } return []reconcile.Request{ {NamespacedName: types.NamespacedName{Namespace: mdbSearch.GetMongoDBResourceRef().Namespace, Name: mdbSearch.GetMongoDBResourceRef().Name}}, } @@ -711,8 +715,8 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb // and that this resource passes search validations. If either fails, proceed without a search target // for the mongod automation config. if len(searchList.Items) == 1 { - searchSource := search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(&mdb) - if search_controller.ValidateSearchSource(searchSource) == nil { + searchSource := search_controller.NewCommunityResourceSearchSource(&mdb) + if searchSource.Validate() == nil { search = &searchList.Items[0] } } @@ -727,6 +731,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb prometheusModification, processPortManager.GetPortsModification(), getMongodConfigSearchModification(search), + searchCoordinatorCustomRoleModification(search), ) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not create an automation config: %s", err) @@ -739,6 +744,66 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb return automationConfig, nil } +// TODO: remove this as soon as searchCoordinator builtin role is backported +func searchCoordinatorCustomRoleModification(search *searchv1.MongoDBSearch) automationconfig.Modification { + if search == nil { + return automationconfig.NOOP() + } + + return func(ac *automationconfig.AutomationConfig) { + searchCoordinatorRole := searchCoordinatorCustomRoleStruct() + ac.Roles = append(ac.Roles, searchCoordinatorRole) + } +} + +func searchCoordinatorCustomRoleStruct() automationconfig.CustomRole { + // direct translation of https://github.com/10gen/mongo/blob/6f8d95a513eea8f91ea9f5d895dd8a288dfcf725/src/mongo/db/auth/builtin_roles.yml#L652 + return automationconfig.CustomRole{ + Role: "searchCoordinator", + DB: "admin", + Roles: []automationconfig.Role{ + { + Role: "clusterMonitor", + Database: "admin", + }, + { + Role: "directShardOperations", + Database: "admin", + }, + { + Role: "readAnyDatabase", + Database: "admin", + }, + }, + Privileges: []automationconfig.Privilege{ + { + Resource: automationconfig.Resource{ + DB: ptr.To("__mdb_internal_search"), + Collection: ptr.To(""), + }, + Actions: []string{ + "changeStream", "collStats", "dbHash", "dbStats", "find", + "killCursors", "listCollections", "listIndexes", "listSearchIndexes", + // performRawDataOperations is available only on mongod master + // "performRawDataOperations", + "planCacheRead", "cleanupStructuredEncryptionData", + "compactStructuredEncryptionData", "convertToCapped", "createCollection", + "createIndex", "createSearchIndexes", "dropCollection", "dropIndex", + "dropSearchIndex", "insert", "remove", "renameCollectionSameDB", + "update", "updateSearchIndex", + }, + }, + { + Resource: automationconfig.Resource{ + Cluster: true, + }, + Actions: []string{"bypassDefaultMaxTimeMS"}, + }, + }, + AuthenticationRestrictions: nil, + } +} + // OverrideToAutomationConfig turns an automation config override from the resource spec into an automation config // which can be used to merge. func OverrideToAutomationConfig(override mdbv1.AutomationConfigOverride) automationconfig.AutomationConfig { @@ -772,10 +837,9 @@ func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Mo // getMongodConfigModification will merge the additional configuration in the CRD // into the configuration set up by the operator. func getMongodConfigSearchModification(search *searchv1.MongoDBSearch) automationconfig.Modification { - if search == nil { - return func(config *automationconfig.AutomationConfig) { - // do nothing - } + // Condition for skipping add parameter if it is external mongod + if search == nil || search.IsExternalMongoDBSource() { + return automationconfig.NOOP() } searchConfigParameters := search_controller.GetMongodConfigParameters(search) diff --git a/mongodb-community-operator/pkg/mongot/mongot_config.go b/mongodb-community-operator/pkg/mongot/mongot_config.go index 80d4c1421..c8126b173 100644 --- a/mongodb-community-operator/pkg/mongot/mongot_config.go +++ b/mongodb-community-operator/pkg/mongot/mongot_config.go @@ -1,41 +1,82 @@ package mongot -type Config struct { - CommunityPrivatePreview CommunityPrivatePreview `json:"communityPrivatePreview"` +type Modification func(*Config) + +func NOOP() Modification { + return func(config *Config) {} } -// CommunityPrivatePreview structure reflects private preview configuration from mongot: -// https://github.com/10gen/mongot/blob/060ec179af062ac2639678f4a613b8ab02c21597/src/main/java/com/xgen/mongot/config/provider/community/CommunityConfig.java#L100 -// Comments are from the default config file: https://github.com/10gen/mongot/blob/375379e56a580916695a2f53e12fd4a99aa24f0b/deploy/community-resources/config.default.yml#L1-L0 -type CommunityPrivatePreview struct { - // Socket (IPv4/6) address of the sync source mongod - MongodHostAndPort string `json:"mongodHostAndPort"` +func Apply(modifications ...Modification) func(*Config) { + return func(config *Config) { + for _, mod := range modifications { + mod(config) + } + } +} - // Socket (IPv4/6) address on which to listen for wire protocol connections - QueryServerAddress string `json:"queryServerAddress"` +type Config struct { + SyncSource ConfigSyncSource `json:"syncSource"` + Storage ConfigStorage `json:"storage"` + Server ConfigServer `json:"server"` + Metrics ConfigMetrics `json:"metrics"` + HealthCheck ConfigHealthCheck `json:"healthCheck"` + Logging ConfigLogging `json:"logging"` +} - // Keyfile used for mongod -> mongot authentication - KeyFilePath string `json:"keyFilePath"` +type ConfigSyncSource struct { + ReplicaSet ConfigReplicaSet `json:"replicaSet"` +} + +type ConfigReplicaSet struct { + HostAndPort []string `json:"hostAndPort"` + Username string `json:"username"` + PasswordFile string `json:"passwordFile"` + TLS *bool `json:"tls,omitempty"` + ReadPreference *string `json:"readPreference,omitempty"` + AuthSource *string `json:"authSource,omitempty"` +} - // Filesystem path that all mongot data will be stored at +type ConfigStorage struct { DataPath string `json:"dataPath"` +} - // Options for metrics - Metrics Metrics `json:"metrics,omitempty"` +type ConfigServer struct { + Wireproto *ConfigWireproto `json:"wireproto,omitempty"` +} + +type ConfigWireproto struct { + Address string `json:"address"` + Authentication *ConfigAuthentication `json:"authentication,omitempty"` + TLS ConfigTLS `json:"tls"` +} - // Options for logging - Logging Logging `json:"logging,omitempty"` +type ConfigAuthentication struct { + Mode string `json:"mode"` + KeyFile string `json:"keyFile"` } -type Metrics struct { - // Whether to enable the Prometheus metrics endpoint - Enabled bool `json:"enabled"` +type ConfigTLSMode string + +const ( + ConfigTLSModeTLS ConfigTLSMode = "TLS" + ConfigTLSModeDisabled ConfigTLSMode = "Disabled" +) + +type ConfigTLS struct { + Mode ConfigTLSMode `json:"mode"` + CertificateKeyFile *string `json:"certificateKeyFile,omitempty"` +} + +type ConfigMetrics struct { + Enabled bool `json:"enabled"` + Address string `json:"address"` +} - // Socket address (IPv4/6) on which the Prometheus /metrics endpoint will be exposed +type ConfigHealthCheck struct { Address string `json:"address"` } -type Logging struct { - // Log level - Verbosity string `json:"verbosity"` +type ConfigLogging struct { + Verbosity string `json:"verbosity"` + LogPath *string `json:"logPath,omitempty"` } diff --git a/mongodb-community-operator/pkg/tls/tls.go b/mongodb-community-operator/pkg/tls/tls.go new file mode 100644 index 000000000..77c49ed2a --- /dev/null +++ b/mongodb-community-operator/pkg/tls/tls.go @@ -0,0 +1,113 @@ +package tls + +import ( + "context" + "crypto/sha256" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/types" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/secret" +) + +const ( + CAMountPath = "/var/lib/tls/ca/" + OperatorSecretMountPath = "/var/lib/tls/server/" //nolint + + tlsSecretCertName = "tls.crt" + tlsSecretKeyName = "tls.key" + tlsSecretPemName = "tls.pem" +) + +type TLSConfigurableResource interface { + metav1.Object + TLSSecretNamespacedName() types.NamespacedName + TLSOperatorSecretNamespacedName() types.NamespacedName +} + +// ensureTLSSecret will create or update the operator-managed Secret containing +// the concatenated certificate and key from the user-provided Secret. +// Returns the file name of the concatenated certificate and key +func EnsureTLSSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, resource TLSConfigurableResource) (string, error) { + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, resource.TLSSecretNamespacedName()) + if err != nil { + return "", err + } + // Calculate file name from certificate and key + fileName := OperatorSecretFileName(certKey) + + operatorSecret := secret.Builder(). + SetName(resource.TLSOperatorSecretNamespacedName().Name). + SetNamespace(resource.TLSOperatorSecretNamespacedName().Namespace). + SetField(fileName, certKey). + SetOwnerReferences(resource.GetOwnerReferences()). + Build() + + return fileName, secret.CreateOrUpdate(ctx, getUpdateCreator, operatorSecret) +} + +// getCertAndKey will fetch the certificate and key from the user-provided Secret. +func getCertAndKey(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) string { + cert, err := secret.ReadKey(ctx, getter, tlsSecretCertName, secretName) + if err != nil { + return "" + } + + key, err := secret.ReadKey(ctx, getter, tlsSecretKeyName, secretName) + if err != nil { + return "" + } + + return combineCertificateAndKey(cert, key) +} + +// getPem will fetch the pem from the user-provided secret +func getPem(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) string { + pem, err := secret.ReadKey(ctx, getter, tlsSecretPemName, secretName) + if err != nil { + return "" + } + return pem +} + +func combineCertificateAndKey(cert, key string) string { + trimmedCert := strings.TrimRight(cert, "\n") + trimmedKey := strings.TrimRight(key, "\n") + return fmt.Sprintf("%s\n%s", trimmedCert, trimmedKey) +} + +// getPemOrConcatenatedCrtAndKey will get the final PEM to write to the secret. +// This is either the tls.pem entry in the given secret, or the concatenation +// of tls.crt and tls.key +// It performs a basic validation on the entries. +func getPemOrConcatenatedCrtAndKey(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) (string, error) { + certKey := getCertAndKey(ctx, getter, secretName) + pem := getPem(ctx, getter, secretName) + if certKey == "" && pem == "" { + return "", fmt.Errorf(`neither "%s" nor the pair "%s"/"%s" were present in the TLS secret`, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) + } + if certKey == "" { + return pem, nil + } + if pem == "" { + return certKey, nil + } + if certKey != pem { + return "", fmt.Errorf(`if all of "%s", "%s" and "%s" are present in the secret, the entry for "%s" must be equal to the concatenation of "%s" with "%s"`, tlsSecretCertName, tlsSecretKeyName, tlsSecretPemName, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) + } + return certKey, nil +} + +// OperatorSecretFileName calculates the file name to use for the mounted +// certificate-key file. The name is based on the hash of the combined cert and key. +// If the certificate or key changes, the file path changes as well which will trigger +// the agent to perform a restart. +// The user-provided secret is being watched and will trigger a reconciliation +// on changes. This enables the operator to automatically handle cert rotations. +func OperatorSecretFileName(certKey string) string { + hash := sha256.Sum256([]byte(certKey)) + return fmt.Sprintf("%x.pem", hash) +} diff --git a/pkg/telemetry/collector.go b/pkg/telemetry/collector.go index f1d0e8712..1e6a2ed09 100644 --- a/pkg/telemetry/collector.go +++ b/pkg/telemetry/collector.go @@ -21,6 +21,7 @@ import ( mdbmultiv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdbmulti" omv1 "github.com/mongodb/mongodb-kubernetes/api/v1/om" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + userv1 "github.com/mongodb/mongodb-kubernetes/api/v1/user" mcov1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/util/envvar" "github.com/mongodb/mongodb-kubernetes/pkg/images" @@ -364,25 +365,50 @@ func addCommunityEvents(ctx context.Context, operatorClusterClient kubeclient.Cl return events } +func resolveSearchSource(ctx context.Context, operatorClusterClient kubeclient.Client, source *userv1.MongoDBResourceRef) (architecture string, isEnterprise bool, ok bool) { + if source == nil { + return "external", false, true // we cheat and hijack the Architecture field to indicate this Search resource is configured with an external MongoDB source + } + + key := kubeclient.ObjectKey{Namespace: source.Namespace, Name: source.Name} + + mdb := &mdbv1.MongoDB{} + if err := operatorClusterClient.Get(ctx, key, mdb); err == nil { + return string(architectures.GetArchitecture(mdb.Annotations)), true, true + } + + mdbc := &mcov1.MongoDBCommunity{} + if err := operatorClusterClient.Get(ctx, key, mdbc); err == nil { + return "static", false, true // Community is always static + } + + return "", false, false // likely the database resource doesn't exist yet, skip telemetry for this item for now +} + func addSearchEvents(ctx context.Context, operatorClusterClient kubeclient.Client, operatorUUID string, now time.Time) []Event { var events []Event searchList := &searchv1.MongoDBSearchList{} if err := operatorClusterClient.List(ctx, searchList); err != nil { Logger.Warnf("failed to fetch MongoDBSearchList from Kubernetes: %v", err) - } else { - for _, item := range searchList.Items { - properties := DeploymentUsageSnapshotProperties{ - DeploymentUID: string(item.UID), - OperatorID: operatorUUID, - Architecture: string(architectures.Static), // Community Search is always static - IsMultiCluster: false, // Community Search doesn't support multi-cluster - Type: "Search", - IsRunningEnterpriseImage: false, // Community search doesn't run enterprise - } - if event := createEvent(properties, now, Deployments); event != nil { - events = append(events, *event) - } + return nil + } + + for _, item := range searchList.Items { + architecture, isEnterprise, ok := resolveSearchSource(ctx, operatorClusterClient, item.GetMongoDBResourceRef()) + if !ok { // search source doesn't exist yet, don't generate a telemetry event + continue + } + properties := DeploymentUsageSnapshotProperties{ + DeploymentUID: string(item.UID), + OperatorID: operatorUUID, + Architecture: architecture, + IsMultiCluster: false, // Search doesn't support multi-cluster + Type: "Search", + IsRunningEnterpriseImage: isEnterprise, + } + if event := createEvent(properties, now, Deployments); event != nil { + events = append(events, *event) } } return events diff --git a/pkg/telemetry/collector_test.go b/pkg/telemetry/collector_test.go index 9b3a8c638..f0ca25986 100644 --- a/pkg/telemetry/collector_test.go +++ b/pkg/telemetry/collector_test.go @@ -2,7 +2,9 @@ package telemetry import ( "context" + "encoding/json" "errors" + "reflect" "runtime" "testing" "time" @@ -21,6 +23,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/api/v1/mdbmulti" omv1 "github.com/mongodb/mongodb-kubernetes/api/v1/om" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" + userv1 "github.com/mongodb/mongodb-kubernetes/api/v1/user" "github.com/mongodb/mongodb-kubernetes/controllers/operator/mock" mcov1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" mockClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" @@ -1157,12 +1160,17 @@ func findEventWithDeploymentUID(events []Event, deploymentUID string) *Event { type MockClient struct { client.Client MockList func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error + MockGet func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error } func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { return m.MockList(ctx, list, opts...) } +func (m *MockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return m.MockGet(ctx, key, obj, opts...) +} + func TestAddCommunityEvents(t *testing.T) { operatorUUID := "test-operator-uuid" @@ -1261,55 +1269,67 @@ func TestAddCommunityEvents(t *testing.T) { func TestAddSearchEvents(t *testing.T) { operatorUUID := "test-operator-uuid" - now := time.Now() + mdbStatic := &mdbv1.MongoDB{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "mdb-static", Annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.Static)}}} + mdbNonStatic := &mdbv1.MongoDB{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "mdb-nonstatic", Annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.NonStatic)}}} + community := &mcov1.MongoDBCommunity{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "community-db"}} + testCases := []struct { - name string - resources searchv1.MongoDBSearchList - events []DeploymentUsageSnapshotProperties + name string + searchItems []searchv1.MongoDBSearch + events []DeploymentUsageSnapshotProperties + sources map[reflect.Type][]client.Object }{ { - name: "With resources", - resources: searchv1.MongoDBSearchList{ - Items: []searchv1.MongoDBSearch{ - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID("search-1"), - Name: "test-search-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID("search-2"), - Name: "test-search-2", - }, - }, - }, + name: "External source", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-external"), Name: "search-external", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{ExternalMongoDBSource: &searchv1.ExternalMongoDBSource{}}}}, + }, + events: []DeploymentUsageSnapshotProperties{{ + DeploymentUID: "search-external", + OperatorID: operatorUUID, + Architecture: "external", + IsMultiCluster: false, + Type: "Search", + IsRunningEnterpriseImage: false, + }}, + }, + { + name: "Enterprise static and non-static", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-static"), Name: "search-static", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "mdb-static"}}}}, + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-nonstatic"), Name: "search-nonstatic", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "mdb-nonstatic"}}}}, }, events: []DeploymentUsageSnapshotProperties{ - { - DeploymentUID: "search-1", - OperatorID: operatorUUID, - Architecture: string(architectures.Static), - IsMultiCluster: false, - Type: "Search", - IsRunningEnterpriseImage: false, - }, - { - DeploymentUID: "search-2", - OperatorID: operatorUUID, - Architecture: string(architectures.Static), - IsMultiCluster: false, - Type: "Search", - IsRunningEnterpriseImage: false, - }, + {DeploymentUID: "search-static", OperatorID: operatorUUID, Architecture: string(architectures.Static), IsMultiCluster: false, Type: "Search", IsRunningEnterpriseImage: true}, + {DeploymentUID: "search-nonstatic", OperatorID: operatorUUID, Architecture: string(architectures.NonStatic), IsMultiCluster: false, Type: "Search", IsRunningEnterpriseImage: true}, + }, + sources: map[reflect.Type][]client.Object{ + reflect.TypeOf(&mdbv1.MongoDB{}): {mdbStatic, mdbNonStatic}, }, }, { - name: "With no resources", - resources: searchv1.MongoDBSearchList{}, - events: []DeploymentUsageSnapshotProperties{}, + name: "Community source", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-community"), Name: "search-community", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "community-db"}}}}, + }, + events: []DeploymentUsageSnapshotProperties{{DeploymentUID: "search-community", OperatorID: operatorUUID, Architecture: "static", IsMultiCluster: false, Type: "Search", IsRunningEnterpriseImage: false}}, + sources: map[reflect.Type][]client.Object{ + reflect.TypeOf(&mcov1.MongoDBCommunity{}): {community}, + }, + }, + { + name: "Missing underlying resource (skipped)", + searchItems: []searchv1.MongoDBSearch{ + {ObjectMeta: metav1.ObjectMeta{UID: types.UID("search-missing"), Name: "search-missing", Namespace: "default"}, Spec: searchv1.MongoDBSearchSpec{Source: &searchv1.MongoDBSource{MongoDBResourceRef: &userv1.MongoDBResourceRef{Name: "does-not-exist"}}}}, + }, + events: []DeploymentUsageSnapshotProperties{}, + }, + { + name: "No search resources", + searchItems: []searchv1.MongoDBSearch{}, + events: []DeploymentUsageSnapshotProperties{}, }, } @@ -1318,10 +1338,23 @@ func TestAddSearchEvents(t *testing.T) { mc := &MockClient{ MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { if l, ok := list.(*searchv1.MongoDBSearchList); ok { - *l = tc.resources + *l = searchv1.MongoDBSearchList{Items: tc.searchItems} } return nil }, + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + objects := tc.sources[reflect.TypeOf(obj)] + for _, o := range objects { + if o.GetName() == key.Name && o.GetNamespace() == key.Namespace { + // copy the arranged object into the obj pointer like controller-runtime's pkg/client/fake does + bytes, _ := json.Marshal(o) + json.Unmarshal(bytes, obj) + return nil + } + } + + return errors.New("not found") + }, } events := addSearchEvents(context.Background(), mc, operatorUUID, now) diff --git a/public/crds.yaml b/public/crds.yaml index 0bdd5e315..4afc63e97 100644 --- a/public/crds.yaml +++ b/public/crds.yaml @@ -4071,6 +4071,8 @@ spec: spec: properties: persistence: + description: Configure MongoDB Search's persistent volume. If not + defined, the operator will request 10GB of storage. properties: multiple: properties: @@ -4117,7 +4119,8 @@ spec: type: object type: object resourceRequirements: - description: ResourceRequirements describes the compute resource requirements. + description: Configure resource requests and limits for the MongoDB + Search pods. properties: claims: description: |- @@ -4175,8 +4178,88 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + security: + description: Configure security settings of the MongoDB Search server + that MongoDB database is connecting to when performing search queries. + properties: + tls: + properties: + certificateKeySecretRef: + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object source: + description: MongoDB database connection details from which MongoDB + Search will synchronize data to build indexes. properties: + external: + properties: + hostAndPorts: + items: + type: string + type: array + keyFileSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + tls: + properties: + ca: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + type: boolean + required: + - enabled + type: object + type: object mongodbResourceRef: properties: name: @@ -4186,11 +4269,26 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. + StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations, + which aren't exposed as fields in the MongoDBSearch.spec. properties: metadata: description: StatefulSetMetadataWrapper is a wrapper around Labels @@ -4212,6 +4310,9 @@ spec: - spec type: object version: + description: Optional version of MongoDB Search component (mongot). + If not set, then the operator will set the most appropriate version + of MongoDB Search. type: string type: object status: diff --git a/scripts/dev/contexts/e2e_mdb_community b/scripts/dev/contexts/e2e_mdb_community index 4f096c2d7..b38563179 100644 --- a/scripts/dev/contexts/e2e_mdb_community +++ b/scripts/dev/contexts/e2e_mdb_community @@ -10,3 +10,5 @@ source "${script_dir}/variables/mongodb_latest" # This variable is needed otherwise the `fetch_om_information.sh` script is called and fails the test export OM_EXTERNALLY_CONFIGURED="true" + +source "${script_dir}/variables/mongodb_search_dev" diff --git a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa index e1f13453c..886ecb8be 100644 --- a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa +++ b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa @@ -16,3 +16,5 @@ export CUSTOM_OM_VERSION export CUSTOM_MDB_VERSION=6.0.5 export CUSTOM_MDB_PREV_VERSION=5.0.7 + +source "${script_dir}/variables/mongodb_search_dev" diff --git a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa index c007157d7..f4fe1ebdb 100644 --- a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa +++ b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa @@ -17,3 +17,5 @@ export CUSTOM_OM_VERSION export CUSTOM_MDB_PREV_VERSION=6.0.16 export CUSTOM_MDB_VERSION=7.0.5 + +source "${script_dir}/variables/mongodb_search_dev" diff --git a/scripts/dev/contexts/evg-private-context b/scripts/dev/contexts/evg-private-context index ed92f125c..1eb2ef0f5 100644 --- a/scripts/dev/contexts/evg-private-context +++ b/scripts/dev/contexts/evg-private-context @@ -109,8 +109,8 @@ export CODE_SNIPPETS_COMMIT_OUTPUT=${code_snippets_commit_output:-"false"} export READINESS_PROBE_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-readinessprobe:${version_id}" export VERSION_UPGRADE_HOOK_IMAGE="${BASE_REPO_URL}/mongodb-kubernetes-operator-version-upgrade-post-start-hook:${version_id}" -# TODO to be removed at public preview stage of community-search -export COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON="${community_private_preview_pullsecret_dockerconfigjson}" +# shellcheck disable=SC2154 +export PRERELEASE_PULLSECRET_DOCKERCONFIGJSON="${community_private_preview_pullsecret_dockerconfigjson}" export cognito_user_pool_id="${cognito_user_pool_id}" export cognito_workload_federation_client_id="${cognito_workload_federation_client_id}" diff --git a/scripts/dev/contexts/root-context b/scripts/dev/contexts/root-context index 24570ac93..542190d4e 100644 --- a/scripts/dev/contexts/root-context +++ b/scripts/dev/contexts/root-context @@ -6,10 +6,12 @@ script_name=$(readlink -f "${BASH_SOURCE[0]}") script_dir=$(dirname "${script_name}") source "${script_dir}/private-context" -export PROJECT_DIR="${PWD}" +PROJECT_DIR="$(realpath "${script_dir}/../../..")" +export PROJECT_DIR export IMAGE_TYPE=ubi export UBI_IMAGE_WITHOUT_SUFFIX=true export WATCH_NAMESPACE=${WATCH_NAMESPACE:-${NAMESPACE}} +export OPERATOR_NAME="mongodb-kubernetes-operator" # # changing variables below should not be necessary @@ -35,7 +37,7 @@ fi export OPERATOR_ENV=${OPERATOR_ENV:-"dev"} -AGENT_VERSION="$(jq -r '.agentVersion' release.json)" +AGENT_VERSION="$(jq -r '.agentVersion' "${PROJECT_DIR}/release.json")" export AGENT_VERSION export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" @@ -111,7 +113,7 @@ export MDB_COMMUNITY_REPO_URL=quay.io/mongodb export MDB_COMMUNITY_AGENT_IMAGE=${AGENT_IMAGE} export MDB_COMMUNITY_IMAGE_TYPE=ubi8 -MDB_SEARCH_COMMUNITY_VERSION="$(jq -r '.search.community.version' release.json)" +MDB_SEARCH_COMMUNITY_VERSION="$(jq -r '.search.community.version' "${PROJECT_DIR}/release.json")" export MDB_SEARCH_COMMUNITY_VERSION export MDB_SEARCH_COMMUNITY_NAME="mongodb-search-community" diff --git a/scripts/dev/contexts/variables/mongodb_search_dev b/scripts/dev/contexts/variables/mongodb_search_dev new file mode 100644 index 000000000..1eda41398 --- /dev/null +++ b/scripts/dev/contexts/variables/mongodb_search_dev @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# Temporary development images built from mongot master +export MDB_SEARCH_COMMUNITY_VERSION="1.53.0-78-g227f2593f" # Sep 2nd mongot master +export MDB_SEARCH_COMMUNITY_NAME="mongot/community" +export MDB_SEARCH_COMMUNITY_REPO_URL="268558157000.dkr.ecr.eu-west-1.amazonaws.com" diff --git a/scripts/funcs/operator_deployment b/scripts/funcs/operator_deployment index 60f44ff0d..77075a39a 100644 --- a/scripts/funcs/operator_deployment +++ b/scripts/funcs/operator_deployment @@ -33,6 +33,9 @@ get_operator_helm_values() { "operator.telemetry.send.enabled=${MDB_OPERATOR_TELEMETRY_SEND_ENABLED:-false}" # lets collect and save in the configmap as frequently as we can "operator.telemetry.collection.frequency=${MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY:-1m}" + "search.community.repo=${MDB_SEARCH_COMMUNITY_REPO_URL}" + "search.community.name=${MDB_SEARCH_COMMUNITY_NAME}" + "search.community.version=${MDB_SEARCH_COMMUNITY_VERSION}" "community.registry.agent=${AGENT_BASE_REGISTRY:-${REGISTRY}}" "search.community.repo=${MDB_SEARCH_COMMUNITY_REPO_URL}" "search.community.name=${MDB_SEARCH_COMMUNITY_NAME}" From ba7910669c1d85398b6b5831df8b3aea3ee8b4ce Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Tue, 9 Sep 2025 21:41:46 +0300 Subject: [PATCH 05/17] Apply suggestions from code review Co-authored-by: Vivek Singh --- api/v1/search/mongodbsearch_types.go | 3 ++- controllers/search_controller/external_search_source.go | 2 +- .../tests/search/fixtures/mongodbuser-mdb-admin.yaml | 3 ++- .../search/fixtures/mongodbuser-search-sync-source-user.yaml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index 5a8037b51..c20884f8d 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -63,8 +63,9 @@ type ExternalMongoDBSource struct { HostAndPorts []string `json:"hostAndPorts,omitempty"` // mongod keyfile used to connect to the external MongoDB deployment KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyFileSecretRef,omitempty"` + // TLS configuration for the external MongoDB deployment // +optional - TLS *ExternalMongodTLS `json:"tls,omitempty"` // TLS configuration for the external MongoDB deployment + TLS *ExternalMongodTLS `json:"tls,omitempty"` } type ExternalMongodTLS struct { diff --git a/controllers/search_controller/external_search_source.go b/controllers/search_controller/external_search_source.go index 5a408246e..d15f49799 100644 --- a/controllers/search_controller/external_search_source.go +++ b/controllers/search_controller/external_search_source.go @@ -12,7 +12,7 @@ func NewExternalSearchSource(namespace string, spec *searchv1.ExternalMongoDBSou return &externalSearchResource{namespace: namespace, spec: spec} } -// externalSearchResource implements SearchSourceDBResource for deployments managed outside the operator. +// externalSearchResource implements SearchSourceDBResource for deployments managed outside the Kubernetes cluster. type externalSearchResource struct { namespace string spec *searchv1.ExternalMongoDBSource diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml index 0b3fe4c77..950f2e5a6 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-mdb-admin.yaml @@ -13,4 +13,5 @@ spec: key: password roles: - name: root - db: admin \ No newline at end of file + db: admin + \ No newline at end of file diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml index cd1eab1a5..d6ce56d8f 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/mongodbuser-search-sync-source-user.yaml @@ -1,4 +1,4 @@ -# user used by MongoDB Search to connect to MongoDB database to synchronize data from +# user used by MongoDB Search to connect to MongoDB database to synchronize data from. # For MongoDB <8.2, the operator will be creating the searchCoordinator custom role automatically # From MongoDB 8.2, searchCoordinator role will be a built-in role. apiVersion: mongodb.com/v1 From 1aa6e3e8ef37f27fedc7c897aef808de8f416d79 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 01:28:26 +0300 Subject: [PATCH 06/17] regenerate CRDs with the latest controller-gen --- .../mongodb.com_clustermongodbroles.yaml | 2 +- config/crd/bases/mongodb.com_mongodb.yaml | 48 +++++++- .../mongodb.com_mongodbmulticluster.yaml | 48 +++++++- .../crd/bases/mongodb.com_mongodbsearch.yaml | 90 +++++++++++++- .../crd/bases/mongodb.com_mongodbusers.yaml | 2 +- config/crd/bases/mongodb.com_opsmanagers.yaml | 112 ++++++++++++++++-- ...ommunity.mongodb.com_mongodbcommunity.yaml | 54 ++++++++- .../crds/mongodb.com_mongodbsearch.yaml | 11 +- public/crds.yaml | 11 +- 9 files changed, 334 insertions(+), 44 deletions(-) diff --git a/config/crd/bases/mongodb.com_clustermongodbroles.yaml b/config/crd/bases/mongodb.com_clustermongodbroles.yaml index 9241b7dad..3d583bcfd 100644 --- a/config/crd/bases/mongodb.com_clustermongodbroles.yaml +++ b/config/crd/bases/mongodb.com_clustermongodbroles.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: clustermongodbroles.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_mongodb.yaml b/config/crd/bases/mongodb.com_mongodb.yaml index 2a7076877..d421d8837 100644 --- a/config/crd/bases/mongodb.com_mongodb.yaml +++ b/config/crd/bases/mongodb.com_mongodb.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodb.mongodb.com spec: group: mongodb.com @@ -1410,7 +1410,29 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic automationUserName: type: string clientCertificateSecretRef: @@ -1445,9 +1467,29 @@ spec: bindQueryUser: type: string caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml index 01fe3f2e6..3f1fa05c9 100644 --- a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml +++ b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodbmulticluster.mongodb.com spec: group: mongodb.com @@ -670,7 +670,29 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic automationUserName: type: string clientCertificateSecretRef: @@ -705,9 +727,29 @@ spec: bindQueryUser: type: string caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbsearch.yaml b/config/crd/bases/mongodb.com_mongodbsearch.yaml index 7c53c195c..08b4fa173 100644 --- a/config/crd/bases/mongodb.com_mongodbsearch.yaml +++ b/config/crd/bases/mongodb.com_mongodbsearch.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodbsearch.mongodb.com spec: group: mongodb.com @@ -97,9 +97,65 @@ spec: type: object type: object resourceRequirements: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ResourceRequirements' description: Configure resource requests and limits for the MongoDB Search pods. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object security: description: Configure security settings of the MongoDB Search server that MongoDB database is connecting to when performing search queries. @@ -107,13 +163,24 @@ spec: tls: properties: certificateKeySecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic enabled: type: boolean required: @@ -142,9 +209,24 @@ spec: - name type: object tls: + description: TLS configuration for the external MongoDB deployment properties: ca: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic enabled: type: boolean required: diff --git a/config/crd/bases/mongodb.com_mongodbusers.yaml b/config/crd/bases/mongodb.com_mongodbusers.yaml index a81f0d449..89713ce7f 100644 --- a/config/crd/bases/mongodb.com_mongodbusers.yaml +++ b/config/crd/bases/mongodb.com_mongodbusers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodbusers.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_opsmanagers.yaml b/config/crd/bases/mongodb.com_opsmanagers.yaml index c830b9a24..3ace001da 100644 --- a/config/crd/bases/mongodb.com_opsmanagers.yaml +++ b/config/crd/bases/mongodb.com_opsmanagers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: opsmanagers.mongodb.com spec: group: mongodb.com @@ -730,7 +730,30 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic automationUserName: type: string clientCertificateSecretRef: @@ -766,9 +789,29 @@ spec: bindQueryUser: type: string caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic servers: items: type: string @@ -1193,7 +1236,29 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: array irsaEnabled: description: |- @@ -1263,7 +1328,29 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: array irsaEnabled: description: |- @@ -1408,6 +1495,8 @@ spec: required: - spec type: object + required: + - members type: object clusterDomain: description: Cluster domain to override the default *.svc.cluster.local @@ -1447,13 +1536,13 @@ spec: Service when creating a ClusterIP type Service type: string externalTrafficPolicy: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local + type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. @@ -1464,12 +1553,12 @@ spec: format: int32 type: integer type: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP + type: string required: - type type: object @@ -1511,6 +1600,7 @@ spec: - spec type: object required: + - clusterName - members type: object type: array @@ -1536,13 +1626,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local + type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1552,12 +1642,12 @@ spec: format: int32 type: integer type: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP + type: string required: - type type: object @@ -1577,13 +1667,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local + type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1593,12 +1683,12 @@ spec: format: int32 type: integer type: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP + type: string required: - type type: object diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index a0004e22e..36d5c892d 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 service.binding: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret service.binding/connectionString: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv service.binding/password: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password @@ -330,13 +330,24 @@ spec: authentication: properties: agentCertificateSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- AgentCertificateSecret is a reference to a Secret containing the certificate and the key for the automation agent The secret needs to have available: - certificate under key: "tls.crt" - private key under key: "tls.key" If additionally, tls.pem is present, then it needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic agentMode: description: AgentMode contains the authentication mode used by the automation agent. @@ -455,24 +466,57 @@ spec: communication properties: caCertificateSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" This field is ignored when CaCertificateSecretRef is configured + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic certificateKeySecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic enabled: type: boolean optional: diff --git a/helm_chart/crds/mongodb.com_mongodbsearch.yaml b/helm_chart/crds/mongodb.com_mongodbsearch.yaml index 38dc38f2a..08b4fa173 100644 --- a/helm_chart/crds/mongodb.com_mongodbsearch.yaml +++ b/helm_chart/crds/mongodb.com_mongodbsearch.yaml @@ -177,9 +177,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -200,10 +198,8 @@ spec: type: string type: array keyFileSecretRef: - description: |- - SecretKeyRef is a reference to a value in a given secret in the same - namespace. Based on: - https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + description: mongod keyfile used to connect to the external + MongoDB deployment properties: key: type: string @@ -213,6 +209,7 @@ spec: - name type: object tls: + description: TLS configuration for the external MongoDB deployment properties: ca: description: |- @@ -226,9 +223,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic diff --git a/public/crds.yaml b/public/crds.yaml index 4afc63e97..42ca6a290 100644 --- a/public/crds.yaml +++ b/public/crds.yaml @@ -4199,9 +4199,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -4222,10 +4220,8 @@ spec: type: string type: array keyFileSecretRef: - description: |- - SecretKeyRef is a reference to a value in a given secret in the same - namespace. Based on: - https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + description: mongod keyfile used to connect to the external + MongoDB deployment properties: key: type: string @@ -4235,6 +4231,7 @@ spec: - name type: object tls: + description: TLS configuration for the external MongoDB deployment properties: ca: description: |- @@ -4248,9 +4245,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic From 94df6129495a1371bf4a8cadafd64e61cb88bec2 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 01:46:49 +0300 Subject: [PATCH 07/17] replace most literals with constants --- api/v1/search/mongodbsearch_types.go | 14 ++++-- .../operator/mongodbreplicaset_controller.go | 4 +- .../enterprise_search_source.go | 2 +- .../mongodbsearch_reconcile_helper.go | 10 ++-- .../search_controller/search_construction.go | 48 ++++++++++++------- 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index c20884f8d..d11980acb 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -156,14 +156,20 @@ func (s *MongoDBSearch) MongotConfigConfigMapNamespacedName() types.NamespacedNa } func (s *MongoDBSearch) SourceUserPasswordSecretRef() *userv1.SecretKeyRef { + var syncUserPasswordSecretKey *userv1.SecretKeyRef if s.Spec.Source != nil && s.Spec.Source.PasswordSecretRef != nil { - return s.Spec.Source.PasswordSecretRef + syncUserPasswordSecretKey = s.Spec.Source.PasswordSecretRef + } else { + syncUserPasswordSecretKey = &userv1.SecretKeyRef{ + Name: fmt.Sprintf("%s-%s-password", s.Name, s.SourceUsername()), + } } - return &userv1.SecretKeyRef{ - Name: fmt.Sprintf("%s-%s-password", s.Name, MongotDefaultSyncSourceUsername), - Key: "password", + if syncUserPasswordSecretKey.Key == "" { + syncUserPasswordSecretKey.Key = "password" } + + return syncUserPasswordSecretKey } func (s *MongoDBSearch) SourceUsername() string { diff --git a/controllers/operator/mongodbreplicaset_controller.go b/controllers/operator/mongodbreplicaset_controller.go index fad1ec6b4..621ec6dd2 100644 --- a/controllers/operator/mongodbreplicaset_controller.go +++ b/controllers/operator/mongodbreplicaset_controller.go @@ -677,12 +677,12 @@ func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, r func (r *ReconcileMongoDbReplicaSet) mirrorKeyfileIntoSecretForMongot(ctx context.Context, d om.Deployment, rs *mdbv1.MongoDB, log *zap.SugaredLogger) error { keyfileContents := maputil.ReadMapValueAsString(d, "auth", "key") - keyfileSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-keyfile", rs.Name), Namespace: rs.Namespace}} + keyfileSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s", rs.Name, search_controller.MongotKeyfileFilename), Namespace: rs.Namespace}} log.Infof("Mirroring the replicaset %s's keyfile into the secret %s", rs.ObjectKey(), kube.ObjectKeyFromApiObject(keyfileSecret)) _, err := controllerutil.CreateOrUpdate(ctx, r.client, keyfileSecret, func() error { - keyfileSecret.StringData = map[string]string{"keyfile": keyfileContents} + keyfileSecret.StringData = map[string]string{search_controller.MongotKeyfileFilename: keyfileContents} return controllerutil.SetOwnerReference(rs, keyfileSecret, r.client.Scheme()) }) if err != nil { diff --git a/controllers/search_controller/enterprise_search_source.go b/controllers/search_controller/enterprise_search_source.go index c1256fbbb..45e2d0f2d 100644 --- a/controllers/search_controller/enterprise_search_source.go +++ b/controllers/search_controller/enterprise_search_source.go @@ -47,7 +47,7 @@ func (r EnterpriseResourceSearchSource) TLSConfig() *TLSSourceConfig { } func (r EnterpriseResourceSearchSource) KeyfileSecretName() string { - return fmt.Sprintf("%s-keyfile", r.Name) + return fmt.Sprintf("%s-%s", r.Name, MongotKeyfileFilename) } func (r EnterpriseResourceSearchSource) Validate() error { diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/search_controller/mongodbsearch_reconcile_helper.go index 4b384a707..635ebe7fc 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper.go @@ -150,7 +150,7 @@ func (r *MongoDBSearchReconcileHelper) ensureSourceKeyfile(ctx context.Context, // make sure mongot pods get restarted if the keyfile changes statefulset.WithPodSpecTemplate(podtemplatespec.WithAnnotations( map[string]string{ - "keyfileHash": hashBytes(keyfileSecret.Data["keyfile"]), + "keyfileHash": hashBytes(keyfileSecret.Data[MongotKeyfileFilename]), }, )), ), nil @@ -211,7 +211,7 @@ func (r *MongoDBSearchReconcileHelper) ensureMongotConfig(ctx context.Context, l op, err := controllerutil.CreateOrUpdate(ctx, r.client, cm, func() error { resourceVersion := cm.ResourceVersion - cm.Data["config.yml"] = string(configData) + cm.Data[MongotConfigFilename] = string(configData) cm.ResourceVersion = resourceVersion @@ -355,21 +355,21 @@ func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResourc ReplicaSet: mongot.ConfigReplicaSet{ HostAndPort: hostAndPorts, Username: search.SourceUsername(), - PasswordFile: "/tmp/sourceUserPassword", + PasswordFile: TempSourceUserPasswordPath, TLS: ptr.To(false), ReadPreference: ptr.To("secondaryPreferred"), AuthSource: ptr.To("admin"), }, } config.Storage = mongot.ConfigStorage{ - DataPath: "/mongot/data/config.yml", + DataPath: MongotDataPath, } config.Server = mongot.ConfigServer{ Wireproto: &mongot.ConfigWireproto{ Address: "0.0.0.0:27027", Authentication: &mongot.ConfigAuthentication{ Mode: "keyfile", - KeyFile: "/tmp/keyfile", + KeyFile: TempKeyfilePath, }, }, } diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index ffaa72358..1d1e1ca5c 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -1,6 +1,8 @@ package search_controller import ( + "fmt" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -19,9 +21,18 @@ import ( ) const ( - MongotContainerName = "mongot" - SearchLivenessProbePath = "/health" - SearchReadinessProbePath = "/health" // Todo: Update this when search GA is available + MongotContainerName = "mongot" + MongotConfigFilename = "config.yml" + MongotConfigPath = "/mongot/" + MongotConfigFilename + MongotDataPath = "/mongot/data" + MongotKeyfileFilename = "keyfile" + MongotKeyfilePath = "/mongot/" + MongotKeyfileFilename + tempVolumePath = "/tmp" + TempKeyfilePath = tempVolumePath + "/" + MongotKeyfileFilename + MongotSourceUserPasswordPath = "/mongot/sourceUserPassword" // #nosec G101 -- This is not a hardcoded password, just a path to a file containing the password + TempSourceUserPasswordPath = tempVolumePath + "/" + "sourceUserPassword" + SearchLivenessProbePath = "/health" + SearchReadinessProbePath = "/health" // Todo: Update this when search GA is available ) // SearchSourceDBResource is an object wrapping a MongoDBCommunity object @@ -48,23 +59,24 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso } tmpVolume := statefulset.CreateVolumeFromEmptyDir("tmp") - tmpVolumeMount := statefulset.CreateVolumeMount(tmpVolume.Name, "/tmp", statefulset.WithReadOnly(false)) + tmpVolumeMount := statefulset.CreateVolumeMount(tmpVolume.Name, tempVolumePath, statefulset.WithReadOnly(false)) dataVolumeName := "data" keyfileVolumeName := "keyfile" sourceUserPasswordVolumeName := "password" mongotConfigVolumeName := "config" - pvcVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/mongot/data", statefulset.WithSubPath("data")) + pvcVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, MongotDataPath, statefulset.WithSubPath("data")) keyfileVolume := statefulset.CreateVolumeFromSecret(keyfileVolumeName, sourceDBResource.KeyfileSecretName()) - keyfileVolumeMount := statefulset.CreateVolumeMount(keyfileVolumeName, "/mongot/keyfile", statefulset.WithReadOnly(true)) + keyfileVolumeMount := statefulset.CreateVolumeMount(keyfileVolumeName, MongotKeyfilePath, statefulset.WithReadOnly(true), statefulset.WithSubPath(MongotKeyfileFilename)) - sourceUserPasswordVolume := statefulset.CreateVolumeFromSecret(sourceUserPasswordVolumeName, mdbSearch.SourceUserPasswordSecretRef().Name) - sourceUserPasswordVolumeMount := statefulset.CreateVolumeMount(sourceUserPasswordVolumeName, "/mongot/sourceUserPassword", statefulset.WithReadOnly(true)) + sourceUserPasswordSecretKey := mdbSearch.SourceUserPasswordSecretRef() + sourceUserPasswordVolume := statefulset.CreateVolumeFromSecret(sourceUserPasswordVolumeName, sourceUserPasswordSecretKey.Name) + sourceUserPasswordVolumeMount := statefulset.CreateVolumeMount(sourceUserPasswordVolumeName, MongotSourceUserPasswordPath, statefulset.WithReadOnly(true), statefulset.WithSubPath(sourceUserPasswordSecretKey.Key)) mongotConfigVolume := statefulset.CreateVolumeFromConfigMap(mongotConfigVolumeName, mdbSearch.MongotConfigConfigMapNamespacedName().Name) - mongotConfigVolumeMount := statefulset.CreateVolumeMount(mongotConfigVolumeName, "/mongot/config", statefulset.WithReadOnly(true)) + mongotConfigVolumeMount := statefulset.CreateVolumeMount(mongotConfigVolumeName, MongotConfigPath, statefulset.WithReadOnly(true), statefulset.WithSubPath(MongotConfigFilename)) var persistenceConfig *common.PersistenceConfig if mdbSearch.Spec.Persistence != nil && mdbSearch.Spec.Persistence.SingleConfig != nil { @@ -136,17 +148,17 @@ func mongodbSearchContainer(mdbSearch *searchv1.MongoDBSearch, volumeMounts []co container.WithCommand([]string{"sh"}), container.WithArgs([]string{ "-c", - ` -cp /mongot/keyfile/keyfile /tmp/keyfile -chown 2000:2000 /tmp/keyfile -chmod 0600 /tmp/keyfile + fmt.Sprintf(` +cp %[1]s %[2]s +chown 2000:2000 %[2]s +chmod 0600 %[2]s -cp /mongot/sourceUserPassword/password /tmp/sourceUserPassword -chown 2000:2000 /tmp/sourceUserPassword -chmod 0600 /tmp/sourceUserPassword +cp %[3]s %[4]s +chown 2000:2000 %[4]s +chmod 0600 %[4]s -/mongot-community/mongot --config /mongot/config/config.yml -`, +/mongot-community/mongot --config /mongot/config.yml +`, MongotKeyfilePath, TempKeyfilePath, MongotSourceUserPasswordPath, TempSourceUserPasswordPath), }), containerSecurityContext, ) From 84a27c864693716aeef74c84078df7bafde95df3 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 02:17:23 +0300 Subject: [PATCH 08/17] rename search_controller to searchcontroller --- .../operator/mongodbreplicaset_controller.go | 14 ++++++------- .../operator/mongodbsearch_controller.go | 20 +++++++++---------- .../operator/mongodbsearch_controller_test.go | 12 +++++------ .../community_search_source.go | 2 +- .../community_search_source_test.go | 2 +- .../enterprise_search_source.go | 2 +- .../enterprise_search_source_test.go | 2 +- .../external_search_source.go | 2 +- .../mongodbsearch_reconcile_helper.go | 2 +- .../mongodbsearch_reconcile_helper_test.go | 2 +- .../search_construction.go | 2 +- main.go | 4 ++-- .../controllers/replica_set_controller.go | 8 ++++---- 13 files changed, 37 insertions(+), 37 deletions(-) rename controllers/{search_controller => searchcontroller}/community_search_source.go (99%) rename controllers/{search_controller => searchcontroller}/community_search_source_test.go (99%) rename controllers/{search_controller => searchcontroller}/enterprise_search_source.go (99%) rename controllers/{search_controller => searchcontroller}/enterprise_search_source_test.go (99%) rename controllers/{search_controller => searchcontroller}/external_search_source.go (98%) rename controllers/{search_controller => searchcontroller}/mongodbsearch_reconcile_helper.go (99%) rename controllers/{search_controller => searchcontroller}/mongodbsearch_reconcile_helper_test.go (99%) rename controllers/{search_controller => searchcontroller}/search_construction.go (99%) diff --git a/controllers/operator/mongodbreplicaset_controller.go b/controllers/operator/mongodbreplicaset_controller.go index 621ec6dd2..66f392250 100644 --- a/controllers/operator/mongodbreplicaset_controller.go +++ b/controllers/operator/mongodbreplicaset_controller.go @@ -44,7 +44,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/controllers/operator/recovery" "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" - "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" + "github.com/mongodb/mongodb-kubernetes/controllers/searchcontroller" mcoConstruct "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/annotations" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/configmap" @@ -657,7 +657,7 @@ func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, r if rs.Spec.AdditionalMongodConfig == nil { rs.Spec.AdditionalMongodConfig = mdbv1.NewEmptyAdditionalMongodConfig() } - searchMongodConfig := search_controller.GetMongodConfigParameters(search) + searchMongodConfig := searchcontroller.GetMongodConfigParameters(search) rs.Spec.AdditionalMongodConfig.AddOption("setParameter", searchMongodConfig["setParameter"]) mdbVersion, err := semver.ParseTolerant(rs.Spec.Version) @@ -669,7 +669,7 @@ func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, r if rs.Spec.Security == nil { rs.Spec.Security = &mdbv1.Security{} } - rs.Spec.Security.Roles = append(rs.Spec.Security.Roles, search_controller.SearchCoordinatorRole()) + rs.Spec.Security.Roles = append(rs.Spec.Security.Roles, searchcontroller.SearchCoordinatorRole()) } return true @@ -677,12 +677,12 @@ func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, r func (r *ReconcileMongoDbReplicaSet) mirrorKeyfileIntoSecretForMongot(ctx context.Context, d om.Deployment, rs *mdbv1.MongoDB, log *zap.SugaredLogger) error { keyfileContents := maputil.ReadMapValueAsString(d, "auth", "key") - keyfileSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s", rs.Name, search_controller.MongotKeyfileFilename), Namespace: rs.Namespace}} + keyfileSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s", rs.Name, searchcontroller.MongotKeyfileFilename), Namespace: rs.Namespace}} log.Infof("Mirroring the replicaset %s's keyfile into the secret %s", rs.ObjectKey(), kube.ObjectKeyFromApiObject(keyfileSecret)) _, err := controllerutil.CreateOrUpdate(ctx, r.client, keyfileSecret, func() error { - keyfileSecret.StringData = map[string]string{search_controller.MongotKeyfileFilename: keyfileContents} + keyfileSecret.StringData = map[string]string{searchcontroller.MongotKeyfileFilename: keyfileContents} return controllerutil.SetOwnerReference(rs, keyfileSecret, r.client.Scheme()) }) if err != nil { @@ -696,7 +696,7 @@ func (r *ReconcileMongoDbReplicaSet) lookupCorrespondingSearchResource(ctx conte var search *searchv1.MongoDBSearch searchList := &searchv1.MongoDBSearchList{} if err := r.client.List(ctx, searchList, &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(search_controller.MongoDBSearchIndexFieldName, rs.GetNamespace()+"/"+rs.GetName()), + FieldSelector: fields.OneTermEqualSelector(searchcontroller.MongoDBSearchIndexFieldName, rs.GetNamespace()+"/"+rs.GetName()), }); err != nil { log.Debugf("Failed to list MongoDBSearch resources: %v", err) } @@ -704,7 +704,7 @@ func (r *ReconcileMongoDbReplicaSet) lookupCorrespondingSearchResource(ctx conte // and that this resource passes search validations. If either fails, proceed without a search target // for the mongod automation config. if len(searchList.Items) == 1 { - searchSource := search_controller.NewEnterpriseResourceSearchSource(rs) + searchSource := searchcontroller.NewEnterpriseResourceSearchSource(rs) if searchSource.Validate() == nil { search = &searchList.Items[0] } diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index a25efed3c..99d8254ba 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -20,7 +20,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/watch" - "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" + "github.com/mongodb/mongodb-kubernetes/controllers/searchcontroller" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client" "github.com/mongodb/mongodb-kubernetes/pkg/kube/commoncontroller" @@ -31,10 +31,10 @@ import ( type MongoDBSearchReconciler struct { kubeClient kubernetesClient.Client watch *watch.ResourceWatcher - operatorSearchConfig search_controller.OperatorSearchConfig + operatorSearchConfig searchcontroller.OperatorSearchConfig } -func newMongoDBSearchReconciler(client client.Client, operatorSearchConfig search_controller.OperatorSearchConfig) *MongoDBSearchReconciler { +func newMongoDBSearchReconciler(client client.Client, operatorSearchConfig searchcontroller.OperatorSearchConfig) *MongoDBSearchReconciler { return &MongoDBSearchReconciler{ kubeClient: kubernetesClient.NewClient(client), watch: watch.NewResourceWatcher(), @@ -74,14 +74,14 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci r.watch.AddWatchedResourceIfNotAdded(mdbSearch.Spec.Security.TLS.CertificateKeySecret.Name, mdbSearch.Namespace, watch.Secret, mdbSearch.NamespacedName()) } - reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, searchSource, r.operatorSearchConfig) + reconcileHelper := searchcontroller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, searchSource, r.operatorSearchConfig) return reconcileHelper.Reconcile(ctx, log).ReconcileResult() } -func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch, log *zap.SugaredLogger) (search_controller.SearchSourceDBResource, error) { +func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch, log *zap.SugaredLogger) (searchcontroller.SearchSourceDBResource, error) { if search.IsExternalMongoDBSource() { - return search_controller.NewExternalSearchSource(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil + return searchcontroller.NewExternalSearchSource(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil } sourceMongoDBResourceRef := search.GetMongoDBResourceRef() @@ -99,7 +99,7 @@ func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, } } else { r.watch.AddWatchedResourceIfNotAdded(sourceMongoDBResourceRef.Name, sourceMongoDBResourceRef.Namespace, watch.MongoDB, search.NamespacedName()) - return search_controller.NewEnterpriseResourceSearchSource(mdb), nil + return searchcontroller.NewEnterpriseResourceSearchSource(mdb), nil } mdbc := &mdbcv1.MongoDBCommunity{} @@ -109,7 +109,7 @@ func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, } } else { r.watch.AddWatchedResourceIfNotAdded(sourceMongoDBResourceRef.Name, sourceMongoDBResourceRef.Namespace, "MongoDBCommunity", search.NamespacedName()) - return search_controller.NewCommunityResourceSearchSource(mdbc), nil + return searchcontroller.NewCommunityResourceSearchSource(mdbc), nil } return nil, xerrors.Errorf("No database resource named %s found", sourceName) @@ -125,8 +125,8 @@ func mdbcSearchIndexBuilder(rawObj client.Object) []string { return []string{resourceRef.Namespace + "/" + resourceRef.Name} } -func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operatorSearchConfig search_controller.OperatorSearchConfig) error { - if err := mgr.GetFieldIndexer().IndexField(ctx, &searchv1.MongoDBSearch{}, search_controller.MongoDBSearchIndexFieldName, mdbcSearchIndexBuilder); err != nil { +func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operatorSearchConfig searchcontroller.OperatorSearchConfig) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &searchv1.MongoDBSearch{}, searchcontroller.MongoDBSearchIndexFieldName, mdbcSearchIndexBuilder); err != nil { return err } diff --git a/controllers/operator/mongodbsearch_controller_test.go b/controllers/operator/mongodbsearch_controller_test.go index 28f966ff8..789d7ccf4 100644 --- a/controllers/operator/mongodbsearch_controller_test.go +++ b/controllers/operator/mongodbsearch_controller_test.go @@ -22,7 +22,7 @@ import ( userv1 "github.com/mongodb/mongodb-kubernetes/api/v1/user" "github.com/mongodb/mongodb-kubernetes/controllers/operator/mock" "github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow" - "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" + "github.com/mongodb/mongodb-kubernetes/controllers/searchcontroller" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1/common" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/mongot" @@ -53,11 +53,11 @@ func newMongoDBSearch(name, namespace, mdbcName string) *searchv1.MongoDBSearch func newSearchReconcilerWithOperatorConfig( mdbc *mdbcv1.MongoDBCommunity, - operatorConfig search_controller.OperatorSearchConfig, + operatorConfig searchcontroller.OperatorSearchConfig, searches ...*searchv1.MongoDBSearch, ) (*MongoDBSearchReconciler, client.Client) { builder := mock.NewEmptyFakeClientBuilder() - builder.WithIndex(&searchv1.MongoDBSearch{}, search_controller.MongoDBSearchIndexFieldName, mdbcSearchIndexBuilder) + builder.WithIndex(&searchv1.MongoDBSearch{}, searchcontroller.MongoDBSearchIndexFieldName, mdbcSearchIndexBuilder) if mdbc != nil { keyfileSecret := &corev1.Secret{ @@ -87,7 +87,7 @@ func newSearchReconciler( mdbc *mdbcv1.MongoDBCommunity, searches ...*searchv1.MongoDBSearch, ) (*MongoDBSearchReconciler, client.Client) { - return newSearchReconcilerWithOperatorConfig(mdbc, search_controller.OperatorSearchConfig{}, searches...) + return newSearchReconcilerWithOperatorConfig(mdbc, searchcontroller.OperatorSearchConfig{}, searches...) } func buildExpectedMongotConfig(search *searchv1.MongoDBSearch, mdbc *mdbcv1.MongoDBCommunity) mongot.Config { @@ -260,7 +260,7 @@ func TestMongoDBSearchReconcile_InvalidSearchImageVersion(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: search_controller.MongotContainerName, + Name: searchcontroller.MongotContainerName, Image: "testrepo/mongot:1.47.0", }, }, @@ -280,7 +280,7 @@ func TestMongoDBSearchReconcile_InvalidSearchImageVersion(t *testing.T) { search.Spec.Version = tc.specVersion search.Spec.StatefulSetConfiguration = tc.statefulSetConfig - operatorConfig := search_controller.OperatorSearchConfig{ + operatorConfig := searchcontroller.OperatorSearchConfig{ SearchVersion: tc.operatorVersion, } reconciler, _ := newSearchReconcilerWithOperatorConfig(mdbc, operatorConfig, search) diff --git a/controllers/search_controller/community_search_source.go b/controllers/searchcontroller/community_search_source.go similarity index 99% rename from controllers/search_controller/community_search_source.go rename to controllers/searchcontroller/community_search_source.go index 8b10a1cbf..81bd38003 100644 --- a/controllers/search_controller/community_search_source.go +++ b/controllers/searchcontroller/community_search_source.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "fmt" diff --git a/controllers/search_controller/community_search_source_test.go b/controllers/searchcontroller/community_search_source_test.go similarity index 99% rename from controllers/search_controller/community_search_source_test.go rename to controllers/searchcontroller/community_search_source_test.go index 90bf9967e..68cf5e449 100644 --- a/controllers/search_controller/community_search_source_test.go +++ b/controllers/searchcontroller/community_search_source_test.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "testing" diff --git a/controllers/search_controller/enterprise_search_source.go b/controllers/searchcontroller/enterprise_search_source.go similarity index 99% rename from controllers/search_controller/enterprise_search_source.go rename to controllers/searchcontroller/enterprise_search_source.go index 45e2d0f2d..6b35f5e48 100644 --- a/controllers/search_controller/enterprise_search_source.go +++ b/controllers/searchcontroller/enterprise_search_source.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "fmt" diff --git a/controllers/search_controller/enterprise_search_source_test.go b/controllers/searchcontroller/enterprise_search_source_test.go similarity index 99% rename from controllers/search_controller/enterprise_search_source_test.go rename to controllers/searchcontroller/enterprise_search_source_test.go index 1362ab605..ac9eaae5b 100644 --- a/controllers/search_controller/enterprise_search_source_test.go +++ b/controllers/searchcontroller/enterprise_search_source_test.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "testing" diff --git a/controllers/search_controller/external_search_source.go b/controllers/searchcontroller/external_search_source.go similarity index 98% rename from controllers/search_controller/external_search_source.go rename to controllers/searchcontroller/external_search_source.go index d15f49799..70025f305 100644 --- a/controllers/search_controller/external_search_source.go +++ b/controllers/searchcontroller/external_search_source.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "k8s.io/apimachinery/pkg/types" diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/searchcontroller/mongodbsearch_reconcile_helper.go similarity index 99% rename from controllers/search_controller/mongodbsearch_reconcile_helper.go rename to controllers/searchcontroller/mongodbsearch_reconcile_helper.go index 635ebe7fc..13e4bc26d 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/searchcontroller/mongodbsearch_reconcile_helper.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "context" diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go b/controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go similarity index 99% rename from controllers/search_controller/mongodbsearch_reconcile_helper_test.go rename to controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go index 5a2a757ce..7fa9a05f0 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go +++ b/controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "testing" diff --git a/controllers/search_controller/search_construction.go b/controllers/searchcontroller/search_construction.go similarity index 99% rename from controllers/search_controller/search_construction.go rename to controllers/searchcontroller/search_construction.go index 1d1e1ca5c..a85c66379 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/searchcontroller/search_construction.go @@ -1,4 +1,4 @@ -package search_controller +package searchcontroller import ( "fmt" diff --git a/main.go b/main.go index 37ae65f94..6a2d755b0 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,7 @@ import ( omv1 "github.com/mongodb/mongodb-kubernetes/api/v1/om" "github.com/mongodb/mongodb-kubernetes/controllers/operator" "github.com/mongodb/mongodb-kubernetes/controllers/operator/construct" - "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" + "github.com/mongodb/mongodb-kubernetes/controllers/searchcontroller" mcov1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" mcoController "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers" mcoConstruct "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/construct" @@ -390,7 +390,7 @@ func setupMongoDBMultiClusterCRD(ctx context.Context, mgr manager.Manager, image } func setupMongoDBSearchCRD(ctx context.Context, mgr manager.Manager) error { - return operator.AddMongoDBSearchController(ctx, mgr, search_controller.OperatorSearchConfig{ + return operator.AddMongoDBSearchController(ctx, mgr, searchcontroller.OperatorSearchConfig{ SearchRepo: env.ReadOrPanic("MDB_SEARCH_COMMUNITY_REPO_URL"), SearchName: env.ReadOrPanic("MDB_SEARCH_COMMUNITY_NAME"), SearchVersion: env.ReadOrPanic("MDB_SEARCH_COMMUNITY_VERSION"), diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index 67e062648..f042aa1c2 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -30,7 +30,7 @@ import ( k8sClient "sigs.k8s.io/controller-runtime/pkg/client" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" - "github.com/mongodb/mongodb-kubernetes/controllers/search_controller" + "github.com/mongodb/mongodb-kubernetes/controllers/searchcontroller" mdbv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/controllers/predicates" @@ -707,7 +707,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb var search *searchv1.MongoDBSearch searchList := &searchv1.MongoDBSearchList{} if err := r.client.List(ctx, searchList, &k8sClient.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(search_controller.MongoDBSearchIndexFieldName, mdb.Namespace+"/"+mdb.Name), + FieldSelector: fields.OneTermEqualSelector(searchcontroller.MongoDBSearchIndexFieldName, mdb.Namespace+"/"+mdb.Name), }); err != nil { r.log.Debug(err) } @@ -715,7 +715,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb // and that this resource passes search validations. If either fails, proceed without a search target // for the mongod automation config. if len(searchList.Items) == 1 { - searchSource := search_controller.NewCommunityResourceSearchSource(&mdb) + searchSource := searchcontroller.NewCommunityResourceSearchSource(&mdb) if searchSource.Validate() == nil { search = &searchList.Items[0] } @@ -842,7 +842,7 @@ func getMongodConfigSearchModification(search *searchv1.MongoDBSearch) automatio return automationconfig.NOOP() } - searchConfigParameters := search_controller.GetMongodConfigParameters(search) + searchConfigParameters := searchcontroller.GetMongodConfigParameters(search) return func(ac *automationconfig.AutomationConfig) { for i := range ac.Processes { err := mergo.Merge(&ac.Processes[i].Args26, objx.New(searchConfigParameters), mergo.WithOverride) From fee1e847f9ea8ee49c8b48b1e4f656ff4e9c8ef4 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 02:21:48 +0300 Subject: [PATCH 09/17] lower test timeouts --- .../tests/search/search_community_basic.py | 4 ++-- .../tests/search/search_community_external_mongod_basic.py | 6 +++--- .../tests/search/search_community_external_mongod_tls.py | 4 ++-- .../tests/search/search_community_tls.py | 4 ++-- .../tests/search/search_enterprise_basic.py | 6 +++--- .../tests/search/search_enterprise_tls.py | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py index 97e9c1af7..aa581bca8 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py @@ -71,7 +71,7 @@ def test_install_secrets(namespace: str, mdbs: MongoDBSearch): @mark.e2e_search_community_basic def test_create_database_resource(mdbc: MongoDBCommunity): mdbc.update() - mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_community_basic @@ -82,7 +82,7 @@ def test_create_search_resource(mdbs: MongoDBSearch): @mark.e2e_search_community_basic def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): - mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @fixture(scope="function") diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py index 110e134dc..e6f096460 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -104,18 +104,18 @@ def test_install_secrets(namespace: str, mdbs: MongoDBSearch): @mark.e2e_search_external_basic def test_create_database_resource(mdbc: MongoDBCommunity): mdbc.update() - mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_external_basic def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): mdbs.update() - mdbs.assert_reaches_phase(Phase.Running, timeout=1000) + mdbs.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_external_basic def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): - mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @fixture(scope="function") diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py index f6a270a48..7a30ae9ed 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -123,7 +123,7 @@ def test_install_tls_secrets_and_configmaps( @mark.e2e_search_external_tls def test_create_database_resource(mdbc: MongoDBCommunity): mdbc.update() - mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_external_tls @@ -154,7 +154,7 @@ def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): @mark.e2e_search_external_tls def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): - mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @fixture(scope="function") diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py index 44418b793..af4e9121e 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py @@ -108,7 +108,7 @@ def test_install_tls_secrets_and_configmaps(namespace: str, mdbc: MongoDBCommuni @mark.e2e_search_community_tls def test_create_database_resource(mdbc: MongoDBCommunity): mdbc.update() - mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_community_tls @@ -119,7 +119,7 @@ def test_create_search_resource(mdbs: MongoDBSearch): @mark.e2e_search_community_tls def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): - mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + mdbc.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_community_tls diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py index a3de8dcf8..2acbb37de 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py @@ -105,7 +105,7 @@ def test_install_operator(namespace: str, operator_installation_config: dict[str @mark.e2e_search_enterprise_basic def test_create_database_resource(mdb: MongoDB): mdb.update() - mdb.assert_reaches_phase(Phase.Running, timeout=1000) + mdb.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_enterprise_basic @@ -140,8 +140,8 @@ def test_create_search_resource(mdbs: MongoDBSearch): @mark.e2e_search_enterprise_basic def test_wait_for_database_resource_ready(mdb: MongoDB): - mdb.assert_abandons_phase(Phase.Running, timeout=1800) - mdb.assert_reaches_phase(Phase.Running, timeout=1800) + mdb.assert_abandons_phase(Phase.Running, timeout=300) + mdb.assert_reaches_phase(Phase.Running, timeout=300) for idx in range(mdb.get_members()): mongod_config = yaml.safe_load( diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py index f4c0bc96f..8050ccd5c 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py @@ -134,7 +134,7 @@ def test_install_tls_secrets_and_configmaps(namespace: str, mdb: MongoDB, mdbs: @mark.e2e_search_enterprise_tls def test_create_database_resource(mdb: MongoDB): mdb.update() - mdb.assert_reaches_phase(Phase.Running, timeout=1000) + mdb.assert_reaches_phase(Phase.Running, timeout=300) @mark.e2e_search_enterprise_tls @@ -169,8 +169,8 @@ def test_create_search_resource(mdbs: MongoDBSearch): @mark.e2e_search_enterprise_tls def test_wait_for_database_resource_ready(mdb: MongoDB): - mdb.assert_abandons_phase(Phase.Running, timeout=1800) - mdb.assert_reaches_phase(Phase.Running, timeout=1800) + mdb.assert_abandons_phase(Phase.Running, timeout=300) + mdb.assert_reaches_phase(Phase.Running, timeout=300) for idx in range(mdb.get_members()): mongod_config = yaml.safe_load( From e2fe9cd0d00990813150f4ce161ab877d74a092d Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 02:28:49 +0300 Subject: [PATCH 10/17] added source resolution comments --- controllers/operator/mongodbsearch_controller.go | 7 +++++++ controllers/searchcontroller/external_search_source.go | 3 +++ 2 files changed, 10 insertions(+) diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index 99d8254ba..b633fe9b4 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -80,6 +80,13 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci } func (r *MongoDBSearchReconciler) getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch, log *zap.SugaredLogger) (searchcontroller.SearchSourceDBResource, error) { + // Resolve the source database for this Search instance. + // If .spec.source.external is defined immediately return the external search source. + // Otherwise, read .spec.source.mongodbResourceRef or use the implicit database resource name (same as the Search resource's name). + // Try to get a MongoDB CR with the computed name and return the enterprise search source if successful. + // Otherwise, try to get a MongoDBCommunity CR with the same name and return the community search source. + // If everything fails just error out and the controller will retry reconciliation. + if search.IsExternalMongoDBSource() { return searchcontroller.NewExternalSearchSource(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil } diff --git a/controllers/searchcontroller/external_search_source.go b/controllers/searchcontroller/external_search_source.go index 70025f305..8b5cc2d56 100644 --- a/controllers/searchcontroller/external_search_source.go +++ b/controllers/searchcontroller/external_search_source.go @@ -19,6 +19,9 @@ type externalSearchResource struct { } func (r *externalSearchResource) Validate() error { + // We don't know anything about the external MongoDB deployment, so we can't validate it. + // Perhaps in the future the Operator could attempt to connect to the external MongoDB instance + // and validate its configuration. return nil } From d238db4f84fcaa7898a40e9014e927fee33a07ed Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 02:46:34 +0300 Subject: [PATCH 11/17] storage and requests constants --- controllers/searchcontroller/search_construction.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/searchcontroller/search_construction.go b/controllers/searchcontroller/search_construction.go index a85c66379..4c9181e44 100644 --- a/controllers/searchcontroller/search_construction.go +++ b/controllers/searchcontroller/search_construction.go @@ -83,7 +83,7 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso persistenceConfig = mdbSearch.Spec.Persistence.SingleConfig } - defaultPersistenceConfig := common.PersistenceConfig{Storage: "10G"} + defaultPersistenceConfig := common.PersistenceConfig{Storage: util.DefaultMongodStorageSize} dataVolumeClaim := statefulset.WithVolumeClaim(dataVolumeName, construct.PvcFunc(dataVolumeName, persistenceConfig, defaultPersistenceConfig, nil)) podSecurityContext, _ := podtemplatespec.WithDefaultSecurityContextsModifications() @@ -209,6 +209,7 @@ func createSearchResourceRequirements(requirements *corev1.ResourceRequirements) } func newSearchDefaultRequirements() corev1.ResourceRequirements { + // TODO: add default limits once there is an official mongot sizing guide return corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: construct.ParseQuantityOrZero("2"), From 72a7340d5241ac7874db90e94d4e882309a0adb0 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 02:51:24 +0300 Subject: [PATCH 12/17] remove comment mentioning tls.pem --- api/v1/search/mongodbsearch_types.go | 2 -- config/crd/bases/mongodb.com_mongodbsearch.yaml | 2 -- helm_chart/crds/mongodb.com_mongodbsearch.yaml | 2 -- public/crds.yaml | 2 -- 4 files changed, 8 deletions(-) diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index d11980acb..9dedaee4f 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -84,8 +84,6 @@ type TLS struct { // CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. // The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". // This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. - // Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. - // If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key // +optional CertificateKeySecret corev1.LocalObjectReference `json:"certificateKeySecretRef"` } diff --git a/config/crd/bases/mongodb.com_mongodbsearch.yaml b/config/crd/bases/mongodb.com_mongodbsearch.yaml index 08b4fa173..524b9b4cb 100644 --- a/config/crd/bases/mongodb.com_mongodbsearch.yaml +++ b/config/crd/bases/mongodb.com_mongodbsearch.yaml @@ -167,8 +167,6 @@ spec: CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. - Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key properties: name: default: "" diff --git a/helm_chart/crds/mongodb.com_mongodbsearch.yaml b/helm_chart/crds/mongodb.com_mongodbsearch.yaml index 08b4fa173..524b9b4cb 100644 --- a/helm_chart/crds/mongodb.com_mongodbsearch.yaml +++ b/helm_chart/crds/mongodb.com_mongodbsearch.yaml @@ -167,8 +167,6 @@ spec: CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. - Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key properties: name: default: "" diff --git a/public/crds.yaml b/public/crds.yaml index 42ca6a290..02988192c 100644 --- a/public/crds.yaml +++ b/public/crds.yaml @@ -4189,8 +4189,6 @@ spec: CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. - Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key properties: name: default: "" From 5c3ea5b1b8b904b88aa15646bfe31306869486d3 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 10:26:19 +0300 Subject: [PATCH 13/17] fix broken unit test --- controllers/operator/mongodbsearch_controller_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/operator/mongodbsearch_controller_test.go b/controllers/operator/mongodbsearch_controller_test.go index 789d7ccf4..736f08e37 100644 --- a/controllers/operator/mongodbsearch_controller_test.go +++ b/controllers/operator/mongodbsearch_controller_test.go @@ -100,21 +100,21 @@ func buildExpectedMongotConfig(search *searchv1.MongoDBSearch, mdbc *mdbcv1.Mong ReplicaSet: mongot.ConfigReplicaSet{ HostAndPort: hostAndPorts, Username: searchv1.MongotDefaultSyncSourceUsername, - PasswordFile: "/tmp/sourceUserPassword", + PasswordFile: searchcontroller.TempSourceUserPasswordPath, TLS: ptr.To(false), ReadPreference: ptr.To("secondaryPreferred"), AuthSource: ptr.To("admin"), }, }, Storage: mongot.ConfigStorage{ - DataPath: "/mongot/data/config.yml", + DataPath: searchcontroller.MongotDataPath, }, Server: mongot.ConfigServer{ Wireproto: &mongot.ConfigWireproto{ Address: "0.0.0.0:27027", Authentication: &mongot.ConfigAuthentication{ Mode: "keyfile", - KeyFile: "/tmp/keyfile", + KeyFile: searchcontroller.TempKeyfilePath, }, TLS: mongot.ConfigTLS{Mode: mongot.ConfigTLSModeDisabled}, }, @@ -185,7 +185,7 @@ func TestMongoDBSearchReconcile_Success(t *testing.T) { expectedConfig := buildExpectedMongotConfig(search, mdbc) configYaml, err := yaml.Marshal(expectedConfig) assert.NoError(t, err) - assert.Equal(t, string(configYaml), cm.Data["config.yml"]) + assert.Equal(t, string(configYaml), cm.Data[searchcontroller.MongotConfigFilename]) sts := &appsv1.StatefulSet{} err = c.Get(ctx, search.StatefulSetNamespacedName(), sts) From 28f660289428c94af7a111b082ece8d1e5f55bd8 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Thu, 11 Sep 2025 10:38:47 +0200 Subject: [PATCH 14/17] update the keyFileSecretRef constant --- api/v1/search/mongodbsearch_types.go | 2 +- config/crd/bases/mongodb.com_mongodbsearch.yaml | 2 +- .../tests/search/search_community_external_mongod_basic.py | 2 +- .../tests/search/search_community_external_mongod_tls.py | 2 +- helm_chart/crds/mongodb.com_mongodbsearch.yaml | 2 +- public/crds.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index 9dedaee4f..ce90a8d13 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -62,7 +62,7 @@ type MongoDBSource struct { type ExternalMongoDBSource struct { HostAndPorts []string `json:"hostAndPorts,omitempty"` // mongod keyfile used to connect to the external MongoDB deployment - KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyFileSecretRef,omitempty"` + KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyfileSecretRef,omitempty"` // TLS configuration for the external MongoDB deployment // +optional TLS *ExternalMongodTLS `json:"tls,omitempty"` diff --git a/config/crd/bases/mongodb.com_mongodbsearch.yaml b/config/crd/bases/mongodb.com_mongodbsearch.yaml index 524b9b4cb..6cdf7d1dc 100644 --- a/config/crd/bases/mongodb.com_mongodbsearch.yaml +++ b/config/crd/bases/mongodb.com_mongodbsearch.yaml @@ -195,7 +195,7 @@ spec: items: type: string type: array - keyFileSecretRef: + keyfileSecretRef: description: mongod keyfile used to connect to the external MongoDB deployment properties: diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py index e6f096460..3a084b7a6 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -70,7 +70,7 @@ def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: "source": { "external": { "hostAndPorts": seeds, - "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile", "key": "keyfile"}, + "keyfileSecretRef": {"name": f"{mdbc.name}-keyfile", "key": "keyfile"}, "tls": {"enabled": False}, }, "passwordSecretRef": {"name": f"{MDBC_RESOURCE_NAME}-{MONGOT_USER_NAME}-password", "key": "password"}, diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py index 7a30ae9ed..8ccd5298c 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -136,7 +136,7 @@ def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): mdbs["spec"]["source"] = { "external": { "hostAndPorts": seeds, - "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, + "keyfileSecretRef": {"name": f"{mdbc.name}-keyfile"}, "tls": { "enabled": True, "ca": {"name": f"{mdbc.name}-ca"}, diff --git a/helm_chart/crds/mongodb.com_mongodbsearch.yaml b/helm_chart/crds/mongodb.com_mongodbsearch.yaml index 524b9b4cb..6cdf7d1dc 100644 --- a/helm_chart/crds/mongodb.com_mongodbsearch.yaml +++ b/helm_chart/crds/mongodb.com_mongodbsearch.yaml @@ -195,7 +195,7 @@ spec: items: type: string type: array - keyFileSecretRef: + keyfileSecretRef: description: mongod keyfile used to connect to the external MongoDB deployment properties: diff --git a/public/crds.yaml b/public/crds.yaml index 02988192c..9fa6a9207 100644 --- a/public/crds.yaml +++ b/public/crds.yaml @@ -4217,7 +4217,7 @@ spec: items: type: string type: array - keyFileSecretRef: + keyfileSecretRef: description: mongod keyfile used to connect to the external MongoDB deployment properties: From 807c1f593dc9bb8fae6110fd983bc18f02644680 Mon Sep 17 00:00:00 2001 From: Yavor Georgiev Date: Thu, 11 Sep 2025 12:45:39 +0300 Subject: [PATCH 15/17] searchCoordinator role detection supports -pre and -ent versions --- .../operator/mongodbreplicaset_controller.go | 8 +--- .../mongodbsearch_reconcile_helper.go | 14 ++++++ .../mongodbsearch_reconcile_helper_test.go | 46 +++++++++++++++++++ .../controllers/replica_set_controller.go | 6 +-- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/controllers/operator/mongodbreplicaset_controller.go b/controllers/operator/mongodbreplicaset_controller.go index 66f392250..fd2ed2cb8 100644 --- a/controllers/operator/mongodbreplicaset_controller.go +++ b/controllers/operator/mongodbreplicaset_controller.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/blang/semver" "go.uber.org/zap" "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/api/errors" @@ -660,11 +659,8 @@ func (r *ReconcileMongoDbReplicaSet) applySearchOverrides(ctx context.Context, r searchMongodConfig := searchcontroller.GetMongodConfigParameters(search) rs.Spec.AdditionalMongodConfig.AddOption("setParameter", searchMongodConfig["setParameter"]) - mdbVersion, err := semver.ParseTolerant(rs.Spec.Version) - if err != nil { - log.Warnf("Failed to parse MongoDB version %q: %w. Proceeding without the automatic creation of the searchCoordinator role that's necessary for MongoDB <8.2", rs.Spec.Version, err) - } else if semver.MustParse("8.2.0").GT(mdbVersion) { - log.Infof("Polyfilling the searchCoordinator role for MongoDB %s", rs.Spec.Version) + if searchcontroller.NeedsSearchCoordinatorRolePolyfill(rs.Spec.GetMongoDBVersion()) { + log.Infof("Polyfilling the searchCoordinator role for MongoDB %s", rs.Spec.GetMongoDBVersion()) if rs.Spec.Security == nil { rs.Spec.Security = &mdbv1.Security{} diff --git a/controllers/searchcontroller/mongodbsearch_reconcile_helper.go b/controllers/searchcontroller/mongodbsearch_reconcile_helper.go index 13e4bc26d..a8a4cc838 100644 --- a/controllers/searchcontroller/mongodbsearch_reconcile_helper.go +++ b/controllers/searchcontroller/mongodbsearch_reconcile_helper.go @@ -7,6 +7,7 @@ import ( "fmt" "strings" + "github.com/blang/semver" "github.com/ghodss/yaml" "go.uber.org/zap" "golang.org/x/xerrors" @@ -514,3 +515,16 @@ func SearchCoordinatorRole() mdbv1.MongoDBRole { AuthenticationRestrictions: nil, } } + +// Because the first Search Public Preview support MongoDB Server 8.0.10 we need to polyfill the searchCoordinator role +// TODO: Remove once we drop support for <8.2 in Search +func NeedsSearchCoordinatorRolePolyfill(mongodbVersion string) bool { + version, err := semver.ParseTolerant(mongodbVersion) + if err != nil { + // if we can't determine the version, assume no need to polyfill + return false + } + + // the searchCoordinator role is built-in from MongoDB 8.2 + return version.Major <= 8 && version.Minor < 2 +} diff --git a/controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go b/controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go index 7fa9a05f0..0de5e0488 100644 --- a/controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go +++ b/controllers/searchcontroller/mongodbsearch_reconcile_helper_test.go @@ -95,3 +95,49 @@ func TestMongoDBSearchReconcileHelper_ValidateSingleMongoDBSearchForSearchSource }) } } + +func TestNeedsSearchCoordinatorRolePolyfill(t *testing.T) { + cases := []struct { + name string + version string + expected bool + }{ + { + name: "MongoDB 8.0.x requires polyfill", + version: "8.0.10", + expected: true, + }, + { + name: "MongoDB 8.1.x requires polyfill", + version: "8.1.0", + expected: true, + }, + { + name: "MongoDB 8.2.0-rc0 treated as 8.2 (no polyfill)", + version: "8.2.0-rc0", + expected: false, + }, + { + name: "MongoDB 8.2.0 and above do not require polyfill", + version: "8.2.0", + expected: false, + }, + { + name: "MongoDB 8.2.0-ent treated as 8.2 (no polyfill)", + version: "8.2.0-ent", + expected: false, + }, + { + name: "MongoDB 9.0.0 and above do not require polyfill", + version: "9.0.0", + expected: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + actual := NeedsSearchCoordinatorRolePolyfill(c.version) + assert.Equal(t, c.expected, actual) + }) + } +} diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index f042aa1c2..0f02b8946 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -731,7 +731,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb prometheusModification, processPortManager.GetPortsModification(), getMongodConfigSearchModification(search), - searchCoordinatorCustomRoleModification(search), + searchCoordinatorCustomRoleModification(search, mdb.GetMongoDBVersion()), ) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not create an automation config: %s", err) @@ -745,8 +745,8 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb } // TODO: remove this as soon as searchCoordinator builtin role is backported -func searchCoordinatorCustomRoleModification(search *searchv1.MongoDBSearch) automationconfig.Modification { - if search == nil { +func searchCoordinatorCustomRoleModification(search *searchv1.MongoDBSearch, mongodbVersion string) automationconfig.Modification { + if search == nil || searchcontroller.NeedsSearchCoordinatorRolePolyfill(mongodbVersion) { return automationconfig.NOOP() } From f3ba070b47ba1483dd8ce4996005598e048a276e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Thu, 11 Sep 2025 11:50:27 +0200 Subject: [PATCH 16/17] Changes after merge conflicts --- .../code_snippets/0045_create_namespaces.sh | 1 - .../04-search-external-mongod/README.md | 547 ++++++++++++++ .../04_0045_create_namespaces.sh | 1 + .../04_0046_create_image_pull_secrets.sh | 3 + ...8_configure_prerelease_image_pullsecret.sh | 11 + ..._0080_helm_login_to_prerelease_registry.sh | 8 + .../04_0090_helm_add_mogodb_repo.sh | 3 + .../code_snippets/04_0100_install_operator.sh | 6 + ...5_create_mongodb_community_user_secrets.sh | 11 + ..._0310_create_mongodb_community_resource.sh | 121 ++++ .../04_0315_wait_for_community_resource.sh | 7 + .../04_0318_create_external_keyfile_secret.sh | 3 + .../04_0320_create_mongodb_search_resource.sh | 29 + ...0322_create_search_loadbalancer_service.sh | 34 + .../04_0323_update_coredns_configmap.sh | 46 ++ .../04_0325_wait_for_search_resource.sh | 3 + .../04_0330_wait_for_community_resource.sh | 3 + .../04_0335_show_running_pods.sh | 6 + .../code_snippets/04_9010_delete_namespace.sh | 1 + .../env_variables.sh | 32 + .../env_variables_e2e_prerelease.sh | 4 + .../env_variables_e2e_private.sh | 6 + .../env_variables_e2e_private_dev.sh | 36 + .../env_variables_e2e_public.sh | 1 + .../env_variables_prerelease.sh | 12 + docs/search/04-search-external-mongod/test.sh | 29 + .../03_0420_import_movies_mflix_database.out | 2 + .../03_0440_wait_for_search_index_ready.out | 1 + .../03_0444_list_search_indexes.out | 32 + .../03_0445_list_vector_search_indexes.out | 40 ++ .../03_0450_execute_search_query.out | 22 + .../03_0455_execute_vector_search_query.out | 54 ++ .../04_0100_install_operator.out | 666 ++++++++++++++++++ .../04_0315_wait_for_community_resource.out | 13 + .../04_0330_wait_for_community_resource.out | 2 + .../04_0335_show_running_pods.out | 16 + ...st_kind_search_external_mongod_snippets.sh | 37 + 37 files changed, 1848 insertions(+), 1 deletion(-) delete mode 100644 docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh create mode 100644 docs/search/04-search-external-mongod/README.md create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0045_create_namespaces.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0046_create_image_pull_secrets.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0048_configure_prerelease_image_pullsecret.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0080_helm_login_to_prerelease_registry.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0090_helm_add_mogodb_repo.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0100_install_operator.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0305_create_mongodb_community_user_secrets.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0310_create_mongodb_community_resource.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0315_wait_for_community_resource.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0318_create_external_keyfile_secret.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0320_create_mongodb_search_resource.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0322_create_search_loadbalancer_service.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0323_update_coredns_configmap.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0325_wait_for_search_resource.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0330_wait_for_community_resource.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_0335_show_running_pods.sh create mode 100644 docs/search/04-search-external-mongod/code_snippets/04_9010_delete_namespace.sh create mode 100644 docs/search/04-search-external-mongod/env_variables.sh create mode 100644 docs/search/04-search-external-mongod/env_variables_e2e_prerelease.sh create mode 100644 docs/search/04-search-external-mongod/env_variables_e2e_private.sh create mode 100644 docs/search/04-search-external-mongod/env_variables_e2e_private_dev.sh create mode 100644 docs/search/04-search-external-mongod/env_variables_e2e_public.sh create mode 100644 docs/search/04-search-external-mongod/env_variables_prerelease.sh create mode 100644 docs/search/04-search-external-mongod/test.sh create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0420_import_movies_mflix_database.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0440_wait_for_search_index_ready.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0444_list_search_indexes.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0445_list_vector_search_indexes.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0450_execute_search_query.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0455_execute_vector_search_query.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0315_wait_for_community_resource.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0330_wait_for_community_resource.out create mode 100644 scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0335_show_running_pods.out create mode 100644 scripts/code_snippets/tests/test_kind_search_external_mongod_snippets.sh diff --git a/docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh b/docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh deleted file mode 100644 index 7d0899c63..000000000 --- a/docs/community-search/quick-start/code_snippets/0045_create_namespaces.sh +++ /dev/null @@ -1 +0,0 @@ -kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create namespace "${MDB_NAMESPACE}" diff --git a/docs/search/04-search-external-mongod/README.md b/docs/search/04-search-external-mongod/README.md new file mode 100644 index 000000000..9d1f059c6 --- /dev/null +++ b/docs/search/04-search-external-mongod/README.md @@ -0,0 +1,547 @@ +# MongoDB Search with External MongoDB (No TLS) - Quick Start + +This guide shows how to deploy MongoDB Search in Kubernetes and connect it to an existing external MongoDB replica set +without TLS. The mongot component runs in the cluster, while the MongoDB replica set exists externally. + +## Prerequisites + +Before you begin, ensure you have the following tools and configurations in place: + +- **Kubernetes cluster**: A running Kubernetes cluster (e.g., Minikube, Kind, GKE, EKS, AKS) with kubeconfig available + locally. +- **kubectl**: The Kubernetes command-line tool, configured to communicate with your cluster. +- **Helm**: The package manager for Kubernetes, used here to install the MongoDB Kubernetes Operator. +- **Bash 5.1+**: All shell commands in this guide are intended to be run in Bash. Scripts in this guide are + automatically tested on Linux with Bash 5.1. +- **External MongoDB**: An existing MongoDB Community replica set (version 8.0.10 or higher) that will serve as the data + source for search. + +## Setup Steps + +The following steps guide you through deploying MongoDB Search to connect to your external MongoDB. Each step provides a +shell script. +**It is important to first source the `env_variables.sh` script provided and customize its values for your environment. +** +The subsequent script snippets rely on the environment variables defined in `env_variables.sh`. You should copy and +paste each script into your Bash terminal. + +### 1. Configure Environment Variables + +First, you need to set up your environment. The `env_variables.sh` script, shown below, contains variables for the +subsequent steps. You should create this file locally or use the linked one. + +Download or copy the content of `env_variables.sh`: +[env_variables.sh](env_variables.sh) + +```shell copy +# set it to the context name of the k8s cluster +export K8S_CTX="" + +# the following namespace will be created if not exists +export MDB_NS="mongodb" + +# minimum required MongoDB version for running MongoDB Search is 8.0.10 +export MDB_VERSION="8.0.10" + +# root admin user for convenience, not used here at all in this guide +export MDB_ADMIN_USER_PASSWORD="admin-user-password-CHANGE-ME" +# regular user performing restore and search queries on sample mflix database +export MDB_USER_PASSWORD="mdb-user-password-CHANGE-ME" +# user for MongoDB Search to connect to the replica set to synchronise data from +export MDB_SEARCH_SYNC_USER_PASSWORD="search-sync-user-password-CHANGE-ME" + +export MDB_SEARCH_HOSTNAME="mdbs-search" + +# External MongoDB replica set members - REPLACE THESE VALUES with your actual external MongoDB hosts +# For testing purposes, these point to the MongoDB Community resource created by test.sh +# In production, replace with your actual external MongoDB replica set members +export MDB_EXTERNAL_HOST_0="mdbc-rs-0.mdbc-rs-svc.${MDB_NS}.svc.cluster.local:27017" +export MDB_EXTERNAL_HOST_1="mdbc-rs-1.mdbc-rs-svc.${MDB_NS}.svc.cluster.local:27017" +export MDB_EXTERNAL_HOST_2="mdbc-rs-2.mdbc-rs-svc.${MDB_NS}.svc.cluster.local:27017" +# REPLACE with your external MongoDB keyfile secret name +export MDB_EXTERNAL_KEYFILE_SECRET_NAME="mdbc-rs-keyfile" +# REPLACE with the actual keyfile content from your external MongoDB replica set +# For testing, this will be automatically generated by the MongoDB Community resource +export MDB_EXTERNAL_KEYFILE_CONTENT="your-mongodb-keyfile-content-CHANGE-ME" +# REPLACE with your actual external MongoDB replica set name +export MDB_EXTERNAL_REPLICA_SET_NAME="mdbc-rs" + +export OPERATOR_HELM_CHART="mongodb/mongodb-kubernetes" +# comma-separated key=value pairs for additional parameters passed to the helm-chart installing the operator +export OPERATOR_ADDITIONAL_HELM_VALUES="" +``` + +This will load the variables into your current shell session, making them available for the commands in the following +steps. + +### 2. Add MongoDB Helm Repository + +First, add the MongoDB Helm repository. This repository contains the Helm chart required to install the MongoDB +Kubernetes Operator. The operator automates the deployment and management of MongoDB Search instances on Kubernetes. + +[code_snippets/090_helm_add_mogodb_repo.sh](code_snippets/090_helm_add_mogodb_repo.sh) + +```shell copy +helm repo add mongodb https://mongodb.github.io/helm-charts +helm repo update mongodb +helm search repo mongodb/mongodb-kubernetes +``` + +### 3. Install MongoDB Kubernetes Operator + +Next, install the MongoDB Kubernetes Operator from the Helm repository you just added. The Operator will watch for +MongoDBSearch custom resources and manage the lifecycle of your MongoDB Search deployments. + +[code_snippets/0100_install_operator.sh](code_snippets/0100_install_operator.sh) + +```shell copy +helm upgrade --install --debug --kube-context "${K8S_CTX}" \ + --create-namespace \ + --namespace="${MDB_NS}" \ + mongodb-kubernetes \ + --set "${OPERATOR_ADDITIONAL_HELM_VALUES:-"dummy=value"}" \ + "${OPERATOR_HELM_CHART}" +``` + +This command installs the operator in the `mongodb` namespace (creating it if it doesn't exist). + +## Deploying MongoDB Search + +With the prerequisites and initial setup complete, you can now deploy MongoDB Search to connect to your external +MongoDB. + +### 4. Create User Secrets + +MongoDB Search requires authentication credentials to connect to your external MongoDB, and you'll also need credentials +for testing the search functionality. This step creates Kubernetes secrets for all required passwords: the search +synchronization user, admin user, and regular user passwords that must exist in your external MongoDB. + +[code_snippets/04_0305_create_mongodb_community_user_secrets.sh](code_snippets/04_0305_create_mongodb_community_user_secrets.sh) + +```shell copy +kubectl --context "${K8S_CTX}" --namespace "${MDB_NS}" \ + create secret generic mdb-admin-user-password \ + --from-literal=password="${MDB_ADMIN_USER_PASSWORD}" + +kubectl --context "${K8S_CTX}" --namespace "${MDB_NS}" \ + create secret generic mdbc-rs-search-sync-source-password \ + --from-literal=password="${MDB_SEARCH_SYNC_USER_PASSWORD}" + +kubectl --context "${K8S_CTX}" --namespace "${MDB_NS}" \ + create secret generic mdb-user-password \ + --from-literal=password="${MDB_USER_PASSWORD}" +``` + +Ensure these secrets are created in the same namespace where you plan to deploy MongoDB Search. + +### 5. Create External MongoDB Keyfile Secret + +Your external MongoDB replica set uses a keyfile for internal authentication between replica set members. MongoDB Search +needs access to this same keyfile to authenticate with your external MongoDB. This step creates a Kubernetes secret +containing the keyfile content from your external MongoDB. + +**Important**: You must obtain the keyfile content from your external MongoDB replica set. This is typically a +base64-encoded string or the raw keyfile content used by your MongoDB instances. + +[code_snippets/04_0318_create_external_keyfile_secret.sh](code_snippets/04_0318_create_external_keyfile_secret.sh) + +```shell copy +kubectl --context "${K8S_CTX}" --namespace "${MDB_NS}" \ + create secret generic "${MDB_EXTERNAL_KEYFILE_SECRET_NAME}" \ + --from-literal=keyfile="${MDB_EXTERNAL_KEYFILE_CONTENT}" +``` + +**Note**: Make sure to set the `MDB_EXTERNAL_KEYFILE_CONTENT` environment variable to the exact keyfile content used by +your external MongoDB replica set. + +### 6. Create MongoDB Search Resource + +Deploy a `MongoDBSearch` resource named `mdbs` that connects to your external MongoDB replica set. The Search resource +lists the external replica set members under `spec.source.external.hostAndPorts` and uses the search synchronization +user credentials. TLS is disabled in this example. + +Note: Public Preview of MongoDB Community Search comes with some limitations, and it is not suitable for production use: + +* Only one instance of the search node is supported (load balancing is not supported) + +[code_snippets/0320_create_mongodb_search_resource.sh](code_snippets/0320_create_mongodb_search_resource.sh) + +```shell copy +kubectl apply --context "${K8S_CTX}" -n "${MDB_NS}" -f - <:` from any cluster node. + +**Ingress with TCP Support** + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: search-ingress + annotations: + nginx.ingress.kubernetes.io/tcp-services-configmap: ${MDB_NS}/tcp-services +spec: +# Configure TCP passthrough for port 27027 +``` + +Requires an ingress controller that supports TCP services (like NGINX). + +**ClusterIP with Port Forwarding** (Development only) + +```bash +kubectl port-forward -n ${MDB_NS} svc/${MDB_SEARCH_HOSTNAME} 27027:27027 +``` + +Suitable for local development and testing only. + +### 8. Update CoreDNS Configuration for External Access + +When using external MongoDB instances, you need to configure DNS resolution so that the external MongoDB can resolve the +search service hostname. This step updates the CoreDNS configuration to map the search hostname to the LoadBalancer's +external IP address. + +[code_snippets/04_0323_update_coredns_configmap.sh](code_snippets/04_0323_update_coredns_configmap.sh) + +```shell copy +# Fetch the LoadBalancer external IP/hostname +SEARCH_IP=$(kubectl --context "${K8S_CTX}" -n "${MDB_NS}" get svc "${MDB_SEARCH_HOSTNAME}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +if [ -z "${SEARCH_IP}" ]; then + SEARCH_IP=$(kubectl --context "${K8S_CTX}" -n "${MDB_NS}" get svc "${MDB_SEARCH_HOSTNAME}" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +fi + +if [ -z "${SEARCH_IP}" ]; then + echo "Error: Could not get LoadBalancer external IP/hostname for service ${MDB_SEARCH_HOSTNAME}" + exit 1 +fi + +echo "Using LoadBalancer external address: ${SEARCH_IP}" + +kubectl --context "${K8S_CTX}" -n kube-system apply -f - < /tmp/mdb_script.js +mongosh --quiet "mongodb://mdb-user:${MDB_USER_PASSWORD}@mdbc-rs-0.mdbc-rs-svc.${MDB_NS}.svc.cluster.local:27017/?replicaSet=mdbc-rs" < /tmp/mdb_script.js +EOF +)" +``` diff --git a/docs/search/04-search-external-mongod/code_snippets/04_0045_create_namespaces.sh b/docs/search/04-search-external-mongod/code_snippets/04_0045_create_namespaces.sh new file mode 100644 index 000000000..076899281 --- /dev/null +++ b/docs/search/04-search-external-mongod/code_snippets/04_0045_create_namespaces.sh @@ -0,0 +1 @@ +kubectl --context "${K8S_CTX}" create namespace "${MDB_NS}" diff --git a/docs/search/04-search-external-mongod/code_snippets/04_0046_create_image_pull_secrets.sh b/docs/search/04-search-external-mongod/code_snippets/04_0046_create_image_pull_secrets.sh new file mode 100644 index 000000000..7158760c4 --- /dev/null +++ b/docs/search/04-search-external-mongod/code_snippets/04_0046_create_image_pull_secrets.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CTX}" -n "${MDB_NS}" \ + create secret generic "image-registries-secret" \ + --from-file=.dockerconfigjson="${HOME}/.docker/config.json" --type=kubernetes.io/dockerconfigjson diff --git a/docs/search/04-search-external-mongod/code_snippets/04_0048_configure_prerelease_image_pullsecret.sh b/docs/search/04-search-external-mongod/code_snippets/04_0048_configure_prerelease_image_pullsecret.sh new file mode 100644 index 000000000..4cb9ccd8a --- /dev/null +++ b/docs/search/04-search-external-mongod/code_snippets/04_0048_configure_prerelease_image_pullsecret.sh @@ -0,0 +1,11 @@ +if [[ "${PRERELEASE_IMAGE_PULLSECRET:-""}" == "" ]]; then return 0; fi + +kubectl apply --context "${K8S_CTX}" -n "${MDB_NS}" -f - </dev/null) + if [ -n "${EXTERNAL_IP}" ] && [ "${EXTERNAL_IP}" != "null" ]; then + echo "External IP assigned: ${EXTERNAL_IP}" + break + fi + echo "Still waiting for external IP assignment... (${ELAPSED}s/${TIMEOUT}s)" + sleep 5 + ELAPSED=$((ELAPSED + 5)) +done + +if [ ${ELAPSED} -ge ${TIMEOUT} ]; then + echo "ERROR: Timeout reached (${TIMEOUT}s) while waiting for external IP assignment" + echo "LoadBalancer service may take longer to provision or there may be an issue" + exit 1 +fi diff --git a/docs/search/04-search-external-mongod/code_snippets/04_0323_update_coredns_configmap.sh b/docs/search/04-search-external-mongod/code_snippets/04_0323_update_coredns_configmap.sh new file mode 100644 index 000000000..c5f457101 --- /dev/null +++ b/docs/search/04-search-external-mongod/code_snippets/04_0323_update_coredns_configmap.sh @@ -0,0 +1,46 @@ +SEARCH_IP=$(kubectl --context "${K8S_CTX}" -n "${MDB_NS}" get svc "${MDB_SEARCH_HOSTNAME}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +if [ -z "${SEARCH_IP}" ]; then + SEARCH_IP=$(kubectl --context "${K8S_CTX}" -n "${MDB_NS}" get svc "${MDB_SEARCH_HOSTNAME}" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +fi + +if [ -z "${SEARCH_IP}" ]; then + echo "Error: Could not get LoadBalancer external IP/hostname for service ${MDB_SEARCH_HOSTNAME}" + exit 1 +fi + +echo "Using LoadBalancer external address: ${SEARCH_IP}" + +kubectl --context "${K8S_CTX}" -n kube-system apply -f - < switched to db sample_mflix +mdbc-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'A sports agent stages an unconventional recruitment strategy to get talented Indian cricket players to play Major League Baseball.', + genres: [ 'Biography', 'Drama', 'Sport' ], + title: 'Million Dollar Arm', + released: ISODate('2014-05-16T00:00:00.000Z') + }, + { + plot: 'A Taiwanese high school baseball team travels to Japan in 1931 to compete in a national tournament.', + genres: [ 'Biography', 'Drama', 'History' ], + title: 'Kano', + released: ISODate('2014-02-27T00:00:00.000Z') + }, + { + plot: "12-year-old Josh is a mixed race boy and a promising baseball player. He is abused by his mother's boyfriend Byrd, and neglected by his mother Debbie. He forges his own path in life when ...", + genres: [ 'Drama' ], + title: 'Calloused Hands', + released: ISODate('2013-03-03T00:00:00.000Z') + } +] +mdbc-rs [primary] sample_mflix> \ No newline at end of file diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0455_execute_vector_search_query.out b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0455_execute_vector_search_query.out new file mode 100644 index 000000000..fb9f3de41 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/03_0455_execute_vector_search_query.out @@ -0,0 +1,54 @@ +mdbc-rs [primary] test> switched to db sample_mflix +mdbc-rs [primary] sample_mflix> ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... [ + { + plot: 'At the age of 21, Tim discovers he can travel in time and change what happens and has happened in his own life. His decision to make his world a better place by getting a girlfriend turns out not to be as easy as you might think.', + title: 'About Time', + score: 0.7704131603240967 + }, + { + plot: 'A psychiatrist makes multiple trips through time to save a woman that was murdered by her brutal husband.', + title: 'Retroactive', + score: 0.7597770690917969 + }, + { + plot: 'An officer for a security agency that regulates time travel, must fend for his life against a shady politician who has a tie to his past.', + title: 'Timecop', + score: 0.7574796676635742 + }, + { + plot: 'A time-travel experiment in which a robot probe is sent from the year 2073 to the year 1973 goes terribly wrong thrusting one of the project scientists, a man named Nicholas Sinclair into a...', + title: 'A.P.E.X.', + score: 0.7573235034942627 + }, + { + plot: 'After visiting 2015, Marty McFly must repeat his visit to 1955 to prevent disastrous changes to 1985... without interfering with his first trip.', + title: 'Back to the Future Part II', + score: 0.751945972442627 + }, + { + plot: 'A reporter, learning of time travelers visiting 20th century disasters, tries to change the history they know by averting upcoming disasters.', + title: 'Thrill Seekers', + score: 0.7503504753112793 + }, + { + plot: 'Hoping to alter the events of the past, a 19th century inventor instead travels 800,000 years into the future, where he finds humankind divided into two warring races.', + title: 'The Time Machine', + score: 0.750007152557373 + }, + { + plot: 'Lyle, a motorcycle champion is traveling the Mexican desert, when he find himself in the action radius of a time machine. So he find himself one century back in the past between rapists, ...', + title: 'Timerider: The Adventure of Lyle Swann', + score: 0.7499568462371826 + }, + { + plot: 'A romantic drama about a Chicago librarian with a gene that causes him to involuntarily time travel, and the complications it creates for his marriage.', + title: "The Time Traveler's Wife", + score: 0.7492842674255371 + }, + { + plot: 'A modern aircraft carrier is thrown back in time to 1941 near Hawaii, just hours before the Japanese attack on Pearl Harbor.', + title: 'The Final Countdown', + score: 0.7472751140594482 + } +] +mdbc-rs [primary] sample_mflix> \ No newline at end of file diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out new file mode 100644 index 000000000..a6002b4b8 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out @@ -0,0 +1,666 @@ +Release "mongodb-kubernetes" does not exist. Installing it now. +NAME: mongodb-kubernetes +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out +LAST DEPLOYED: Thu Sep 4 16:47:34 2025 +======== +LAST DEPLOYED: Mon Sep 1 19:22:18 2025 +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out +NAMESPACE: mongodb +STATUS: deployed +REVISION: 1 +TEST SUITE: None +USER-SUPPLIED VALUES: +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out +agent: + version: 108.0.12.8846-1 +community: + registry: + agent: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +database: + name: mongodb-kubernetes-database + version: 68a83a6562a059000707301b +initAppDb: + version: 68a83a6562a059000707301b +initDatabase: + version: 68a83a6562a059000707301b +initOpsManager: + version: 68a83a6562a059000707301b +managedSecurityContext: false +mongodb: + imageType: ubi9 + name: mongodb-enterprise-server +operator: + enablePVCResize: true + maxConcurrentReconciles: 10 + mdbDefaultArchitecture: non-static + telemetry: + collection: + frequency: 1m + send: + enabled: false + version: 68a83a6562a059000707301b +opsManager: + name: mongodb-enterprise-ops-manager-ubi +registry: + agent: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + appDb: quay.io/mongodb + database: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + imagePullSecrets: image-registries-secret + initAppDb: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + initDatabase: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + initOpsManager: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + operator: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + opsManager: quay.io/mongodb +search: + community: + name: mongot/community + repo: 268558157000.dkr.ecr.eu-west-1.amazonaws.com + version: 1.53.0-78-g227f2593f +======== +registry: + imagePullSecrets: prerelease-image-pullsecret +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + +COMPUTED VALUES: +agent: + name: mongodb-agent + version: 108.0.12.8846-1 +community: + agent: + name: mongodb-agent + version: 108.0.2.8729-1 + mongodb: + imageType: ubi8 + name: mongodb-community-server + repo: quay.io/mongodb + name: mongodb-database + registry: + agent: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + resource: + members: 3 + name: mongodb-replica-set + tls: + caCertificateSecretRef: tls-ca-key-pair + certManager: + certDuration: 8760h + renewCertBefore: 720h + certificateKeySecretRef: tls-certificate + enabled: false + sampleX509User: false + useCertManager: true + useX509: false + version: 4.4.0 +database: + name: mongodb-kubernetes-database +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + version: 68a83a6562a059000707301b +initAppDb: + name: mongodb-kubernetes-init-appdb + version: 68a83a6562a059000707301b +initDatabase: + name: mongodb-kubernetes-init-database + version: 68a83a6562a059000707301b +initOpsManager: + name: mongodb-kubernetes-init-ops-manager + version: 68a83a6562a059000707301b +======== + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +initAppDb: + name: mongodb-kubernetes-init-appdb + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +initDatabase: + name: mongodb-kubernetes-init-database + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +initOpsManager: + name: mongodb-kubernetes-init-ops-manager + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out +managedSecurityContext: false +mongodb: + appdbAssumeOldFormat: false + imageType: ubi9 + name: mongodb-enterprise-server + repo: quay.io/mongodb +multiCluster: + clusterClientTimeout: 10 + clusters: [] + kubeConfigSecretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + performFailOver: true +operator: + additionalArguments: [] + affinity: {} + baseName: mongodb-kubernetes + createOperatorServiceAccount: true + createResourcesServiceAccountsAndRoles: true + deployment_name: mongodb-kubernetes-operator + enableClusterMongoDBRoles: true + enablePVCResize: true + env: prod + maxConcurrentReconciles: 10 + mdbDefaultArchitecture: non-static + name: mongodb-kubernetes-operator + nodeSelector: {} + operator_image_name: mongodb-kubernetes + replicas: 1 + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + telemetry: + collection: + clusters: {} + deployments: {} + frequency: 1m + operators: {} + send: + enabled: false + frequency: 168h + tolerations: [] + vaultSecretBackend: + enabled: false + tlsSecretRef: "" +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + version: 68a83a6562a059000707301b +======== + version: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + watchedResources: + - mongodb + - opsmanagers + - mongodbusers + - mongodbcommunity + - mongodbsearch + webhook: + installClusterRole: true + registerConfiguration: true +opsManager: + name: mongodb-enterprise-ops-manager-ubi +readinessProbe: + name: mongodb-kubernetes-readinessprobe + version: 1.0.22 +registry: + agent: quay.io/mongodb + appDb: quay.io/mongodb/staging + database: quay.io/mongodb/staging + imagePullSecrets: prerelease-image-pullsecret + initAppDb: quay.io/mongodb/staging + initDatabase: quay.io/mongodb/staging + initOpsManager: quay.io/mongodb/staging + operator: quay.io/mongodb/staging + opsManager: quay.io/mongodb + pullPolicy: Always + readinessProbe: quay.io/mongodb + versionUpgradeHook: quay.io/mongodb +search: + community: +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + name: mongot/community + repo: 268558157000.dkr.ecr.eu-west-1.amazonaws.com + version: 1.53.0-78-g227f2593f +======== + name: mongodb-search + repo: quay.io/mongodb/staging + version: latest +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out +versionUpgradeHook: + name: mongodb-kubernetes-operator-version-upgrade-post-start-hook + version: 1.0.9 + +HOOKS: +MANIFEST: +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-appdb + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-database-pods + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-ops-manager + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/operator-sa.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +imagePullSecrets: + - name: prerelease-image-pullsecret +--- +# Source: mongodb-kubernetes/templates/operator-roles-clustermongodbroles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-cluster-mongodb-role +rules: + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - clustermongodbroles +--- +# Source: mongodb-kubernetes/templates/operator-roles-telemetry.yaml +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: mongodb-kubernetes/templates/operator-roles-clustermongodbroles.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-cluster-mongodb-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-kubernetes-operator-mongodb-cluster-mongodb-role +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-telemetry.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-kubernetes-operator-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-webhook.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-mongodb-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-kubernetes-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-appdb + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get +--- +# Source: mongodb-kubernetes/templates/operator-roles-base.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodbcommunity.mongodb.com + resources: + - mongodbcommunity + - mongodbcommunity/status + - mongodbcommunity/spec + - mongodbcommunity/finalizers + verbs: + - '*' + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodbsearch + - mongodbsearch/finalizers + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + - mongodbsearch/status +--- +# Source: mongodb-kubernetes/templates/operator-roles-pvc-resize.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-pvc-resize + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +--- +# Source: mongodb-kubernetes/templates/database-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-appdb + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-appdb +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-appdb + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-base.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-operator +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator-roles-pvc-resize.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator-pvc-resize-binding + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-kubernetes-operator-pvc-resize +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator + namespace: mongodb +--- +# Source: mongodb-kubernetes/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb-kubernetes-operator + namespace: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-kubernetes-operator + app.kubernetes.io/instance: mongodb-kubernetes-operator + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-kubernetes-operator + app.kubernetes.io/instance: mongodb-kubernetes-operator + spec: + serviceAccountName: mongodb-kubernetes-operator + securityContext: + runAsNonRoot: true + runAsUser: 2000 + imagePullSecrets: + - name: prerelease-image-pullsecret + containers: + - name: mongodb-kubernetes-operator +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + image: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes:68a83a6562a059000707301b" +======== + image: "quay.io/mongodb/staging/mongodb-kubernetes:1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2" +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + - -watch-resource=mongodbcommunity + - -watch-resource=mongodbsearch + - -watch-resource=clustermongodbroles + command: + - /usr/local/bin/mongodb-kubernetes-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: prod + - name: MDB_DEFAULT_ARCHITECTURE + value: non-static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1m" + - name: MDB_OPERATOR_TELEMETRY_SEND_ENABLED + value: "false" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/staging/mongodb-kubernetes-database + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/staging/mongodb-kubernetes-init-database + - name: INIT_DATABASE_VERSION +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + value: 68a83a6562a059000707301b + - name: DATABASE_VERSION + value: 68a83a6562a059000707301b +======== + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + - name: DATABASE_VERSION + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/staging/mongodb-kubernetes-init-ops-manager + - name: INIT_OPS_MANAGER_VERSION +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + value: 68a83a6562a059000707301b +======== + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/staging/mongodb-kubernetes-init-appdb + - name: INIT_APPDB_VERSION +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + value: 68a83a6562a059000707301b + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-agent:108.0.12.8846-1" +======== + value: 1.4.0-prerelease-68b5c0bb136a0d0007a4c8a2 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent:108.0.12.8846-1" +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: ubi9 + - name: PERFORM_FAILOVER + value: 'true' + - name: IMAGE_PULL_SECRETS + value: prerelease-image-pullsecret + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "10" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: mongodb-kubernetes-operator + # Community Env Vars Start + - name: MDB_COMMUNITY_AGENT_IMAGE + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-agent:108.0.2.8729-1" + - name: VERSION_UPGRADE_HOOK_IMAGE + value: "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9" + - name: READINESS_PROBE_IMAGE + value: "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22" + - name: MDB_COMMUNITY_IMAGE + value: "mongodb-community-server" + - name: MDB_COMMUNITY_REPO_URL + value: "quay.io/mongodb" + - name: MDB_COMMUNITY_IMAGE_TYPE + value: "ubi8" + # Community Env Vars End + - name: MDB_SEARCH_COMMUNITY_REPO_URL +<<<<<<<< HEAD:scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0100_install_operator.out + value: "268558157000.dkr.ecr.eu-west-1.amazonaws.com" + - name: MDB_SEARCH_COMMUNITY_NAME + value: "mongot/community" + - name: MDB_SEARCH_COMMUNITY_VERSION + value: "1.53.0-78-g227f2593f" +======== + value: "quay.io/mongodb/staging" + - name: MDB_SEARCH_COMMUNITY_NAME + value: "mongodb-search" + - name: MDB_SEARCH_COMMUNITY_VERSION + value: "latest" +>>>>>>>> fealebenpae/enterprise-search-snippets:scripts/code_snippets/tests/outputs/test_kind_search_community_snippets/01_0100_install_operator.out + diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0315_wait_for_community_resource.out b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0315_wait_for_community_resource.out new file mode 100644 index 000000000..97e4b8d32 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0315_wait_for_community_resource.out @@ -0,0 +1,13 @@ +Waiting for MongoDBCommunity resource to reach Running phase... +mongodbcommunity.mongodbcommunity.mongodb.com/mdbc-rs condition met + +MongoDBCommunity resource +NAME PHASE VERSION +mdbc-rs Running 8.0.10 + +Pods running in cluster kind-kind +NAME READY STATUS RESTARTS AGE +mdbc-rs-0 2/2 Running 0 2m32s +mdbc-rs-1 2/2 Running 0 96s +mdbc-rs-2 2/2 Running 0 48s +mongodb-kubernetes-operator-5d946f899b-fdfhp 1/1 Running 0 2m34s diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0330_wait_for_community_resource.out b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0330_wait_for_community_resource.out new file mode 100644 index 000000000..6971e0146 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0330_wait_for_community_resource.out @@ -0,0 +1,2 @@ +Waiting for MongoDBCommunity resource to reach Running phase... +mongodbcommunity.mongodbcommunity.mongodb.com/mdbc-rs condition met diff --git a/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0335_show_running_pods.out b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0335_show_running_pods.out new file mode 100644 index 000000000..65c9bbdd4 --- /dev/null +++ b/scripts/code_snippets/tests/outputs/test_kind_search_external_mongod_snippets/04_0335_show_running_pods.out @@ -0,0 +1,16 @@ + +MongoDBSearch resource +NAME PHASE AGE +mdbs Running 27s + +Search pods running in cluster kind-kind +NAME READY STATUS RESTARTS AGE +mdbs-search-0 1/1 Running 0 27s + +All pods in namespace mongodb +NAME READY STATUS RESTARTS AGE +mdbc-rs-0 2/2 Running 0 3m +mdbc-rs-1 2/2 Running 0 2m4s +mdbc-rs-2 2/2 Running 0 76s +mdbs-search-0 1/1 Running 0 27s +mongodb-kubernetes-operator-5d946f899b-fdfhp 1/1 Running 0 3m2s diff --git a/scripts/code_snippets/tests/test_kind_search_external_mongod_snippets.sh b/scripts/code_snippets/tests/test_kind_search_external_mongod_snippets.sh new file mode 100644 index 000000000..54f8fc3ac --- /dev/null +++ b/scripts/code_snippets/tests/test_kind_search_external_mongod_snippets.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eou pipefail +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + +source scripts/dev/set_env_context.sh + +script_name=$(readlink -f "${BASH_SOURCE[0]}") + +_SNIPPETS_OUTPUT_DIR="$(dirname "${script_name}")/outputs/$(basename "${script_name%.*}")" +export _SNIPPETS_OUTPUT_DIR +mkdir -p "${_SNIPPETS_OUTPUT_DIR}" + +dump_logs() { + if [[ "${SKIP_DUMP:-"false"}" != "true" ]]; then + scripts/evergreen/e2e/dump_diagnostic_information_from_all_namespaces.sh "${K8S_CTX}" + fi +} +trap dump_logs EXIT + +test_dir="./docs/search/04-search-external-mongod" +source "${test_dir}/env_variables.sh" +echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" +# shellcheck disable=SC1090 +test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" + +export MDB_RESOURCE_NAME="mdbc-rs" +export MDB_CONNECTION_STRING="mongodb://mdb-user:${MDB_USER_PASSWORD}@${MDB_RESOURCE_NAME}-0.${MDB_RESOURCE_NAME}-svc.${MDB_NS}.svc.cluster.local:27017/?replicaSet=${MDB_RESOURCE_NAME}" + +${test_dir}/test.sh + +test_dir="./docs/search/03-search-query-usage" +echo "Sourcing env variables for ${CODE_SNIPPETS_FLAVOR} flavor" +# shellcheck disable=SC1090 +test -f "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" && source "${test_dir}/env_variables_${CODE_SNIPPETS_FLAVOR}.sh" + +${test_dir}/test.sh From e5764ae6643346bccd404ffe95ed13a5b2bb1fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Fri, 12 Sep 2025 12:53:18 +0200 Subject: [PATCH 17/17] Updated crds --- .../mongodb.com_clustermongodbroles.yaml | 2 +- config/crd/bases/mongodb.com_mongodb.yaml | 48 +++++++- .../mongodb.com_mongodbmulticluster.yaml | 48 +++++++- .../crd/bases/mongodb.com_mongodbusers.yaml | 2 +- config/crd/bases/mongodb.com_opsmanagers.yaml | 112 ++++++++++++++++-- ...ommunity.mongodb.com_mongodbcommunity.yaml | 54 ++++++++- 6 files changed, 242 insertions(+), 24 deletions(-) diff --git a/config/crd/bases/mongodb.com_clustermongodbroles.yaml b/config/crd/bases/mongodb.com_clustermongodbroles.yaml index 9241b7dad..3d583bcfd 100644 --- a/config/crd/bases/mongodb.com_clustermongodbroles.yaml +++ b/config/crd/bases/mongodb.com_clustermongodbroles.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: clustermongodbroles.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_mongodb.yaml b/config/crd/bases/mongodb.com_mongodb.yaml index 2a7076877..d421d8837 100644 --- a/config/crd/bases/mongodb.com_mongodb.yaml +++ b/config/crd/bases/mongodb.com_mongodb.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodb.mongodb.com spec: group: mongodb.com @@ -1410,7 +1410,29 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic automationUserName: type: string clientCertificateSecretRef: @@ -1445,9 +1467,29 @@ spec: bindQueryUser: type: string caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml index 01fe3f2e6..3f1fa05c9 100644 --- a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml +++ b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodbmulticluster.mongodb.com spec: group: mongodb.com @@ -670,7 +670,29 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic automationUserName: type: string clientCertificateSecretRef: @@ -705,9 +727,29 @@ spec: bindQueryUser: type: string caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic servers: items: type: string diff --git a/config/crd/bases/mongodb.com_mongodbusers.yaml b/config/crd/bases/mongodb.com_mongodbusers.yaml index a81f0d449..89713ce7f 100644 --- a/config/crd/bases/mongodb.com_mongodbusers.yaml +++ b/config/crd/bases/mongodb.com_mongodbusers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: mongodbusers.mongodb.com spec: group: mongodb.com diff --git a/config/crd/bases/mongodb.com_opsmanagers.yaml b/config/crd/bases/mongodb.com_opsmanagers.yaml index c830b9a24..3ace001da 100644 --- a/config/crd/bases/mongodb.com_opsmanagers.yaml +++ b/config/crd/bases/mongodb.com_opsmanagers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 name: opsmanagers.mongodb.com spec: group: mongodb.com @@ -730,7 +730,30 @@ spec: automationLdapGroupDN: type: string automationPasswordSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic automationUserName: type: string clientCertificateSecretRef: @@ -766,9 +789,29 @@ spec: bindQueryUser: type: string caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ConfigMapKeySelector' description: Allows to point at a ConfigMap/key with a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic servers: items: type: string @@ -1193,7 +1236,29 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: array irsaEnabled: description: |- @@ -1263,7 +1328,29 @@ spec: CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets that apply to the associated S3 bucket. items: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: array irsaEnabled: description: |- @@ -1408,6 +1495,8 @@ spec: required: - spec type: object + required: + - members type: object clusterDomain: description: Cluster domain to override the default *.svc.cluster.local @@ -1447,13 +1536,13 @@ spec: Service when creating a ClusterIP type Service type: string externalTrafficPolicy: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local + type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. @@ -1464,12 +1553,12 @@ spec: format: int32 type: integer type: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP + type: string required: - type type: object @@ -1511,6 +1600,7 @@ spec: - spec type: object required: + - clusterName - members type: object type: array @@ -1536,13 +1626,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local + type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1552,12 +1642,12 @@ spec: format: int32 type: integer type: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP + type: string required: - type type: object @@ -1577,13 +1667,13 @@ spec: when creating a ClusterIP type Service type: string externalTrafficPolicy: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceExternalTrafficPolicy' description: |- ExternalTrafficPolicy mechanism to preserve the client source IP. Only supported on GCE and Google Kubernetes Engine. enum: - Cluster - Local + type: string loadBalancerIP: description: LoadBalancerIP IP that will be assigned to this LoadBalancer. type: string @@ -1593,12 +1683,12 @@ spec: format: int32 type: integer type: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0ServiceType' description: Type of the `Service` to be created. enum: - LoadBalancer - NodePort - ClusterIP + type: string required: - type type: object diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index a0004e22e..36d5c892d 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.18.0 service.binding: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret service.binding/connectionString: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv service.binding/password: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password @@ -330,13 +330,24 @@ spec: authentication: properties: agentCertificateSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- AgentCertificateSecret is a reference to a Secret containing the certificate and the key for the automation agent The secret needs to have available: - certificate under key: "tls.crt" - private key under key: "tls.key" If additionally, tls.pem is present, then it needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic agentMode: description: AgentMode contains the authentication mode used by the automation agent. @@ -455,24 +466,57 @@ spec: communication properties: caCertificateSecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic caConfigMapRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates The certificate is expected to be available under the key "ca.crt" This field is ignored when CaCertificateSecretRef is configured + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic certificateKeySecretRef: - $ref: '#/definitions/k8s.io~1api~1core~1v1~0LocalObjectReference' description: |- CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic enabled: type: boolean optional: