diff --git a/CHANGELOG.md b/CHANGELOG.md index 274d4d65..ab0134e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. +- BREAKING: Added listener support for Superset ([#625]). ### Changed @@ -31,6 +32,7 @@ [#615]: https://github.com/stackabletech/superset-operator/pull/615 [#617]: https://github.com/stackabletech/superset-operator/pull/617 [#623]: https://github.com/stackabletech/superset-operator/pull/623 +[#625]: https://github.com/stackabletech/superset-operator/pull/625 [#628]: https://github.com/stackabletech/superset-operator/pull/628 ## [25.3.0] - 2025-03-21 diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index 686550e4..0a449df7 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -129,23 +129,6 @@ spec: credentialsSecret: description: The name of the Secret object containing the admin user credentials and database connection details. Read the [getting started guide first steps](https://docs.stackable.tech/home/nightly/superset/getting_started/first_steps) to find out more. type: string - listenerClass: - default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this SupersetCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - * external-stable: Use a LoadBalancer service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - enum: - - cluster-internal - - external-unstable - - external-stable - type: string mapboxSecret: description: The name of a Secret object. The Secret should contain a key `connections.mapboxApiKey`. This is the API key required for map charts to work that use mapbox. The token should be in the JWT format. nullable: true @@ -251,6 +234,10 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + nullable: true + type: string logging: default: containers: {} @@ -479,6 +466,10 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + nullable: true + type: string logging: default: containers: {} diff --git a/deploy/helm/superset-operator/templates/roles.yaml b/deploy/helm/superset-operator/templates/roles.yaml index 02c97400..939df91b 100644 --- a/deploy/helm/superset-operator/templates/roles.yaml +++ b/deploy/helm/superset-operator/templates/roles.yaml @@ -125,6 +125,17 @@ rules: - bind resourceNames: - {{ include "operator.name" . }}-clusterrole + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get + - list + - watch + - patch + - create + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/docs/modules/superset/pages/usage-guide/listenerclass.adoc b/docs/modules/superset/pages/usage-guide/listenerclass.adoc index 29fec9ea..ebae1266 100644 --- a/docs/modules/superset/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/superset/pages/usage-guide/listenerclass.adoc @@ -1,18 +1,15 @@ = Service exposition with ListenerClasses -:description: Superset service exposition with ListenerClass: configure access via internal, external-unstable, or external-stable services. +:description: Configure the Superset service exposure with listener classes: cluster-internal, external-unstable, or external-stable. Apache Superset offers a web UI and an API. -The Operator deploys a service called `-external` (where `` is the name of the SupersetCluster) through which Superset can be reached. - -This service can have three different types: `cluster-internal`, `external-unstable` and `external-stable`. -Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. - -This is how the ListenerClass is configured: +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Nodes pod. +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.config.listenerClass`: [source,yaml] ---- spec: - clusterConfig: - listenerClass: cluster-internal # <1> + nodes: + config: + listenerClass: external-stable # <1> ---- -<1> The default `cluster-internal` setting. +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 31eb9b70..a0a83821 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -16,8 +16,7 @@ use stackable_operator::{ }, }, config::{ - fragment, - fragment::{Fragment, ValidationError}, + fragment::{self, Fragment, ValidationError}, merge::Merge, }, k8s_openapi::apimachinery::pkg::api::resource::Quantity, @@ -48,6 +47,14 @@ pub const MAX_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { unit: BinaryMultiple::Mebi, }; +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + +pub const APP_PORT_NAME: &str = "http"; +pub const APP_PORT: u16 = 8088; +pub const METRICS_PORT_NAME: &str = "metrics"; +pub const METRICS_PORT: u16 = 9102; + const DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(2); #[derive(Debug, Snafu)] @@ -57,6 +64,12 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, + + #[snafu(display("Configuration/Executor conflict!"))] + NoRoleForExecutorFailure, + + #[snafu(display("object has no associated namespace"))] + NoNamespace, } #[derive(Display, EnumIter, EnumString)] @@ -157,20 +170,6 @@ pub mod versioned { #[serde(default)] pub cluster_operation: ClusterOperation, - /// This field controls which type of Service the Operator creates for this SupersetCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// * external-stable: Use a LoadBalancer service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: v1alpha1::CurrentlySupportedListenerClasses, - /// The name of a Secret object. /// The Secret should contain a key `connections.mapboxApiKey`. /// This is the API key required for map charts to work that use mapbox. @@ -224,21 +223,10 @@ pub mod versioned { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, - } - - // TODO: Temporary solution until listener-operator is finished - #[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] - #[serde(rename_all = "PascalCase")] - pub enum CurrentlySupportedListenerClasses { - #[default] - #[serde(rename = "cluster-internal")] - ClusterInternal, - #[serde(rename = "external-unstable")] - ExternalUnstable, - - #[serde(rename = "external-stable")] - ExternalStable, + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default)] + pub listener_class: String, } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -406,18 +394,6 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { } } -impl v1alpha1::CurrentlySupportedListenerClasses { - pub fn k8s_service_type(&self) -> String { - match self { - v1alpha1::CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - v1alpha1::CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - v1alpha1::CurrentlySupportedListenerClasses::ExternalStable => { - "LoadBalancer".to_string() - } - } - } -} - impl v1alpha1::SupersetConfig { pub const CREDENTIALS_SECRET_PROPERTY: &'static str = "credentialsSecret"; pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; @@ -440,6 +416,7 @@ impl v1alpha1::SupersetConfig { graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), row_limit: None, webserver_timeout: None, + listener_class: Some("cluster-internal".to_owned()), } } } @@ -505,17 +482,19 @@ impl HasStatusCondition for v1alpha1::SupersetCluster { } impl v1alpha1::SupersetCluster { + /// The name of the group-listener provided for a specific role-group. + /// The UI will use this group listener so that only one load balancer + /// is needed (per role group). + pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { + rolegroup.object_name() + } + pub fn get_role(&self, role: &SupersetRole) -> Option<&Role> { match role { SupersetRole::Node => self.spec.nodes.as_ref(), } } - /// The name of the role-level load-balanced Kubernetes `Service` - pub fn node_role_service_name(&self) -> Option { - self.metadata.name.clone() - } - /// Metadata about a node rolegroup pub fn node_rolegroup_ref( &self, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 7cbef3c2..5dd88051 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -52,7 +52,6 @@ mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } -pub const APP_PORT: u16 = 8088; pub const OPERATOR_NAME: &str = "superset.stackable.tech"; #[derive(Parser)] diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 41e9b126..893ab7f6 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -20,13 +20,21 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, + }, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ - authentication::oidc, product_image_selection::ResolvedProductImage, + authentication::oidc, + listener::{Listener, ListenerPort, ListenerSpec}, + product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, }, k8s_openapi::{ @@ -68,18 +76,19 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - APP_PORT, OPERATOR_NAME, + OPERATOR_NAME, authorization::opa::{OPA_IMPORTS, SupersetOpaConfigResolved}, commands::add_cert_to_python_certifi_command, config::{self, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ - APP_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, SupersetRole, + APP_NAME, APP_PORT, APP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, + METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, SupersetRole, authentication::{ SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, }, - v1alpha1::{Container, SupersetCluster, SupersetClusterStatus, SupersetConfig}, + v1alpha1::{self, Container, SupersetCluster, SupersetClusterStatus, SupersetConfig}, }, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{LOG_CONFIG_FILE, extend_config_map_with_log_config}, @@ -91,9 +100,6 @@ pub const SUPERSET_FULL_CONTROLLER_NAME: &str = concatcp!(SUPERSET_CONTROLLER_NAME, '.', OPERATOR_NAME); pub const DOCKER_IMAGE_BASE_NAME: &str = "superset"; -const METRICS_PORT_NAME: &str = "metrics"; -const METRICS_PORT: i32 = 9102; - pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, @@ -286,6 +292,17 @@ pub enum Error { InvalidOpaConfig { source: stackable_operator::commons::opa::Error, }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, + + #[snafu(display("failed to apply group listener for {rolegroup}"))] + ApplyGroupListener { + source: stackable_operator::cluster_resources::Error, + rolegroup: RoleGroupRef, + }, } type Result = std::result::Result; @@ -389,12 +406,6 @@ pub async fn reconcile_superset( .await .context(ApplyRoleBindingSnafu)?; - let node_role_service = build_node_role_service(superset, &resolved_product_image)?; - cluster_resources - .add(client, node_role_service) - .await - .context(ApplyRoleServiceSnafu)?; - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); for (rolegroup_name, rolegroup_config) in role_node_config.iter() { @@ -425,6 +436,20 @@ pub async fn reconcile_superset( &rbac_sa.name_any(), &config, )?; + + let rg_group_listener = build_group_listener( + superset, + &resolved_product_image, + &rolegroup, + config.listener_class, + )?; + cluster_resources + .add(client, rg_group_listener) + .await + .with_context(|_| ApplyGroupListenerSnafu { + rolegroup: rolegroup.clone(), + })?; + cluster_resources .add(client, rg_service) .await @@ -482,56 +507,6 @@ pub async fn reconcile_superset( Ok(Action::await_change()) } -/// The server-role service is the primary endpoint that should be used by clients that do not perform internal load balancing, -/// including targets outside of the cluster. -fn build_node_role_service( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, -) -> Result { - let role_name = SupersetRole::Node.to_string(); - let role_svc_name = superset - .node_role_service_name() - .context(GlobalServiceNameNotFoundSnafu)?; - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(format!("{}-external", &role_svc_name)) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &role_name, - "global", - )) - .context(MetadataBuildSnafu)? - .build(), - spec: Some(ServiceSpec { - type_: Some( - superset - .spec - .cluster_config - .listener_class - .k8s_service_type(), - ), - ports: Some(vec![ServicePort { - name: Some("http".to_string()), - port: APP_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]), - selector: Some( - Labels::role_selector(superset, APP_NAME, &role_name) - .context(LabelBuildSnafu)? - .into(), - ), - ..ServiceSpec::default() - }), - status: None, - }) -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( @@ -646,55 +621,92 @@ fn build_node_rolegroup_service( resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(vec![ - ServicePort { - name: Some("http".to_string()), - port: APP_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ServicePort { - name: Some(METRICS_PORT_NAME.into()), - port: METRICS_PORT, - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ]), - selector: Some( - Labels::role_group_selector( - superset, - APP_NAME, - &rolegroup.role, - &rolegroup.role_group, - ) + let metadata = ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(format!("{name}-metrics", name = rolegroup.object_name())) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) + .build(); + + let spec = Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_owned()), + cluster_ip: Some("None".to_owned()), + ports: Some(vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_owned()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_owned()), + ..ServicePort::default() + }]), + selector: Some( + Labels::role_group_selector(superset, APP_NAME, &rolegroup.role, &rolegroup.role_group) .context(LabelBuildSnafu)? .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }); + + let service = Service { + metadata, + spec, status: None, - }) + }; + + Ok(service) +} + +pub fn build_group_listener( + superset: &v1alpha1::SupersetCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, + listener_class: String, +) -> Result { + let metadata = ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(superset.group_listener_name(rolegroup)) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(); + + let spec = ListenerSpec { + class_name: Some(listener_class), + ports: Some(listener_ports()), + ..Default::default() + }; + + let listener = Listener { + metadata, + spec, + status: None, + }; + + Ok(listener) +} + +fn listener_ports() -> Vec { + vec![ListenerPort { + name: APP_PORT_NAME.to_owned(), + port: APP_PORT.into(), + protocol: Some("TCP".to_owned()), + }] } /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. @@ -717,14 +729,26 @@ fn build_server_rolegroup_statefulset( .get(&rolegroup_ref.role_group) .context(NoNodeRoleSnafu)?; + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; + let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels.clone()) .context(MetadataBuildSnafu)? .build(); @@ -850,6 +874,21 @@ fn build_server_rolegroup_statefulset( superset_cb.readiness_probe(probe.clone()); superset_cb.liveness_probe(probe); + // listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses and + // that it is possible to connect to a consistent address + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(superset.group_listener_name(rolegroup_ref)), + &unversioned_recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_owned()) + .context(BuildListenerVolumeSnafu)?; + + superset_cb + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + pb.add_container(superset_cb.build()); add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; @@ -870,7 +909,7 @@ fn build_server_rolegroup_statefulset( /stackable/statsd_exporter & wait_for_termination $! "}]) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) .resources( ResourceRequirementsBuilder::new() .with_cpu_request("100m") @@ -928,13 +967,7 @@ fn build_server_rolegroup_statefulset( .name(rolegroup_ref.object_name()) .ownerreference_from_resource(superset, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(MetadataBuildSnafu)? .with_label( Label::try_from(("restarter.stackable.tech/enabled", "true")) @@ -960,6 +993,7 @@ fn build_server_rolegroup_statefulset( }, service_name: rolegroup_ref.object_name(), template: pod_template, + volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() }), status: None, diff --git a/tests/templates/kuttl/external-access/00-limit-range.yaml b/tests/templates/kuttl/external-access/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/10-assert.yaml b/tests/templates/kuttl/external-access/10-assert.yaml new file mode 100644 index 00000000..e9c60b15 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/10-install-postgresql.yaml b/tests/templates/kuttl/external-access/10-install-postgresql.yaml new file mode 100644 index 00000000..50c5ad67 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-postgresql + --namespace $NAMESPACE + --version 12.5.6 + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/external-access/20-listener-classes.yaml b/tests/templates/kuttl/external-access/20-listener-classes.yaml new file mode 100644 index 00000000..893032c5 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-listener-classes.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml new file mode 100644 index 00000000..7c33f457 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-superset +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-node-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-node-external-unstable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-node-cluster-internal +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-node +status: + expectedPods: 4 + currentHealthy: 4 + disruptionsAllowed: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: superset-node-cluster-internal +spec: + type: ClusterIP # cluster-internal +--- +apiVersion: v1 +kind: Service +metadata: + name: superset-node-default +spec: + type: NodePort # external-stable +--- +apiVersion: v1 +kind: Service +metadata: + name: superset-node-external-unstable +spec: + type: NodePort # external-unstable diff --git a/tests/templates/kuttl/external-access/30-install-superset.yaml b/tests/templates/kuttl/external-access/30-install-superset.yaml new file mode 100644 index 00000000..fbb502ee --- /dev/null +++ b/tests/templates/kuttl/external-access/30-install-superset.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: > + envsubst < install-superset.yaml | + kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..7991d27e --- /dev/null +++ b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,31 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: superset + password: superset + database: superset diff --git a/tests/templates/kuttl/external-access/install-superset.yaml.j2 b/tests/templates/kuttl/external-access/install-superset.yaml.j2 new file mode 100644 index 00000000..439547f5 --- /dev/null +++ b/tests/templates/kuttl/external-access/install-superset.yaml.j2 @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-credentials +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Superset + adminUser.lastname: Admin + adminUser.email: admin@superset.com + adminUser.password: admin + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: superset.stackable.tech/v1alpha1 +kind: SupersetCluster +metadata: + name: superset +spec: + image: +{% if test_scenario['values']['superset'].find(",") > 0 %} + custom: "{{ test_scenario['values']['superset'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['superset'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['superset'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + credentialsSecret: superset-credentials + nodes: + config: + listenerClass: test-external-stable-$NAMESPACE + roleGroups: + default: + replicas: 2 + external-unstable: + replicas: 1 + config: + listenerClass: test-external-unstable-$NAMESPACE + cluster-internal: + replicas: 1 + config: + listenerClass: test-cluster-internal-$NAMESPACE diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml new file mode 100644 index 00000000..4131526a --- /dev/null +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal-$NAMESPACE +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable-$NAMESPACE +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/opa/50_get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py index 7765d526..eabb0ab9 100644 --- a/tests/templates/kuttl/opa/50_get_user_roles.py +++ b/tests/templates/kuttl/opa/50_get_user_roles.py @@ -111,8 +111,10 @@ def main(): global bearer_token global csrf_token - base_ui_url = f"http://superset-external.{namespace}.svc.cluster.local:8088" - base_api_url = f"http://superset-external.{namespace}.svc.cluster.local:8088/api/v1" + base_ui_url = f"http://superset-node-default.{namespace}.svc.cluster.local:8088" + base_api_url = ( + f"http://superset-node-default.{namespace}.svc.cluster.local:8088/api/v1" + ) bearer_token = get_bearer_token() csrf_token = get_csrf_token() diff --git a/tests/templates/kuttl/resources/20-assert.yaml.j2 b/tests/templates/kuttl/resources/20-assert.yaml.j2 index d0eaeed5..1aafc064 100644 --- a/tests/templates/kuttl/resources/20-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/20-assert.yaml.j2 @@ -32,7 +32,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: superset-node-resources-from-role-group + name: superset-node-resources-from-rolegroup spec: template: spec: diff --git a/tests/templates/kuttl/resources/20-install-superset.yaml.j2 b/tests/templates/kuttl/resources/20-install-superset.yaml.j2 index 5196e307..99a5bbee 100644 --- a/tests/templates/kuttl/resources/20-install-superset.yaml.j2 +++ b/tests/templates/kuttl/resources/20-install-superset.yaml.j2 @@ -48,7 +48,7 @@ spec: roleGroups: resources-from-role: replicas: 1 - resources-from-role-group: + resources-from-rolegroup: replicas: 1 config: resources: diff --git a/tests/templates/kuttl/smoke/metrics.py b/tests/templates/kuttl/smoke/metrics.py index 6c86f2f2..3f86b54c 100644 --- a/tests/templates/kuttl/smoke/metrics.py +++ b/tests/templates/kuttl/smoke/metrics.py @@ -8,7 +8,7 @@ # Wait for the counter to be consumed by the statsd-exporter time.sleep(2) -metrics_response = requests.get("http://superset-node-default:9102/metrics") +metrics_response = requests.get("http://superset-node-default-metrics:9102/metrics") assert metrics_response.status_code == 200, "Metrics could not be retrieved." assert "superset_welcome" in metrics_response.text, ( diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 3ce36205..87d2b0f0 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -60,6 +60,10 @@ tests: dimensions: - superset - openshift + - name: external-access + dimensions: + - superset + - openshift suites: - name: nightly patch: