From fc2ff8cb1dc4ce6664af45587fcb9bfd29505a50 Mon Sep 17 00:00:00 2001 From: Timma Reddy Kunduru Date: Tue, 11 Nov 2025 02:43:51 -0800 Subject: [PATCH 1/2] Added code to pass the spark config required for some of the internal services --- service_configuration_lib/spark_config.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 1536ae4..3c25e11 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -920,6 +920,7 @@ def _append_event_log_conf( spark_opts['spark.eventLog.enabled'] = 'false' return spark_opts + spark_opts['aws_account_id'] = account_id for conf in self.spark_srv_conf.get('environments', {}).values(): if account_id == conf['account_id']: spark_opts['spark.eventLog.enabled'] = 'true' @@ -1262,6 +1263,27 @@ def get_spark_conf( if aws_creds: spark_conf = _append_aws_credentials_conf(spark_conf, *aws_creds, aws_region) + # gather all data needed by our internal Spark configuration service + scs_args = { + 'cluster_manager': cluster_manager, + 'spark_app_base_name': spark_app_base_name, + 'docker_image': docker_img, + 'user_spark_opts': user_spark_opts, + 'paasta_cluster': paasta_cluster, + 'paasta_pool': paasta_pool, + 'paasta_service': paasta_service, + 'paasta_instance': paasta_instance, + 'extra_volumes': extra_volumes, + 'force_spark_resource_configs': force_spark_resource_configs, + 'k8s_server_address': k8s_server_address, + 'jira_ticket': jira_ticket, + 'service_account_name': service_account_name, + 'ui_port': int(ui_port), + 'user': os.environ.get('USER'), + 'aws_account_id': spark_conf.pop('aws_account_id', None), + } + spark_conf['scs_conf'] = json.dumps(scs_args, indent=4) + return spark_conf From 687606e22f08dfa83329f8a4b63bb73ba733d605 Mon Sep 17 00:00:00 2001 From: Timma Reddy Kunduru Date: Tue, 11 Nov 2025 09:48:50 -0800 Subject: [PATCH 2/2] Fixed the failing test by adding scs_conf to the keys list --- tests/spark_config_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/spark_config_test.py b/tests/spark_config_test.py index 13edbc0..fca6edb 100644 --- a/tests/spark_config_test.py +++ b/tests/spark_config_test.py @@ -1333,7 +1333,8 @@ def test_leaders_get_spark_conf_kubernetes( list(mock_append_spark_prometheus_conf.return_value.keys()) + list(mock_append_event_log_conf.return_value.keys()) + list(mock_append_aws_credentials_conf.return_value.keys()) + - list(mock_append_sql_partitions_conf.return_value.keys()), + list(mock_append_sql_partitions_conf.return_value.keys()) + + ['scs_conf'], # services config key ) assert set(output.keys()) == verified_keys mock_adjust_spark_requested_resources_kubernetes.mocker.assert_called_once_with( @@ -1464,7 +1465,8 @@ def test_local_spark( list(mock_adjust_spark_requested_resources_kubernetes.return_value.keys()) + list(mock_get_dra_configs.return_value.keys()) + list(mock_append_aws_credentials_conf.return_value.keys()) + - list(mock_append_sql_partitions_conf.return_value.keys()), + list(mock_append_sql_partitions_conf.return_value.keys()) + + ['scs_conf'], # services config key ) assert set(output.keys()) == verified_keys mock_append_event_log_conf.mocker.assert_called_once_with(