Skip to content

spark clustername fix #100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sanketjadhavSF merged 3 commits into from
Apr 6, 2026
Merged

spark clustername fix #100

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6dfa3c9
Add ClusterName field to clusterContext and update related log messages
sanketjadhavSF Apr 6, 2026
9639b55
remove test changes
sanketjadhavSF Apr 6, 2026
9637968
Rename ClusterName to KubernetesClusterName in clusterContext for cla…
sanketjadhavSF Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions internal/pkg/object/command/sparkeks/sparkeks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ type clusterContext struct {
Properties map[string]string `yaml:"properties,omitempty" json:"properties,omitempty"`
Image *string `yaml:"image,omitempty" json:"image,omitempty"`
Region *string `yaml:"region,omitempty" json:"region,omitempty"`
KubernetesClusterName *string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"`
SparkApplicationFile string `yaml:"spark_application_file,omitempty" json:"spark_application_file,omitempty"`
RequiredSparkSQLExtensions string `yaml:"required_spark_sql_extensions,omitempty" json:"required_spark_sql_extensions,omitempty"`
}
Expand Down Expand Up @@ -563,7 +564,11 @@ func createSparkClients(ctx context.Context, execCtx *executionContext) error {
execCtx.kubeClient = kubeClient

if execCtx.runtime != nil && execCtx.runtime.Stdout != nil {
execCtx.runtime.Stdout.WriteString(fmt.Sprintf("Successfully created Spark Operator and Kubernetes clients for cluster: %s\n", execCtx.cluster.Name))
clusterName := ""
if execCtx.clusterContext.KubernetesClusterName != nil {
clusterName = *execCtx.clusterContext.KubernetesClusterName
}
execCtx.runtime.Stdout.WriteString(fmt.Sprintf("Successfully created Spark Operator and Kubernetes clients for cluster: %s\n", clusterName))
}
return nil
}
Expand All @@ -590,10 +595,15 @@ func updateKubeConfig(ctx context.Context, execCtx *executionContext) (string, e
kubeconfigPath := tmpfile.Name()
tmpfile.Close() // Close the file so `aws` can write to it

clusterName := ""
if execCtx.clusterContext.KubernetesClusterName != nil {
clusterName = *execCtx.clusterContext.KubernetesClusterName
}

args := []string{
"eks", "update-kubeconfig",
"--region", region,
"--name", execCtx.cluster.Name,
"--name", clusterName,
"--kubeconfig", kubeconfigPath,
}
if roleArn != "" {
Expand Down Expand Up @@ -671,7 +681,7 @@ func applySparkOperatorConfig(execCtx *executionContext) {
// Add default spark properties
sparkApp.Spec.SparkConf[sparkAppNameProperty] = execCtx.appName

// Set spark event log directory for spark history server
// Set spark event log directory for spark history server
if execCtx.commandContext.EventLogURI != "" {
eventLogURI := updateS3ToS3aURI(execCtx.commandContext.EventLogURI)
sparkApp.Spec.SparkConf[sparkEventLogDirProperty] = eventLogURI
Expand Down
Loading