From 8260cde5ae58be6d49b04fe1727d7ec5242f0baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Tue, 28 Jul 2026 08:47:32 +0000 Subject: [PATCH 1/5] Document lifecycle problem --- ...essionQueryTestBeforeAfterHooksSuite.scala | 24 ++++++++++ ...essionQueryTestBeforeAfterHooksSuite.scala | 47 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/SessionQueryTestBeforeAfterHooksSuite.scala diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala new file mode 100644 index 0000000000000..c0bc0421e1cf4 --- /dev/null +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connect + +import org.apache.spark.sql + +class SessionQueryTestBeforeAfterHooksSuite + extends sql.SessionQueryTestBeforeAfterHooksSuite + with SessionQueryTest diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SessionQueryTestBeforeAfterHooksSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SessionQueryTestBeforeAfterHooksSuite.scala new file mode 100644 index 0000000000000..f3253def7113a --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/SessionQueryTestBeforeAfterHooksSuite.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +class SessionQueryTestBeforeAfterHooksSuite extends SessionQueryTest { + + override protected def beforeAll(): Unit = { + super.beforeAll() + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + } + + override protected def beforeEach(): Unit = { + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + super.beforeEach() + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + } + + override protected def afterEach(): Unit = { + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + super.afterEach() + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + } + + override protected def afterAll(): Unit = { + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + super.afterAll() + } + + test("assert spark is available in BeforeAndAfter hooks") { + checkAnswer(spark.sql("SELECT 1"), Seq(Row(1))) + } +} From 13c38f0ea28aaf6b097b5f58b00c4e0568c7798d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Tue, 28 Jul 2026 15:36:50 +0000 Subject: [PATCH 2/5] Fix connect.SessionQueryTest beforeAll/afterAll lifecycle --- .../org/apache/spark/sql/connect/SparkSessionBinder.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala index b30bc55f7df14..c3486f3df704d 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala @@ -38,7 +38,8 @@ trait SparkSessionBinder extends sql.SparkSessionBinder { self: SparkFunSuite => protected override def spark: SparkSession = _connectSpark override protected def beforeAll(): Unit = { - super.beforeAll() + initializeSession() + // Other suites using mocks leave a mess in the global executionManager, // shut it down so that it's cleared before starting server. SparkConnectService.executionManager.shutdown() @@ -60,14 +61,15 @@ trait SparkSessionBinder extends sql.SparkSessionBinder { self: SparkFunSuite => .builder() .client(client) .create() + super.beforeAll() } override def afterAll(): Unit = { + super.afterAll() if (_connectSpark != null) { _connectSpark.close() _connectSpark = null } SparkConnectService.stop() - super.afterAll() } } From 8932c97ff5c7d6e3492bd37408cdd711619d40d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Wed, 5 Aug 2026 23:45:29 +0200 Subject: [PATCH 3/5] fix format --- .../sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala index c0bc0421e1cf4..bb1818fe089a9 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala @@ -21,4 +21,4 @@ import org.apache.spark.sql class SessionQueryTestBeforeAfterHooksSuite extends sql.SessionQueryTestBeforeAfterHooksSuite - with SessionQueryTest + with SessionQueryTest From 7d609bb9e714844e31dbe6bd03a5c12313cc83af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Thu, 6 Aug 2026 08:36:08 +0000 Subject: [PATCH 4/5] fixup! fix format --- .../sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala index bb1818fe089a9..9cc30139859bd 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SessionQueryTestBeforeAfterHooksSuite.scala @@ -20,5 +20,5 @@ package org.apache.spark.sql.connect import org.apache.spark.sql class SessionQueryTestBeforeAfterHooksSuite - extends sql.SessionQueryTestBeforeAfterHooksSuite - with SessionQueryTest + extends sql.SessionQueryTestBeforeAfterHooksSuite + with SessionQueryTest From 68a95a759ccf391901b0f7e0daa5bd1907586ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthis=20G=C3=B6rdel?= Date: Fri, 7 Aug 2026 13:05:08 +0000 Subject: [PATCH 5/5] Shutdown SC service in finally block --- .../spark/sql/connect/SparkSessionBinder.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala index c3486f3df704d..fea98122ed8ae 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkSessionBinder.scala @@ -65,11 +65,14 @@ trait SparkSessionBinder extends sql.SparkSessionBinder { self: SparkFunSuite => } override def afterAll(): Unit = { - super.afterAll() - if (_connectSpark != null) { - _connectSpark.close() - _connectSpark = null + try { + super.afterAll() + } finally { + if (_connectSpark != null) { + _connectSpark.close() + _connectSpark = null + } + SparkConnectService.stop() } - SparkConnectService.stop() } }