diff --git a/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.cc b/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.cc
index 34346502aa02..19d5e3864b5c 100644
--- a/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.cc
+++ b/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.cc
@@ -146,12 +146,17 @@ std::wstring ODBCRemoteTestBase::GetQueryAllDataTypes() {
 
 void ODBCRemoteTestBase::SetUp() {
   if (arrow::internal::GetEnvVar(kTestConnectStr.data()).ValueOr("").empty()) {
+    skipping_test_ = true;
     GTEST_SKIP() << "Skipping test: kTestConnectStr not set";
   }
 }
 
 void FlightSQLODBCRemoteTestBase::SetUp() {
   ODBCRemoteTestBase::SetUp();
+  if (skipping_test_) {
+    return;
+  }
+
   this->Connect();
   connected_ = true;
 }
@@ -165,6 +170,10 @@ void FlightSQLODBCRemoteTestBase::TearDown() {
 
 void FlightSQLOdbcV2RemoteTestBase::SetUp() {
   ODBCRemoteTestBase::SetUp();
+  if (skipping_test_) {
+    return;
+  }
+
   this->Connect(SQL_OV_ODBC2);
   connected_ = true;
 }
diff --git a/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.h b/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.h
index d02ed6cce964..5cac67d810c4 100644
--- a/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.h
+++ b/cpp/src/arrow/flight/sql/odbc/tests/odbc_test_suite.h
@@ -77,6 +77,8 @@ class ODBCRemoteTestBase : public ::testing::Test {
 
  protected:
   void SetUp() override;
+
+  bool skipping_test_ = false;
 };
 
 /// \brief Base test fixture for running tests against a remote server.