Skip to content

Commit d26d11d

Browse files
authored
acc: Run run-local tests on fixed ports and kill any running processes on these ports after the tests (#3510)
## Changes Run `run-local` tests on fixed ports and kill any running processes on these ports beforehand ## Why This makes the tests more stable, ran 7 times in a row with no failures.
1 parent a062f13 commit d26d11d

File tree

9 files changed

+148
-56
lines changed

9 files changed

+148
-56
lines changed

acceptance/bin/allocate_ports.py

Lines changed: 0 additions & 22 deletions
This file was deleted.

acceptance/bin/kill_port.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Cross-platform script to kill processes using specific ports.
4+
Usage: kill_port.py <port1> [port2] [port3] ...
5+
"""
6+
7+
import sys
8+
import subprocess
9+
import platform
10+
import re
11+
12+
13+
def kill_processes_on_port(port):
14+
"""Kill all processes using the specified port."""
15+
system = platform.system().lower()
16+
17+
try:
18+
if system in ["linux", "darwin"]: # Linux or macOS
19+
# Try lsof first (most reliable)
20+
try:
21+
result = subprocess.run(
22+
["lsof", "-ti", f":{port}"],
23+
capture_output=True,
24+
text=True,
25+
check=False,
26+
)
27+
if result.returncode == 0 and result.stdout.strip():
28+
pids = result.stdout.strip().split("\n")
29+
for pid in pids:
30+
if pid.strip():
31+
subprocess.run(["kill", "-9", pid.strip()], check=False)
32+
except FileNotFoundError:
33+
# Fall back to netstat if lsof not available
34+
result = subprocess.run(["netstat", "-tlnp"], capture_output=True, text=True, check=False)
35+
if result.returncode == 0:
36+
for line in result.stdout.split("\n"):
37+
if f":{port} " in line:
38+
# Extract PID from output like "tcp 0 0 :::8080 :::* LISTEN 12345/python"
39+
match = re.search(r"(\d+)/", line)
40+
if match:
41+
pid = match.group(1)
42+
subprocess.run(["kill", "-9", pid], check=False)
43+
44+
elif system == "windows":
45+
# Windows using netstat and taskkill
46+
result = subprocess.run(["netstat", "-ano"], capture_output=True, text=True, check=False)
47+
if result.returncode == 0:
48+
for line in result.stdout.split("\n"):
49+
if f":{port} " in line and "LISTENING" in line:
50+
# Extract PID from last column
51+
parts = line.split()
52+
if len(parts) >= 5:
53+
pid = parts[-1]
54+
if pid.isdigit():
55+
subprocess.run(
56+
["taskkill", "/PID", pid, "/F"],
57+
check=False,
58+
capture_output=True,
59+
)
60+
61+
except Exception as e:
62+
# Silently continue - port cleanup is best effort
63+
pass
64+
65+
66+
def main():
67+
if len(sys.argv) < 2:
68+
print("Usage: kill_port.py <port1> [port2] [port3] ...")
69+
sys.exit(1)
70+
71+
for port_str in sys.argv[1:]:
72+
try:
73+
port = int(port_str)
74+
kill_processes_on_port(port)
75+
except ValueError:
76+
print(f"Warning: Invalid port number '{port_str}'")
77+
78+
79+
if __name__ == "__main__":
80+
main()
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Local = false
1+
Local = true
22
Cloud = false
33

44
[EnvMatrix]
5-
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
5+
DATABRICKS_CLI_DEPLOYMENT = ["direct-exp"]

acceptance/cmd/workspace/apps/run-local-node/script

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,16 @@ cd app
44
# so we don't need to start it in background. It will install the dependencies as part of the command
55
trace $CLI apps run-local --prepare-environment --entry-point test.yml 2>&1 | grep -w "Hello, world"
66

7-
# Get 3 unique ports sequentially to avoid conflicts
8-
PORTS=$(allocate_ports.py 3 | tr -d '\r')
7+
PORT=8080
8+
DEBUG_PORT=5252
9+
PROXY_PORT=8081
910

10-
# Read ports into array
11-
PORTS_ARR=($(echo "$PORTS"))
12-
PORT="${PORTS_ARR[0]}"
13-
DEBUG_PORT="${PORTS_ARR[1]}"
14-
PROXY_PORT="${PORTS_ARR[2]}"
11+
cleanup() {
12+
# Kill any still running processes on these ports when the script exits
13+
kill_port.py $PORT $DEBUG_PORT $PROXY_PORT
14+
}
15+
16+
trap cleanup EXIT
1517

1618
title "Starting the app in background..."
1719
trace $CLI apps run-local --prepare-environment --debug --port "$PROXY_PORT" --debug-port "$DEBUG_PORT" --app-port "$PORT" > ../out.run.txt 2>&1 &
@@ -28,6 +30,11 @@ while [ -z "$(grep -o "Server is running on port " out.run.txt 2>/dev/null)" ];
2830
sleep 1
2931
done
3032

33+
# Make sure the proxy is ready to serve requests
34+
while [ -z "$(grep -o "To access your app go to " out.run.txt 2>/dev/null)" ]; do
35+
sleep 1
36+
done
37+
3138
title "Checking app is running..."
3239
trace curl -s -o - http://127.0.0.1:$PROXY_PORT | grep -w "Hello From App"
3340

acceptance/cmd/workspace/apps/run-local-node/test.toml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
# Temporarily disabled due to the flakiness like here https://github.com/databricks/cli/actions/runs/17234131593/job/48894892423
21
Cloud = false
3-
Local = false
2+
Local = true
43
RecordRequests = false
54
Timeout = '2m'
65
TimeoutWindows = '10m'
@@ -21,3 +20,6 @@ New='127.0.0.1:$(port)'
2120
[[Repls]]
2221
Old='To debug your app, attach a debugger to port [0-9]+'
2322
New='To debug your app, attach a debugger to port $(debug_port)'
23+
24+
[EnvMatrix]
25+
DATABRICKS_CLI_DEPLOYMENT = ["direct-exp"]
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Local = false
1+
Local = true
22
Cloud = false
33

44
[EnvMatrix]
5-
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
5+
DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

acceptance/cmd/workspace/apps/run-local/output.txt

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,9 @@ Hello, world
99
=== Starting the app in background...
1010
=== Waiting
1111
=== Checking app is running...
12+
=== Checking the proxy is running...
1213
>>> curl -s -o - http://127.0.0.1:$(port)
13-
{
14-
"Accept": "*/*",
15-
"Accept-Encoding": "gzip",
16-
"Host": "127.0.0.1:$(port)",
17-
"User-Agent": "curl/(version)",
18-
"X-Forwarded-Email": "[USERNAME]",
19-
"X-Forwarded-Host": "localhost",
20-
"X-Forwarded-Preferred-Username": "",
21-
"X-Forwarded-User": "[USERNAME]",
22-
"X-Real-Ip": "127.0.0.1",
23-
"X-Request-Id": "[UUID]"
24-
}
14+
{"Accept":"*/*","Accept-Encoding":"gzip","Host":"127.0.0.1:$(port)","User-Agent":"curl/(version)","X-Forwarded-Email":"[USERNAME]","X-Forwarded-Host":"localhost","X-Forwarded-Preferred-Username":"","X-Forwarded-User":"[USERNAME]","X-Real-Ip":"127.0.0.1","X-Request-Id":"[UUID]"}
2515

2616
=== Sending shutdown request...
2717
>>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown

acceptance/cmd/workspace/apps/run-local/script

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,16 @@ trace errcode $CLI apps run-local --entry-point value-from.yml 2>&1
66
# so we don't need to start it in background. It will install the dependencies as part of the command
77
trace $CLI apps run-local --prepare-environment --entry-point test.yml 2>&1 | grep -w "Hello, world"
88

9-
# Get 3 unique ports sequentially to avoid conflicts
10-
PORTS=$(allocate_ports.py 3 | tr -d '\r')
9+
PORT=8090
10+
DEBUG_PORT=5252
11+
PROXY_PORT=8091
1112

12-
# Read ports into array
13-
PORTS_ARR=($(echo "$PORTS"))
14-
PORT="${PORTS_ARR[0]}"
15-
DEBUG_PORT="${PORTS_ARR[1]}"
16-
PROXY_PORT="${PORTS_ARR[2]}"
13+
cleanup() {
14+
# Kill any still running processes on these ports when the script exits
15+
kill_port.py $PORT $DEBUG_PORT $PROXY_PORT
16+
}
17+
18+
trap cleanup EXIT
1719

1820
title "Starting the app in background..."
1921
trace $CLI apps run-local --prepare-environment --debug --port "$PROXY_PORT" --debug-port "$DEBUG_PORT" --app-port "$PORT" > ../out.run.txt 2>&1 &
@@ -30,8 +32,38 @@ while [ -z "$(grep -o "Python Flask app has started with" out.run.txt 2>/dev/nul
3032
sleep 1
3133
done
3234

35+
# Make sure the proxy is ready to serve requests
36+
while [ -z "$(grep -o "To access your app go to " out.run.txt 2>/dev/null)" ]; do
37+
sleep 1
38+
done
39+
3340
title "Checking app is running..."
34-
trace curl -s -o - http://127.0.0.1:$PROXY_PORT | jq
41+
# Wait for the proxy to be ready to serve requests (with timeout)
42+
timeout=5
43+
counter=0
44+
while ! curl -s -o /dev/null http://127.0.0.1:$PORT 2>/dev/null; do
45+
sleep 1
46+
counter=$((counter + 1))
47+
if [ $counter -ge $timeout ]; then
48+
echo "Timeout waiting for app to be ready after ${timeout}s"
49+
exit 1
50+
fi
51+
done
52+
53+
title "Checking the proxy is running..."
54+
# Wait for the proxy to be ready to serve requests (with timeout)
55+
timeout=5
56+
counter=0
57+
while ! curl -s -o /dev/null http://127.0.0.1:$PROXY_PORT 2>/dev/null; do
58+
sleep 1
59+
counter=$((counter + 1))
60+
if [ $counter -ge $timeout ]; then
61+
echo "Timeout waiting for proxy to be ready after ${timeout}s"
62+
exit 1
63+
fi
64+
done
65+
66+
trace curl -s -o - http://127.0.0.1:$PROXY_PORT
3567

3668
title "Sending shutdown request..."
3769
trace curl -s -o /dev/null http://127.0.0.1:$PROXY_PORT/shutdown || true

acceptance/cmd/workspace/apps/run-local/test.toml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Cloud = false
2-
Local = false
2+
Local = true
33
RecordRequests = false
44
Timeout = '2m'
55
TimeoutWindows = '10m'
@@ -20,3 +20,6 @@ New='127.0.0.1:$(port)'
2020
[[Repls]]
2121
Old='To debug your app, attach a debugger to port [0-9]+'
2222
New='To debug your app, attach a debugger to port $(debug_port)'
23+
24+
[EnvMatrix]
25+
DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

0 commit comments

Comments
 (0)