Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
## [1.14.0] - 2026-08-03
- k8s scope deployments now report launched and healthy instance counts, so the deployment page shows live "X/Y launched" and "X/Y healthy" progress
- Fix: triggering a scheduled task job on a scope that is not deployed now fails with a clear "deploy the scope first" message instead of an opaque error
- Add "Kill instance" action to scheduled task scopes to terminate an individual running job instance

## [1.13.0] - 2026-07-10
- Add support to get AWS credentials via assume role
Expand Down
36 changes: 36 additions & 0 deletions azure-aro/specs/actions/kill-instance.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"name": "Kill instance",
"type": "custom",
"icon": "material-symbols:delete-outline",
"results": {
"schema": { "type": "object", "required": [], "properties": {} },
"values": {}
},
"service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}",
"parameters": {
"schema": {
"type": "object",
"required": [
"instance_id",
"scope_id",
"deployment_id"
],
"properties": {
"scope_id": {
"type": "string"
},
"instance_id": {
"type": "string"
},
"deployment_id": {
"type": "string"
}
}
},
"values": {}
},
"annotations": {
"show_on": ["performance"],
"runs_over": "scope"
}
}
18 changes: 0 additions & 18 deletions azure-aro/specs/actions/kill-instances.json.tpl

This file was deleted.

36 changes: 36 additions & 0 deletions azure/specs/actions/kill-instance.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"name": "Kill instance",
"type": "custom",
"icon": "material-symbols:delete-outline",
"results": {
"schema": { "type": "object", "required": [], "properties": {} },
"values": {}
},
"service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}",
"parameters": {
"schema": {
"type": "object",
"required": [
"instance_id",
"scope_id",
"deployment_id"
],
"properties": {
"scope_id": {
"type": "string"
},
"instance_id": {
"type": "string"
},
"deployment_id": {
"type": "string"
}
}
},
"values": {}
},
"annotations": {
"show_on": ["performance"],
"runs_over": "scope"
}
}
18 changes: 0 additions & 18 deletions azure/specs/actions/kill-instances.json.tpl

This file was deleted.

40 changes: 20 additions & 20 deletions k8s/deployment/kill_instances → k8s/deployment/kill_instance
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ set -euo pipefail
log debug "🔍 Starting instance kill operation..."

DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.parameters.deployment_id // .notification.parameters.deployment_id // empty')
INSTANCE_NAME=$(echo "$CONTEXT" | jq -r '.parameters.instance_name // .notification.parameters.instance_name // empty')
INSTANCE_ID=$(echo "$CONTEXT" | jq -r '.parameters.instance_id // .notification.parameters.instance_id // empty')

if [[ -z "$DEPLOYMENT_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then
DEPLOYMENT_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.deployment_id // empty')
fi

if [[ -z "$INSTANCE_NAME" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then
INSTANCE_NAME=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_name // empty')
if [[ -z "$INSTANCE_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then
INSTANCE_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_id // empty')
fi

if [[ -z "$DEPLOYMENT_ID" ]]; then
Expand All @@ -26,18 +26,18 @@ if [[ -z "$DEPLOYMENT_ID" ]]; then
exit 1
fi

if [[ -z "$INSTANCE_NAME" ]]; then
log error "❌ instance_name parameter not found"
if [[ -z "$INSTANCE_ID" ]]; then
log error "❌ instance_id parameter not found"
log error "💡 Possible causes:"
log error " - Parameter not provided in action request"
log error " - Context structure is different than expected"
log error "🔧 How to fix:"
log error " - Ensure instance_name is passed in the action parameters"
log error " - Ensure instance_id is passed in the action parameters"
exit 1
fi

log debug "📋 Deployment ID: $DEPLOYMENT_ID"
log debug "📋 Instance name: $INSTANCE_NAME"
log debug "📋 Instance name: $INSTANCE_ID"

SCOPE_ID=$(echo "$CONTEXT" | jq -r '.tags.scope_id // .scope.id // .notification.tags.scope_id // empty')

Expand All @@ -63,8 +63,8 @@ log debug "📋 Scope ID: $SCOPE_ID"
log debug "📋 Namespace: $K8S_NAMESPACE"

log debug "🔍 Verifying pod exists..."
if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
log error "❌ Pod $INSTANCE_NAME not found in namespace $K8S_NAMESPACE"
if ! kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
log error "❌ Pod $INSTANCE_ID not found in namespace $K8S_NAMESPACE"
log error "💡 Possible causes:"
log error " - Pod was already terminated"
log error " - Pod name is incorrect"
Expand All @@ -75,15 +75,15 @@ if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
fi

log debug "📋 Fetching pod details..."
POD_STATUS=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
POD_NODE=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}')
POD_START_TIME=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}')
POD_STATUS=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
POD_NODE=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}')
POD_START_TIME=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}')

log debug "📋 Pod: $INSTANCE_NAME | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME"
log debug "📋 Pod: $INSTANCE_ID | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME"

DEPLOYMENT_NAME="d-$SCOPE_ID-$DEPLOYMENT_ID"

POD_DEPLOYMENT=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
POD_DEPLOYMENT=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
if [[ -n "$POD_DEPLOYMENT" ]]; then
REPLICASET_DEPLOYMENT=$(kubectl get replicaset "$POD_DEPLOYMENT" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
log debug "📋 Pod ownership: ReplicaSet=$POD_DEPLOYMENT -> Deployment=$REPLICASET_DEPLOYMENT"
Expand All @@ -95,14 +95,14 @@ else
log warn "⚠️ Could not verify pod ownership"
fi

log debug "📝 Deleting pod $INSTANCE_NAME with 30s grace period..."
kubectl delete pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" --grace-period=30
log debug "📝 Deleting pod $INSTANCE_ID with 30s grace period..."
kubectl delete pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" --grace-period=30

log debug "📝 Waiting for pod termination..."
kubectl wait --for=delete pod/"$INSTANCE_NAME" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached"
kubectl wait --for=delete pod/"$INSTANCE_ID" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached"

if kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
if kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
log warn "⚠️ Pod still exists after deletion attempt (status: $POD_STATUS_AFTER)"
else
log info "✅ Pod successfully terminated and removed"
Expand All @@ -123,4 +123,4 @@ else
log warn "⚠️ Deployment $DEPLOYMENT_NAME not found"
fi

log info "✨ Instance kill operation completed for $INSTANCE_NAME"
log info "✨ Instance kill operation completed for $INSTANCE_ID"
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bats
# =============================================================================
# Unit tests for deployment/kill_instances - pod termination
# Unit tests for deployment/kill_instance - pod termination
# =============================================================================

setup() {
Expand All @@ -15,7 +15,7 @@ setup() {
export CONTEXT='{
"parameters": {
"deployment_id": "deploy-456",
"instance_name": "my-pod-abc123"
"instance_id": "my-pod-abc123"
},
"tags": {
"scope_id": "scope-123"
Expand Down Expand Up @@ -84,8 +84,8 @@ teardown() {
# =============================================================================
# Success Case
# =============================================================================
@test "kill_instances: successfully kills pod with correct logging" {
run bash "$BATS_TEST_DIRNAME/../kill_instances"
@test "kill_instance: successfully kills pod with correct logging" {
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 0 ]
# Start message
Expand All @@ -110,14 +110,14 @@ teardown() {
# =============================================================================
# Error Cases
# =============================================================================
@test "kill_instances: fails with troubleshooting when deployment_id missing" {
@test "kill_instance: fails with troubleshooting when deployment_id missing" {
export CONTEXT='{
"parameters": {
"instance_name": "my-pod-abc123"
"instance_id": "my-pod-abc123"
}
}'

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ deployment_id parameter not found"
Expand All @@ -127,32 +127,32 @@ teardown() {
assert_contains "$output" "Ensure deployment_id is passed in the action parameters"
}

@test "kill_instances: fails with troubleshooting when instance_name missing" {
@test "kill_instance: fails with troubleshooting when instance_id missing" {
export CONTEXT='{
"parameters": {
"deployment_id": "deploy-456"
}
}'

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ instance_name parameter not found"
assert_contains "$output" "❌ instance_id parameter not found"
assert_contains "$output" "💡 Possible causes:"
assert_contains "$output" "Parameter not provided in action request"
assert_contains "$output" "🔧 How to fix:"
assert_contains "$output" "Ensure instance_name is passed in the action parameters"
assert_contains "$output" "Ensure instance_id is passed in the action parameters"
}

@test "kill_instances: fails with troubleshooting when scope_id missing" {
@test "kill_instance: fails with troubleshooting when scope_id missing" {
export CONTEXT='{
"parameters": {
"deployment_id": "deploy-456",
"instance_name": "my-pod-abc123"
"instance_id": "my-pod-abc123"
}
}'

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ scope_id not found in context"
Expand All @@ -162,7 +162,7 @@ teardown() {
assert_contains "$output" "Verify the action is invoked with proper scope context"
}

@test "kill_instances: fails with troubleshooting when pod not found" {
@test "kill_instance: fails with troubleshooting when pod not found" {
kubectl() {
case "$1" in
get)
Expand All @@ -175,7 +175,7 @@ teardown() {
}
export -f kubectl

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ Pod my-pod-abc123 not found in namespace test-namespace"
Expand All @@ -188,7 +188,7 @@ teardown() {
# =============================================================================
# Warning Cases
# =============================================================================
@test "kill_instances: warns when pod belongs to different deployment" {
@test "kill_instance: warns when pod belongs to different deployment" {
kubectl() {
case "$1" in
get)
Expand Down Expand Up @@ -230,13 +230,13 @@ teardown() {
}
export -f kubectl

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 0 ]
assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456"
}

@test "kill_instances: warns when pod still exists after deletion" {
@test "kill_instance: warns when pod still exists after deletion" {
local delete_called=0
kubectl() {
case "$1" in
Expand Down Expand Up @@ -279,7 +279,7 @@ teardown() {
}
export -f kubectl

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 0 ]
assert_contains "$output" "⚠️ Pod deletion timeout reached"
Expand Down
Loading
Loading