From b1de5c05e314b1416f468601f98aab7ca733654a Mon Sep 17 00:00:00 2001 From: Sean Sundberg Date: Wed, 11 May 2022 22:38:17 -0500 Subject: [PATCH 1/4] Changes logic from ibmcloud cli to the api closes #30 Signed-off-by: Sean Sundberg --- .github/workflows/verify-workflow.yaml | 1 + main.tf | 21 +++------ scripts/bind-instance.sh | 59 +++++++++++++++++--------- scripts/ibmcloud-login.sh | 17 -------- scripts/unbind-instance.sh | 33 +++++++++++--- test/stages/stage1-resource-group.tf | 2 +- 6 files changed, 75 insertions(+), 58 deletions(-) delete mode 100755 scripts/ibmcloud-login.sh diff --git a/.github/workflows/verify-workflow.yaml b/.github/workflows/verify-workflow.yaml index 7a9fed6..cd71966 100644 --- a/.github/workflows/verify-workflow.yaml +++ b/.github/workflows/verify-workflow.yaml @@ -33,6 +33,7 @@ jobs: with: clusterId: ${{ matrix.testcase }} validateDeployScript: .github/scripts/validate-deploy.sh + testPlan: "true" env: TF_VAR_ibmcloud_api_key: ${{ secrets.IBMCLOUD_API_KEY }} IBMCLOUD_API_KEY: ${{ secrets.IBMCLOUD_API_KEY }} diff --git a/main.tf b/main.tf index 84f8964..c483950 100644 --- a/main.tf +++ b/main.tf @@ -8,7 +8,7 @@ locals { module setup_clis { source = "cloud-native-toolkit/clis/util" - clis = ["ibmcloud-ob"] + clis = ["jq"] } resource null_resource print_names { @@ -38,31 +38,21 @@ resource "ibm_resource_key" "logdna_instance_key" { } } -resource null_resource ibmcloud_login { - provisioner "local-exec" { - command = "${path.module}/scripts/ibmcloud-login.sh ${var.region} ${var.resource_group_name}" - - environment = { - BIN_DIR = module.setup_clis.bin_dir - APIKEY = var.ibmcloud_api_key - } - } -} - resource "null_resource" "logdna_bind" { count = local.bind ? 1 : 0 - depends_on = [null_resource.ibmcloud_login] triggers = { bin_dir = module.setup_clis.bin_dir cluster_id = var.cluster_id instance_id = var.logdna_id + ibmcloud_api_key = var.ibmcloud_api_key } provisioner "local-exec" { - command = "${path.module}/scripts/bind-instance.sh ${self.triggers.cluster_id} ${self.triggers.instance_id} ${ibm_resource_key.logdna_instance_key[0].name} ${var.private_endpoint}" + command = "${path.module}/scripts/bind-instance.sh '${self.triggers.cluster_id}' '${self.triggers.instance_id}' '${ibm_resource_key.logdna_instance_key[0].name}' '${var.private_endpoint}'" environment = { + IBMCLOUD_API_KEY = self.triggers.ibmcloud_api_key BIN_DIR = self.triggers.bin_dir SYNC = var.sync } @@ -70,9 +60,10 @@ resource "null_resource" "logdna_bind" { provisioner "local-exec" { when = destroy - command = "${path.module}/scripts/unbind-instance.sh ${self.triggers.cluster_id} ${self.triggers.instance_id}" + command = "${path.module}/scripts/unbind-instance.sh '${self.triggers.cluster_id}' '${self.triggers.instance_id}'" environment = { + IBMCLOUD_API_KEY = self.triggers.ibmcloud_api_key BIN_DIR = self.triggers.bin_dir } } diff --git a/scripts/bind-instance.sh b/scripts/bind-instance.sh index fb41c6c..7318c4b 100755 --- a/scripts/bind-instance.sh +++ b/scripts/bind-instance.sh @@ -6,46 +6,65 @@ MODULE_DIR=$(cd ${SCRIPT_DIR}/..; pwd -P) CLUSTER_ID="$1" INSTANCE_ID="$2" INGESTION_KEY="$3" -PRIVATE="$4" - -if [[ "${PRIVATE}" == "true" ]]; then - PRIVATE="--private-endpoint" -else - PRIVATE="" -fi +PRIVATE="${4:-false}" if [[ -n "${BIN_DIR}" ]]; then export PATH="${BIN_DIR}:${PATH}" fi +if [[ -z "${IBMCLOUD_API_KEY}" ]]; then + echo "IBMCLOUD_API_KEY must be provided as an environment variable" >&2 + exit 1 +fi + +export TOKEN=$(curl -s -X POST "https://iam.cloud.ibm.com/identity/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${IBMCLOUD_API_KEY}" | jq -r '.access_token') + +BASE_URL="https://containers.cloud.ibm.com/global/v2/observe/logging" + echo "Configuring LogDNA for ${CLUSTER_ID} cluster and ${INSTANCE_ID} LogDNA instance" -ibmcloud target -if ibmcloud ob logging config ls --cluster "${CLUSTER_ID}" | grep -q "Instance ID"; then - EXISTING_INSTANCE_ID=$(ibmcloud ob logging config ls --cluster "${CLUSTER_ID}" | grep "Instance ID" | sed -E "s/Instance ID: +([^ ]+)/\1/g") +EXISTING_INSTANCE_ID=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID}" \ + -H "Authorization: Bearer ${TOKEN}" \ + jq -r '.[] | .instanceId // empty') + +if [[ -n "${EXISTING_INSTANCE_ID}" ]]; then if [[ "${EXISTING_INSTANCE_ID}" == "${INSTANCE_ID}" ]]; then echo "LogDNA configuration already exists on this cluster" exit 0 else echo "Existing LogDNA configuration found on this cluster for a different LogDNA instance: ${EXISTING_INSTANCE_ID}." echo "Removing the config before creating the new one" - ibmcloud ob logging config delete \ - --cluster "${CLUSTER_ID}" \ - --instance "${EXISTING_INSTANCE_ID}" \ - --force + + curl -s -X POST "${URL}/removeConfig" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d $(jq -n --arg CLUSTER "${CLUSTER_ID}" --arg INSTANCE "${EXISTING_INSTANCE_ID}" '{"cluster": $CLUSTER, "instance": $INSTANCE}') echo " Waiting for the old configuration to be removed..." - sleep 300 + while true; do + RESPONSE=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID}" \ + -H "Authorization: Bearer ${TOKEN}" \ + jq -r '.[] | .instanceId // empty') + + if [[ -z "${RESPONSE}" ]]; then + echo " LogDNA instances removed" + break + else + echo " LogDNA instance still exists. Waiting..." + sleep 30 + fi + done fi else echo "No existing logging config found for ${CLUSTER_ID} cluster" - ibmcloud ob logging config ls --cluster "${CLUSTER_ID}" fi set -e echo "Creating LogDNA configuration for ${CLUSTER_ID} cluster and ${INSTANCE_ID} LogDNA instance" -ibmcloud ob logging config create \ - --cluster "${CLUSTER_ID}" \ - --instance "${INSTANCE_ID}" \ - --logdna-ingestion-key "${INGESTION_KEY}" ${PRIVATE} +curl -s -X POST "${URL}/createConfig" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d $(jq -n --arg CLUSTER "${CLUSTER_ID}" --arg INGESTION "${INGESTION_KEY}" --arg INSTANCE "${INSTANCE_ID}" --argjson PRIVATE "${PRIVATE}" '{"cluster": $CLUSTER, "instance": $INSTANCE, "ingestionKey": $INGESTION, "privateEndpoint": $PRIVATE}') diff --git a/scripts/ibmcloud-login.sh b/scripts/ibmcloud-login.sh deleted file mode 100755 index ffad49c..0000000 --- a/scripts/ibmcloud-login.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -REGION="$1" -RESOURCE_GROUP="$2" - -if [[ -z "${APIKEY}" ]]; then - echo "The APIKEY is required" - exit 1 -fi - -if [[ -n "${BIN_DIR}" ]]; then - export PATH="${BIN_DIR}:${PATH}" -fi - -ibmcloud config --check-version=false - -ibmcloud login -r "${REGION}" -g "${RESOURCE_GROUP}" --apikey "${APIKEY}" diff --git a/scripts/unbind-instance.sh b/scripts/unbind-instance.sh index 515c108..e7846b3 100755 --- a/scripts/unbind-instance.sh +++ b/scripts/unbind-instance.sh @@ -10,8 +10,31 @@ if [[ -n "${BIN_DIR}" ]]; then export PATH="${BIN_DIR}:${PATH}" fi -ibmcloud ob logging config delete \ - --cluster "${CLUSTER_ID}" \ - --instance "${INSTANCE_ID}" \ - --force || \ - echo "Error deleting logging instance from cluster" +if [[ -z "${IBMCLOUD_API_KEY}" ]]; then + echo "IBMCLOUD_API_KEY must be provided as an environment variable" >&2 + exit 1 +fi + +export TOKEN=$(curl -s -X POST "https://iam.cloud.ibm.com/identity/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${IBMCLOUD_API_KEY}" | jq -r '.access_token') + +curl -s -X POST "${URL}/removeConfig" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d $(jq -n --arg CLUSTER "${CLUSTER_ID}" --arg INSTANCE "${INSTANCE_ID}" '{"cluster": $CLUSTER, "instance": $INSTANCE}') + +echo " Waiting for the instance to be removed..." +while true; do + RESPONSE=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID}" \ + -H "Authorization: Bearer ${TOKEN}" \ + jq -r '.[] | .instanceId // empty') + + if [[ -z "${RESPONSE}" ]]; then + echo " LogDNA instances removed" + break + else + echo " LogDNA instance still exists. Waiting..." + sleep 30 + fi +done diff --git a/test/stages/stage1-resource-group.tf b/test/stages/stage1-resource-group.tf index b7cfaaf..b44c2e0 100644 --- a/test/stages/stage1-resource-group.tf +++ b/test/stages/stage1-resource-group.tf @@ -2,5 +2,5 @@ module "resource_group" { source = "github.com/cloud-native-toolkit/terraform-ibm-resource-group.git" resource_group_name = var.resource_group_name - provision = false + ibmcloud_api_key = var.ibmcloud_api_key } From 3a754eccc259a12d2993009dbc8ef0f0904b1600 Mon Sep 17 00:00:00 2001 From: Sean Sundberg Date: Wed, 11 May 2022 22:50:20 -0500 Subject: [PATCH 2/4] Adds logging for binding instances Signed-off-by: Sean Sundberg --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index c483950..9325f67 100644 --- a/main.tf +++ b/main.tf @@ -52,7 +52,7 @@ resource "null_resource" "logdna_bind" { command = "${path.module}/scripts/bind-instance.sh '${self.triggers.cluster_id}' '${self.triggers.instance_id}' '${ibm_resource_key.logdna_instance_key[0].name}' '${var.private_endpoint}'" environment = { - IBMCLOUD_API_KEY = self.triggers.ibmcloud_api_key + IBMCLOUD_API_KEY = nonsensitive(self.triggers.ibmcloud_api_key) BIN_DIR = self.triggers.bin_dir SYNC = var.sync } @@ -63,7 +63,7 @@ resource "null_resource" "logdna_bind" { command = "${path.module}/scripts/unbind-instance.sh '${self.triggers.cluster_id}' '${self.triggers.instance_id}'" environment = { - IBMCLOUD_API_KEY = self.triggers.ibmcloud_api_key + IBMCLOUD_API_KEY = nonsensitive(self.triggers.ibmcloud_api_key) BIN_DIR = self.triggers.bin_dir } } From 95a7357b50bd604698f90fc219e3426eef7d2f98 Mon Sep 17 00:00:00 2001 From: Sean Sundberg Date: Wed, 11 May 2022 22:53:54 -0500 Subject: [PATCH 3/4] Updates logging Signed-off-by: Sean Sundberg --- scripts/bind-instance.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/bind-instance.sh b/scripts/bind-instance.sh index 7318c4b..7b9ac43 100755 --- a/scripts/bind-instance.sh +++ b/scripts/bind-instance.sh @@ -29,6 +29,8 @@ EXISTING_INSTANCE_ID=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID -H "Authorization: Bearer ${TOKEN}" \ jq -r '.[] | .instanceId // empty') +echo "Existing instance id: ${EXISTING_INSTANCE_ID}" + if [[ -n "${EXISTING_INSTANCE_ID}" ]]; then if [[ "${EXISTING_INSTANCE_ID}" == "${INSTANCE_ID}" ]]; then echo "LogDNA configuration already exists on this cluster" @@ -53,6 +55,7 @@ if [[ -n "${EXISTING_INSTANCE_ID}" ]]; then break else echo " LogDNA instance still exists. Waiting..." + echo " ${RESPONSE}" sleep 30 fi done From 558f138bc2313f99d8a11b49e02cee5054047849 Mon Sep 17 00:00:00 2001 From: Sean Sundberg Date: Wed, 11 May 2022 23:09:44 -0500 Subject: [PATCH 4/4] Adds refresh token header Signed-off-by: Sean Sundberg --- scripts/bind-instance.sh | 10 ++++++++-- scripts/unbind-instance.sh | 12 +++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/bind-instance.sh b/scripts/bind-instance.sh index 7b9ac43..07a339b 100755 --- a/scripts/bind-instance.sh +++ b/scripts/bind-instance.sh @@ -17,9 +17,11 @@ if [[ -z "${IBMCLOUD_API_KEY}" ]]; then exit 1 fi -export TOKEN=$(curl -s -X POST "https://iam.cloud.ibm.com/identity/token" \ +TOKEN_RESULT=$(curl -s -X POST "https://iam.cloud.ibm.com/identity/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${IBMCLOUD_API_KEY}" | jq -r '.access_token') + -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${IBMCLOUD_API_KEY}") +TOKEN=$(echo "${TOKEN_RESULT}" | jq -r '.access_token') +REFRESH_TOKEN=$(echo "${TOKEN_RESULT}" | jq -r '.refresh_token') BASE_URL="https://containers.cloud.ibm.com/global/v2/observe/logging" @@ -27,6 +29,7 @@ echo "Configuring LogDNA for ${CLUSTER_ID} cluster and ${INSTANCE_ID} LogDNA ins EXISTING_INSTANCE_ID=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID}" \ -H "Authorization: Bearer ${TOKEN}" \ + -H "X-Auth-Refresh-Token: ${REFRESH_TOKEN}" \ jq -r '.[] | .instanceId // empty') echo "Existing instance id: ${EXISTING_INSTANCE_ID}" @@ -42,12 +45,14 @@ if [[ -n "${EXISTING_INSTANCE_ID}" ]]; then curl -s -X POST "${URL}/removeConfig" \ -H "Authorization: Bearer ${TOKEN}" \ -H "Content-Type: application/json" \ + -H "X-Auth-Refresh-Token: ${REFRESH_TOKEN}" \ -d $(jq -n --arg CLUSTER "${CLUSTER_ID}" --arg INSTANCE "${EXISTING_INSTANCE_ID}" '{"cluster": $CLUSTER, "instance": $INSTANCE}') echo " Waiting for the old configuration to be removed..." while true; do RESPONSE=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID}" \ -H "Authorization: Bearer ${TOKEN}" \ + -H "X-Auth-Refresh-Token: ${REFRESH_TOKEN}" \ jq -r '.[] | .instanceId // empty') if [[ -z "${RESPONSE}" ]]; then @@ -69,5 +74,6 @@ set -e echo "Creating LogDNA configuration for ${CLUSTER_ID} cluster and ${INSTANCE_ID} LogDNA instance" curl -s -X POST "${URL}/createConfig" \ -H "Authorization: Bearer ${TOKEN}" \ + -H "X-Auth-Refresh-Token: ${REFRESH_TOKEN}" \ -H "Content-Type: application/json" \ -d $(jq -n --arg CLUSTER "${CLUSTER_ID}" --arg INGESTION "${INGESTION_KEY}" --arg INSTANCE "${INSTANCE_ID}" --argjson PRIVATE "${PRIVATE}" '{"cluster": $CLUSTER, "instance": $INSTANCE, "ingestionKey": $INGESTION, "privateEndpoint": $PRIVATE}') diff --git a/scripts/unbind-instance.sh b/scripts/unbind-instance.sh index e7846b3..a45e5c5 100755 --- a/scripts/unbind-instance.sh +++ b/scripts/unbind-instance.sh @@ -15,19 +15,25 @@ if [[ -z "${IBMCLOUD_API_KEY}" ]]; then exit 1 fi -export TOKEN=$(curl -s -X POST "https://iam.cloud.ibm.com/identity/token" \ +TOKEN_RESULT=$(curl -s -X POST "https://iam.cloud.ibm.com/identity/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${IBMCLOUD_API_KEY}" | jq -r '.access_token') + -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=${IBMCLOUD_API_KEY}") +TOKEN=$(echo "${TOKEN_RESULT}" | jq -r '.access_token') +REFRESH_TOKEN=$(echo "${TOKEN_RESULT}" | jq -r '.refresh_token') -curl -s -X POST "${URL}/removeConfig" \ +BASE_URL="https://containers.cloud.ibm.com/global/v2/observe/logging" + +curl -s -X POST "${BASE_URL}/removeConfig" \ -H "Authorization: Bearer ${TOKEN}" \ -H "Content-Type: application/json" \ + -H "X-Auth-Refresh-Token: ${REFRESH_TOKEN}" \ -d $(jq -n --arg CLUSTER "${CLUSTER_ID}" --arg INSTANCE "${INSTANCE_ID}" '{"cluster": $CLUSTER, "instance": $INSTANCE}') echo " Waiting for the instance to be removed..." while true; do RESPONSE=$(curl -s -X GET "${BASE_URL}/getConfigs?query=${CLUSTER_ID}" \ -H "Authorization: Bearer ${TOKEN}" \ + -H "X-Auth-Refresh-Token: ${REFRESH_TOKEN}" \ jq -r '.[] | .instanceId // empty') if [[ -z "${RESPONSE}" ]]; then