Skip to content

Commit

Permalink
Improve k8s test
Browse files Browse the repository at this point in the history
  • Loading branch information
Jonathan Turner committed Feb 24, 2020
1 parent 72f2005 commit bf71d5e
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 129 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------
ARG NODE_VERSION=10

# docker can't tell when the repo has changed and will therefore cache this layer
FROM alpine/git as repo
RUN git clone https://github.com/jonathandturner/azure-sdk-for-js --single-branch --branch manual_integration_test_1 --depth 1 /azure-sdk-for-js

FROM node:${NODE_VERSION}-slim

COPY --from=repo /azure-sdk-for-js/sdk/identity /sdk/identity
COPY --from=repo /azure-sdk-for-js/sdk/core /sdk/core
COPY --from=repo /azure-sdk-for-js/sdk/keyvault/keyvault-secrets /sdk/keyvault/keyvault-secrets

WORKDIR /sdk/identity/identity/tests/manual_integration/kubernetes
RUN node install
CMD ["node", "index"]
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ az identity create -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME

Save its `clientId`, `id` (ARM URI), and `principalId` (object ID) for later:
```sh
MANAGED_IDENTITY_CLIENT_ID=az identity show -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME --query clientId -o tsv
MANAGED_IDENTITY_ID=az identity show -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME --query id -o tsv
MANAGED_IDENTITY_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME --query principalId -o tsv
$MANAGED_IDENTITY_CLIENT_ID=az identity show -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME --query clientId -o tsv
$MANAGED_IDENTITY_ID=az identity show -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME --query id -o tsv
$MANAGED_IDENTITY_PRINCIPAL_ID=az identity show -g $RESOURCE_GROUP -n $MANAGED_IDENTITY_NAME --query principalId -o tsv
```

### Key Vault
Expand All @@ -85,9 +85,7 @@ az aks create -g $RESOURCE_GROUP -n $AKS_NAME --generate-ssh-keys --node-count 1

Grant the cluster's service principal permission to use the managed identity:
```sh
az role assignment create --role "Managed Identity Operator" \
--assignee $(az aks show -g $RESOURCE_GROUP -n $AKS_NAME --query servicePrincipalProfile.clientId -o tsv) \
--scope $MANAGED_IDENTITY_ID
az role assignment create --role "Managed Identity Operator" --assignee $(az aks show -g $RESOURCE_GROUP -n $AKS_NAME --query servicePrincipalProfile.clientId -o tsv) --scope $MANAGED_IDENTITY_ID
```


Expand All @@ -106,20 +104,20 @@ git clone https://github.com/Azure/azure-sdk-for-js/ --branch master --single-br

The rest of this section assumes this working directory:
```sh
cd azure-sdk-for-js/sdk/identity/identity/test/integration-test
cd azure-sdk-for-js/sdk/identity/identity/test/manual-integration
```

### build images and push them to the container registry
Set environment variables:
```sh
REPOSITORY=$ACR_NAME.azurecr.io
IMAGE_NAME=test-pod-identity
NODE_VERSION=10
$REPOSITORY="$($ACR_NAME).azurecr.io"
$IMAGE_NAME="test-pod-identity"
$NODE_VERSION=10
```

Build an image:
```sh
docker build --no-cache --build-arg NODE_VERSION=$NODE_VERSION -t $REPOSITORY/$IMAGE_NAME:$NODE_VERSION ./managed-identity-live
docker build --no-cache --build-arg NODE_VERSION=$NODE_VERSION -t "$($REPOSITORY)/$($IMAGE_NAME):$($NODE_VERSION)" ./managed-identity-live
```

Push it to ACR:
Expand Down Expand Up @@ -150,7 +148,7 @@ helm init --wait

### run the test script
```sh
node ./pod-identity/run-test.js \
node ./index.js \
--client-id $MANAGED_IDENTITY_CLIENT_ID \
--resource-id $MANAGED_IDENTITY_ID \
--vault-url https://$KEY_VAULT_NAME.vault.azure.net \
Expand Down
132 changes: 15 additions & 117 deletions sdk/identity/identity/test/manual-integration/Kubernetes/index.ts
Original file line number Diff line number Diff line change
@@ -1,123 +1,21 @@
import * as yargs from "yargs";
import * as util from "util";
import * as path from "path";
const exec = util.promisify(require('child_process').exec);

function sleep (time: number) {
return new Promise((resolve) => setTimeout(resolve, time));
}

const JOB_NAME = "test"
const HELM_APP_NAME = "test"

const argv = yargs
.option('client-id', {
description: "managed identity's client ID",
type: 'string',
})
.option('resource-id', {
description: "managed identity's ARM ID",
type: 'string',
})
.option('vault-url', {
description: "URL of a vault whose secrets the managed identity may manage",
type: 'string',
})
.option('repository', {
description: "repository holding the test image",
type: 'string',
})
.option('image-name', {
description: "name of the test image",
type: 'string',
})
.option('image-tag', {
description: "test image tag",
type: 'string',
})
.option('verbose', {
alias: "v",
description: "print all executed commands and their output",
type: 'boolean',
})
.demandOption(["client-id", "resource-id", "vault-url", "repository", "image-name", "image-tag"], "please provide all required parameters")
.help()
.alias('help', 'h')
.argv;

async function runCommand(command: string[], exitOnError = true) {
try {
if (argv.verbose) {
console.log(command);
}
let commandString = command.join(" ");
let { stdout, stderr } = await exec(commandString);
if (argv.verbose) {
console.log(stdout);
}
return stdout;
} catch (e) {
if (exitOnError) {
console.log(e);
process.exit(1);
}
return e;
}
}
import { SecretClient } from "@azure/keyvault-secrets";
import { DefaultAzureCredential } from "@azure/identity";

async function main(): Promise<void> {
// install the chart
let helm_install = [
"helm",
"install",
path.resolve(__dirname, "test-pod-identity"),
"-n",
HELM_APP_NAME,
"--set",
`aad-pod-identity.azureIdentity.resourceID=${argv["resource-id"]},aad-pod-identity.azureIdentity.clientID=${argv["client-id"]}`,
"--set",
"vaultUrl=" + argv.vault_url,
"--set",
`image.repository=${argv.repository},image.name=${argv["image-name"]},image.tag=${argv["image-tag"]}`
];
// DefaultAzureCredential expects the following three environment variables:
// - AZURE_TENANT_ID: The tenant ID in Azure Active Directory
// - AZURE_CLIENT_ID: The application (client) ID registered in the AAD tenant
// - AZURE_CLIENT_SECRET: The client secret for the registered application
const credential = new DefaultAzureCredential();

runCommand(helm_install);
const url = "https://" + process.env.KEY_VAULT_NAME + ".vault.azure.net";
const client = new SecretClient(url, credential);

// get the name of the test pod
let podName = await runCommand(
["kubectl", "get", "pods", "--selector=job-name=" + JOB_NAME, "--output=jsonpath='{.items[*].metadata.name}'"]
);

let logs = ""

// poll the number of active pods to determine when the test has finished
let count_active_pods = ["kubectl", "get", "job", JOB_NAME, "--output=jsonpath='{.status.active}'"]
for (let x = 0; x < 10; ++x) {
// kubectl will return '' when there are no active pods
let active_pods = runCommand(count_active_pods)
logs = await runCommand(["kubectl", "logs", "-f", podName], false)
if (!active_pods)
break
await sleep(30)
}

// output logs from the most recent run
console.log(logs)

// uninstall the chart
runCommand(["helm", "del", "--purge", HELM_APP_NAME])

// delete CRDs because Helm didn't
let pod_identity_CRDs = [
"azureassignedidentities.aadpodidentity.k8s.io",
"azureidentities.aadpodidentity.k8s.io",
"azureidentitybindings.aadpodidentity.k8s.io",
"azurepodidentityexceptions.aadpodidentity.k8s.io",
]
runCommand(["kubectl", "delete", "crd"].concat(pod_identity_CRDs))
await client.setSecret("secret-name", "secret-value");
}

main().catch((err) => {
console.log("error code: ", err.code);
console.log("error message: ", err.message);
console.log("error stack: ", err.stack);
});
console.log("error code: ", err.code);
console.log("error message: ", err.message);
console.log("error stack: ", err.stack);
});
124 changes: 124 additions & 0 deletions sdk/identity/identity/test/manual-integration/Kubernetes/run_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import * as yargs from "yargs";
import * as util from "util";
import * as path from "path";
const exec = util.promisify(require('child_process').exec);

function sleep (time: number) {
return new Promise((resolve) => setTimeout(resolve, time));
}

const JOB_NAME = "test"
const HELM_APP_NAME = "test"

const argv = yargs
.option('client-id', {
description: "managed identity's client ID",
type: 'string',
})
.option('resource-id', {
description: "managed identity's ARM ID",
type: 'string',
})
.option('vault-url', {
description: "URL of a vault whose secrets the managed identity may manage",
type: 'string',
})
.option('repository', {
description: "repository holding the test image",
type: 'string',
})
.option('image-name', {
description: "name of the test image",
type: 'string',
})
.option('image-tag', {
description: "test image tag",
type: 'string',
})
.option('verbose', {
alias: "v",
description: "print all executed commands and their output",
type: 'boolean',
})
.demandOption(["client-id", "resource-id", "vault-url", "repository", "image-name", "image-tag"], "please provide all required parameters")
.help()
.alias('help', 'h')
.argv;

async function runCommand(command: string[], exitOnError = true) {
try {
if (argv.verbose) {
console.log(command);
}
let commandString = command.join(" ");
let { stdout, stderr } = await exec(commandString);
if (argv.verbose) {
console.log(stdout);
console.log(stderr);
}
return stdout;
} catch (e) {
if (exitOnError) {
console.log(e);
process.exit(1);
}
return e;
}
}

async function main(): Promise<void> {
// install the chart
let helm_install = [
"helm",
"install",
path.resolve(__dirname, "test-pod-identity"),
"-n",
HELM_APP_NAME,
"--set",
`aad-pod-identity.azureIdentity.resourceID=${argv["resource-id"]},aad-pod-identity.azureIdentity.clientID=${argv["client-id"]}`,
"--set",
"vaultUrl=" + argv.vault_url,
"--set",
`image.repository=${argv.repository},image.name=${argv["image-name"]},image.tag=${argv["image-tag"]}`
];

runCommand(helm_install);

// get the name of the test pod
let podName = await runCommand(
["kubectl", "get", "pods", "--selector=job-name=" + JOB_NAME, "--output=jsonpath='{.items[*].metadata.name}'"]
);

let logs = ""

// poll the number of active pods to determine when the test has finished
let count_active_pods = ["kubectl", "get", "job", JOB_NAME, "--output=jsonpath='{.status.active}'"]
for (let x = 0; x < 10; ++x) {
// kubectl will return '' when there are no active pods
let active_pods = runCommand(count_active_pods)
logs = await runCommand(["kubectl", "logs", "-f", podName], false)
if (!active_pods)
break
await sleep(30)
}

// output logs from the most recent run
console.log(logs)

// uninstall the chart
runCommand(["helm", "del", "--purge", HELM_APP_NAME])

// delete CRDs because Helm didn't
let pod_identity_CRDs = [
"azureassignedidentities.aadpodidentity.k8s.io",
"azureidentities.aadpodidentity.k8s.io",
"azureidentitybindings.aadpodidentity.k8s.io",
"azurepodidentityexceptions.aadpodidentity.k8s.io",
]
runCommand(["kubectl", "delete", "crd"].concat(pod_identity_CRDs))
}
main().catch((err) => {
console.log("error code: ", err.code);
console.log("error message: ", err.message);
console.log("error stack: ", err.stack);
});

0 comments on commit bf71d5e

Please sign in to comment.