Skip to content

Commit

Permalink
Merge pull request #239 from keboola/martinj-ct-1354-manage-api-setup…
Browse files Browse the repository at this point in the history
…-automation

CT-1354 manage api setup automation
  • Loading branch information
martinjandl authored Apr 16, 2024
2 parents c154953 + 2048824 commit 17828e4
Show file tree
Hide file tree
Showing 35 changed files with 913 additions and 52 deletions.
110 changes: 67 additions & 43 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,66 +72,81 @@ so you can disable this by creating new file `phpunit-retry.xml` from `phpunit-r
Create file `.env` with environment variables`:

```bash
#REQUIRED - must be filled before running any test
KBC_MANAGE_API_URL=https://connection.keboola.com
KBC_MANAGE_API_TOKEN=your_token
KBC_SUPER_API_TOKEN=your_token
# REQUIRED - must be filled before running any test
KBC_MANAGE_API_URL=https://connection.keboola.com # URL where Keboola Connection is running
KBC_MANAGE_API_TOKEN=your_token # manage api token assigned to user **with** **superadmin** privileges. Can be created in Account Settings under the title Personal Access Tokens. User must have Multi-Factor Authentication disabled.
KBC_SUPER_API_TOKEN=your_token # can be created in manage-apps on the Tokens tab
KBC_MANAGE_API_SUPER_TOKEN_WITH_PROJECTS_READ_SCOPE=super_token_with_projects_read_scope
KBC_MANAGE_API_SUPER_TOKEN_WITHOUT_SCOPES=super_token_without_scopes
KBC_MANAGE_API_SUPER_TOKEN_WITH_DELETED_PROJECTS_READ_SCOPE=super_token_with_deleted_projects_read_scope
KBC_MANAGE_API_SUPER_TOKEN_WITH_UI_MANAGE_SCOPE=super_token_with_ui_manage_scope
KBC_TEST_MAINTAINER_ID=2
KBC_TEST_ADMIN_EMAIL=email_of_another_admin_having_mfa_disabled
KBC_TEST_ADMIN_TOKEN=token_of_another_admin_having_mfa_disabled
KBC_TEST_ADMIN_WITH_MFA_EMAIL=email_of_another_admin_having_mfa_enabled
KBC_TEST_ADMIN_WITH_MFA_TOKEN=token_of_another_admin_having_mfa_enabled

# OPTIONAL - required only for running file storage test (tests are skipped by default)
TEST_ABS_ACCOUNT_KEY=
TEST_ABS_ACCOUNT_NAME=
TEST_ABS_CONTAINER_NAME=
TEST_ABS_REGION=
TEST_ABS_ROTATE_ACCOUNT_KEY=
TEST_S3_ROTATE_KEY=
TEST_S3_ROTATE_SECRET=
TEST_S3_FILES_BUCKET=
TEST_S3_KEY=
TEST_S3_REGION=
TEST_S3_SECRET=
TEST_GCS_KEYFILE_JSON=
TEST_GCS_KEYFILE_ROTATE_JSON=
TEST_GCS_FILES_BUCKET=
TEST_GCS_REGION=
KBC_MANAGE_API_SUPER_TOKEN_WITH_UI_MANAGE_SCOPE=super_token_with_ui_manage_scope # can be created in manage-apps on the Tokens tab. Token must have "ui manage" scope
KBC_TEST_MAINTAINER_ID=id # `id` of maintainer. Please create a new maintainer dedicated to test suite. All maintainer's organizations and projects all purged before tests!
KBC_TEST_ADMIN_EMAIL=email_of_another_admin_having_mfa_disabled # email address of another user without any organizations
KBC_TEST_ADMIN_TOKEN=token_of_another_admin_having_mfa_disabled # is also a Personal Access Token of user **without** **superadmin** privileges , but for a different user than that which has `KBC_MANAGE_API_TOKEN`. User must have Multi-Factor Authentication disabled.
KBC_TEST_ADMIN_WITH_MFA_EMAIL=email_of_another_admin_having_mfa_enabled # email address of another user without any organizations and having Multi-Factor Authentication enabled
KBC_TEST_ADMIN_WITH_MFA_TOKEN=token_of_another_admin_having_mfa_enabled # is also a Personal Access Token of user **without** **superadmin** privileges , but for a different user than that which has `KBC_MANAGE_API_TOKEN` or `KBC_TEST_ADMIN_TOKEN`

# OPTIONAL - required only for running testCreateStorageBackend, you have to have new snowflake backend and fill credentials into following environment variables

KBC_TEST_SNOWFLAKE_BACKEND_NAME=
KBC_TEST_SNOWFLAKE_BACKEND_PASSWORD=
KBC_TEST_SNOWFLAKE_HOST=
KBC_TEST_SNOWFLAKE_WAREHOUSE=
KBC_TEST_SNOWFLAKE_BACKEND_REGION=

```

Source newly created file and run tests:

Run tests
```bash
docker-compose run --rm dev composer tests
```

### Required variables

- `KBC_MANAGE_API_URL` - URL where Keboola Connection is running
- `KBC_MANAGE_API_TOKEN` - manage api token assigned to user **with** **superadmin** privileges. Can be created in Account Settings under the title Personal Access Tokens. User must have Multi-Factor Authentication disabled.
- `KBC_SUPER_API_TOKEN` - can be created in manage-apps on the Tokens tab
- `KBC_MANAGE_API_SUPER_TOKEN_WITH_UI_MANAGE_SCOPE` - can be created in manage-apps on the Tokens tab. Token must have "ui manage" scope
- `KBC_TEST_MAINTAINER_ID` - `id` of maintainer. Please create a new maintainer dedicated to test suite. All maintainer's organizations and projects all purged before tests!
- `KBC_TEST_ADMIN_EMAIL` - email address of another user without any organizations
- `KBC_TEST_ADMIN_TOKEN` - is also a Personal Access Token of user **without** **superadmin** privileges , but for a different user than that which has `KBC_MANAGE_API_TOKEN`. User must have Multi-Factor Authentication disabled.
- `KBC_TEST_ADMIN_WITH_MFA_EMAIL` - email address of another user without any organizations and having Multi-Factor Authentication enabled
- `KBC_TEST_ADMIN_WITH_MFA_TOKEN` - is also a Personal Access Token of user **without** **superadmin** privileges , but for a different user than that which has `KBC_MANAGE_API_TOKEN` or `KBC_TEST_ADMIN_TOKEN`
## File Storage tests

### Setup cloud resources for File Storage tests

#### Prerequisites:

- configured and logged in az, aws and gcp CLI tools
- `az login`
- `aws sso login --profile=<your_profile>`
- `gcloud auth application-default login`
- installed terraform (https://www.terraform.io) and jq (https://stedolan.github.io/jq) to setup local env

```shell
# create terraform.tfvars file from terraform.tfvars.dist
cp ./provisioning/terraform.tfvars.dist ./provisioning/terraform.tfvars

# set terraform variables
name_prefix = "<your_nick>" # your name/nickname to make your resource unique & recognizable, allowed characters are [a-zA-Z0-9-]
gcp_storage_location = "<your_region>" # region of GCP resources
gcp_project_id = "<your_project_id>" # GCP project id
gcp_project_region = "<your_region>" # region of GCP project
azure_storage_location = "<your_region>" # region of Azure resources
azure_tenant_id = "<your_tenant_id>" # Azure tenant id
azure_subscription_id = "<your_subscription_id>" # Azure subscription id
aws_profile = "<your_profile>" # your aws profile name
aws_region = "<your_region>" # region of AWS resources
aws_account = "<your_account_id>" # your aws account id


# Initialize terraform
terraform -chdir=./provisioning init
# Create resources
terraform -chdir=./provisioning apply

# For destroying resources run
terraform -chdir=./provisioning apply -destroy

### Optional variables
# Setup terraform variables to .env file (will be prepended to .env file)
# For Azure
./provisioning/update-env.sh azure
# For Aws
./provisioning/update-env.sh aws
# For GCP
./provisioning/update-env.sh gcp
```

### Required variables for File Storage tests

These variables are used for testing file storage. You have to copy these values from Azure and AWS portal.
- `TEST_ABS_ACCOUNT_KEY` - First secret key for Azure Storage account
Expand All @@ -152,7 +167,13 @@ These variables are used for testing file storage. You have to copy these values

Variable prefixed with _ROTATE_ are used for rotating credentials and they MUST be working credentials.

## License

### Run File Storage tests

```bash
docker-compose run --rm dev composer tests-file-storage
```


## Build OpenAPI document

Expand All @@ -167,4 +188,7 @@ Then run following commands
$ cat apiary.apib | grep -v "X-KBC-ManageApiToken:" | apib2swagger -o openapi.yml -y --open-api-3 --info-title="Manage API"
$ php AdjustApi.php
```

## License

MIT licensed, see [LICENSE](./LICENSE) file.
3 changes: 2 additions & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
"phpcs": "phpcs -s -n -p --extensions=php src tests",
"phpcbf": "phpcbf -s -n -p --extensions=php src tests",
"phpstan": "phpstan analyse ./src ./tests -c phpstan.neon",
"tests": "phpunit"
"tests": "phpunit --testsuite 'Main'",
"tests-file-storage": "phpunit --testsuite 'File Storage'"
},
"authors": [
{
Expand Down
7 changes: 6 additions & 1 deletion phpunit.xml.dist
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@
<phpunit bootstrap="./tests/bootstrap.php"
colors="true">
<testsuites>
<testsuite>
<testsuite name="Main">
<directory>tests</directory>
<exclude>tests/FileStorageAbsTest.php</exclude>
<exclude>tests/FileStorageS3Test.php</exclude>
<exclude>tests/FileStorageGcsTest.php</exclude>
</testsuite>
<testsuite name="File Storage">
<file>tests/FileStorageAbsTest.php</file>
<file>tests/FileStorageS3Test.php</file>
<file>tests/FileStorageGcsTest.php</file>
</testsuite>
</testsuites>
<filter>
<whitelist>
Expand Down
5 changes: 5 additions & 0 deletions provisioning/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
.terraform.lock.hcl
.terraform
terraform.tfstate
terraform.tfstate.backup
terraform.tfvars
77 changes: 77 additions & 0 deletions provisioning/aws.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
provider "aws" {
allowed_account_ids = [var.aws_account]
region = var.aws_region
profile = var.aws_profile
}

variable "aws_profile" {
type = string
}
variable "aws_region" {
type = string
}
variable "aws_account" {
type = string
}

data "aws_region" "current" {}
data "aws_caller_identity" "current" {}

# Create resources

module "aws_storage" {
source = "./modules/aws/storage"

service_name = local.service_name
}

module "aws_policy" {
source = "./modules/aws/policy"

service_name = local.service_name
bucket_arn = module.aws_storage.bucket_arn
}

module "aws_user" {
source = "./modules/aws/user"

service_name = local.service_name
policy_arn = module.aws_policy.policy_arn
}

module "aws_user_rotate" {
source = "./modules/aws/user"

service_name = local.rotate_service_name
policy_arn = module.aws_policy.policy_arn
}

# Outputs

output "TEST_S3_REGION" {
value = data.aws_region.current.id
description = "Region where your S3 is located"
}
output "TEST_S3_KEY" {
value = module.aws_user.access_key_id
description = "First AWS key"
}
output "TEST_S3_SECRET" {
value = module.aws_user.access_key_secret
sensitive = true
description = "First AWS secret"
}
output "TEST_S3_ROTATE_KEY" {
value = module.aws_user_rotate.access_key_id
description = "Second AWS key"
}
output "TEST_S3_ROTATE_SECRET" {
value = module.aws_user_rotate.access_key_secret
sensitive = true
description = "Second AWS secret"
}
output "TEST_S3_FILES_BUCKET" {
value = module.aws_storage.bucket_name
description = "Name of file bucket on S3"
}

62 changes: 62 additions & 0 deletions provisioning/azure.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
provider "azurerm" {
features {}
tenant_id = var.azure_tenant_id
subscription_id = var.azure_subscription_id
}
provider "azuread" {
tenant_id = var.azure_tenant_id
}

variable "azure_storage_location" {
type = string
}
variable "azure_tenant_id" {
type = string
}
variable "azure_subscription_id" {
type = string
}

data "azurerm_client_config" "current" {}
data "azuread_client_config" "current" {}

# Create resources

resource "azurerm_resource_group" "mapi_resource_group" {
name = "${var.name_prefix}-mapi_resource_group"
location = var.azure_storage_location
}

module "azure_storage" {
source = "./modules/azure/storage"

resource_group_location = azurerm_resource_group.mapi_resource_group.location
resource_group_uuid = substr(md5(azurerm_resource_group.mapi_resource_group.id), 0, 17)
resource_group_name = azurerm_resource_group.mapi_resource_group.name
files_container = "dummy"
}

# Outputs

output "TEST_ABS_ACCOUNT_NAME" {
value = module.azure_storage.storage_account_files_name
description = "Name of Azure Storage account"
}
output "TEST_ABS_ACCOUNT_KEY" {
value = module.azure_storage.storage_account_primary_access_key
sensitive = true
description = "First secret key for Azure Storage account"
}
output "TEST_ABS_CONTAINER_NAME" {
value = module.azure_storage.storage_container_files_test_container_name
description = "Name of container created inside Azure Storage Account"
}
output "TEST_ABS_REGION" {
value = module.azure_storage.storage_account_location
description = "Name of region where Azure Storage Account is located"
}
output "TEST_ABS_ROTATE_ACCOUNT_KEY" {
value = module.azure_storage.storage_account_secondary_access_key
sensitive = true
description = "Second secret key for Azure Storage account"
}
12 changes: 12 additions & 0 deletions provisioning/env-scripts/extract-variables-aws.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -Eeuo pipefail

cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source ./functions.sh

output_var 'TEST_S3_REGION' "$(terraform_output 'TEST_S3_REGION')"
output_var 'TEST_S3_FILES_BUCKET' "$(terraform_output 'TEST_S3_FILES_BUCKET')"
output_var 'TEST_S3_KEY' "$(terraform_output 'TEST_S3_KEY')"
output_var 'TEST_S3_SECRET' "$(terraform_output 'TEST_S3_SECRET')"
output_var 'TEST_S3_ROTATE_KEY' "$(terraform_output 'TEST_S3_ROTATE_KEY')"
output_var 'TEST_S3_ROTATE_SECRET' "$(terraform_output 'TEST_S3_ROTATE_SECRET')"
11 changes: 11 additions & 0 deletions provisioning/env-scripts/extract-variables-azure.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -Eeuo pipefail

cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source ./functions.sh

output_var 'TEST_ABS_ACCOUNT_KEY' "$(terraform_output 'TEST_ABS_ACCOUNT_KEY')"
output_var 'TEST_ABS_ACCOUNT_NAME' "$(terraform_output 'TEST_ABS_ACCOUNT_NAME')"
output_var 'TEST_ABS_CONTAINER_NAME' "$(terraform_output 'TEST_ABS_CONTAINER_NAME')"
output_var 'TEST_ABS_REGION' "$(terraform_output 'TEST_ABS_REGION')"
output_var 'TEST_ABS_ROTATE_ACCOUNT_KEY' "$(terraform_output 'TEST_ABS_ROTATE_ACCOUNT_KEY')"
10 changes: 10 additions & 0 deletions provisioning/env-scripts/extract-variables-gcp.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -Eeuo pipefail

cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source ./functions.sh

output_var 'TEST_GCS_REGION' "$(terraform_output 'TEST_GCS_REGION')"
output_var 'TEST_GCS_FILES_BUCKET' "$(terraform_output 'TEST_GCS_FILES_BUCKET')"
output_var_json 'TEST_GCS_KEYFILE_JSON' "$(terraform_output 'TEST_GCS_KEYFILE_JSON')"
output_var_json 'TEST_GCS_KEYFILE_ROTATE_JSON' "$(terraform_output 'TEST_GCS_KEYFILE_ROTATE_JSON')"
26 changes: 26 additions & 0 deletions provisioning/env-scripts/functions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -Eeuo pipefail

SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="${SCRIPT_PATH}/../.."

terraform_output() {
jq ".${1}.value" -r $PROJECT_ROOT/provisioning/tfoutput.json
}

terraform_output_json() {
jq ".${1}.value" -r $PROJECT_ROOT/provisioning/tfoutput.json | jq -c
}

output_var() {
echo "${1}=\"${2}\""
}

output_var_json() {
echo "${1}='${2}'"
}

output_file() {
mkdir -p "${PROJECT_ROOT}/$(dirname "${1}")"
echo "${2}" >"${PROJECT_ROOT}/${1}"
}
Loading

0 comments on commit 17828e4

Please sign in to comment.