From ae45350af6bf6ca94af1850c9c04652553e43ecb Mon Sep 17 00:00:00 2001 From: John ODonnell Date: Fri, 15 Dec 2023 10:26:46 -0500 Subject: [PATCH 1/9] CNJR-2127: Use Docker Compose v2 --- ci/start_dev_environment | 2 +- ci/test_e2e | 2 +- ci/test_integration | 2 +- ci/utils | 8 ++++---- conjur-env/build.sh | 4 ++-- tests/retrieve-secrets/README.md | 2 +- tests/retrieve-secrets/start | 8 ++++---- tests/retrieve-secrets/stop | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ci/start_dev_environment b/ci/start_dev_environment index 30e9249..40e1988 100755 --- a/ci/start_dev_environment +++ b/ci/start_dev_environment @@ -9,7 +9,7 @@ build_test_images start_conjur # Runs the cflinux4 image in interactive mode with the project files mounted -docker-compose \ +docker compose \ -f "$DOCKER_COMPOSE_FILE" \ run --rm \ -e BUILDPACK_BUILD_DIR="/cyberark/cloudfoundry-conjur-buildpack/conjur_buildpack" \ diff --git a/ci/test_e2e b/ci/test_e2e index afa3cf3..3fa4d4b 100755 --- a/ci/test_e2e +++ b/ci/test_e2e @@ -49,7 +49,7 @@ popd announce 'Running Cucumber tests...' # Run tests against latest build of buildpack (including integration tests against remote foundation) -docker-compose \ +docker compose \ -f "$DOCKER_COMPOSE_FILE" \ run --rm \ -w "$CONTAINER_FEATURES_DIR" \ diff --git a/ci/test_integration b/ci/test_integration index 048e8ce..3334a97 100755 --- a/ci/test_integration +++ b/ci/test_integration @@ -23,7 +23,7 @@ start_conjur [[ -z $JENKINS_HOME ]] && package_and_unpack_buildpack announce 'Running Cucumber tests...' -docker-compose \ +docker compose \ -f "$DOCKER_COMPOSE_FILE" \ run --rm \ -w "$CONTAINER_FEATURES_DIR" \ diff --git a/ci/utils b/ci/utils index 45dba29..4dcab15 100755 --- a/ci/utils +++ b/ci/utils @@ -13,7 +13,7 @@ function announce { function finish { announce 'Removing environment...' - docker-compose -f "$DOCKER_COMPOSE_FILE" down -v + docker compose -f "$DOCKER_COMPOSE_FILE" down -v } function package_and_unpack_buildpack { @@ -36,14 +36,14 @@ function setup_env { function build_test_images { announce 'Building test images...' - docker-compose -f "$DOCKER_COMPOSE_FILE" build + docker compose -f "$DOCKER_COMPOSE_FILE" build } function start_conjur { announce 'Waiting for Conjur to start...' - docker-compose -f "$DOCKER_COMPOSE_FILE" up -d conjur + docker compose -f "$DOCKER_COMPOSE_FILE" up -d conjur - docker-compose -f "$DOCKER_COMPOSE_FILE" exec -T conjur conjurctl wait -r 45 -p 80 + docker compose -f "$DOCKER_COMPOSE_FILE" exec -T conjur conjurctl wait -r 45 -p 80 } diff --git a/conjur-env/build.sh b/conjur-env/build.sh index 935f9bb..094875a 100755 --- a/conjur-env/build.sh +++ b/conjur-env/build.sh @@ -4,5 +4,5 @@ cd "$(dirname "$0")" rm -rf ../vendor/conjur-env -docker-compose build -docker-compose run --rm conjur-env-builder +docker compose build +docker compose run --rm conjur-env-builder diff --git a/tests/retrieve-secrets/README.md b/tests/retrieve-secrets/README.md index 03be222..706caf2 100644 --- a/tests/retrieve-secrets/README.md +++ b/tests/retrieve-secrets/README.md @@ -49,7 +49,7 @@ environment, then assert upon the output. BATS offers the `$output` variable, w contains the output from the previous command executed using the `run` prefix. ### `stop` -Removes existing docker-compose containers +Removes existing Docker Compose environment ### Configuration diff --git a/tests/retrieve-secrets/start b/tests/retrieve-secrets/start index b25ef94..f9e16b1 100755 --- a/tests/retrieve-secrets/start +++ b/tests/retrieve-secrets/start @@ -10,13 +10,13 @@ mkdir -p temp_clone_dir git clone https://github.com/ztombol/bats-support temp_clone_dir/bats-support git clone https://github.com/ztombol/bats-assert temp_clone_dir/bats-assert -docker-compose build -docker-compose run --rm mock-conjur-env-builder +docker compose build +docker compose run --rm mock-conjur-env-builder echo "Starting tests for 0001_retrieve-secrets.sh..." # Uncomment when running locally -# docker-compose run --rm tester --formatter tap ./test +# docker compose run --rm tester --formatter tap ./test # Comment out when running locally -docker-compose run --rm tester --formatter junit ./test +docker compose run --rm tester --formatter junit ./test diff --git a/tests/retrieve-secrets/stop b/tests/retrieve-secrets/stop index 9cc9f94..356cea7 100755 --- a/tests/retrieve-secrets/stop +++ b/tests/retrieve-secrets/stop @@ -4,7 +4,7 @@ function finish { echo 'Removing environment...' rm -rf temp_clone_dir - docker-compose down -v + docker compose down -v } finish From 8c883f3235276404fb5663a4e493317a92c6aa6a Mon Sep 17 00:00:00 2001 From: John ODonnell Date: Fri, 15 Dec 2023 13:09:33 -0500 Subject: [PATCH 2/9] Ruby test app: swap Thin for Puma --- tests/integration/apps/ruby/Gemfile | 2 +- tests/integration/apps/ruby/Gemfile.lock | 11 ++++------- tests/integration/apps/ruby/Procfile | 1 + 3 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 tests/integration/apps/ruby/Procfile diff --git a/tests/integration/apps/ruby/Gemfile b/tests/integration/apps/ruby/Gemfile index d7336e8..8114b04 100644 --- a/tests/integration/apps/ruby/Gemfile +++ b/tests/integration/apps/ruby/Gemfile @@ -5,5 +5,5 @@ ruby '~> 3.2' gem 'conjur-api' gem 'conjur-cli' -gem 'thin' +gem 'puma' gem 'roda' diff --git a/tests/integration/apps/ruby/Gemfile.lock b/tests/integration/apps/ruby/Gemfile.lock index 93938dd..b5c1e2b 100644 --- a/tests/integration/apps/ruby/Gemfile.lock +++ b/tests/integration/apps/ruby/Gemfile.lock @@ -23,11 +23,9 @@ GEM netrc (~> 0.10) table_print (~> 1.5) xdg (= 2.2.3) - daemons (1.4.1) deep_merge (1.2.2) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - eventmachine (1.2.7) gli (2.21.0) highline (2.1.0) http-accept (1.7.0) @@ -40,7 +38,10 @@ GEM mime-types-data (3.2023.0218.1) minitest (5.18.0) netrc (0.11.0) + nio4r (2.7.0) public_suffix (5.0.1) + puma (6.4.0) + nio4r (~> 2.0) rack (3.0.1) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -50,10 +51,6 @@ GEM roda (3.68.0) rack table_print (1.5.7) - thin (1.8.2) - daemons (~> 1.0, >= 1.0.9) - eventmachine (~> 1.0, >= 1.0.4) - rack (>= 1, < 3) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unf (0.1.4) @@ -68,8 +65,8 @@ PLATFORMS DEPENDENCIES conjur-api conjur-cli + puma roda - thin RUBY VERSION ruby 3.2.2p53 diff --git a/tests/integration/apps/ruby/Procfile b/tests/integration/apps/ruby/Procfile new file mode 100644 index 0000000..8136a6f --- /dev/null +++ b/tests/integration/apps/ruby/Procfile @@ -0,0 +1 @@ +web: bundle exec puma \ No newline at end of file From 7882860034369b04417647c7a566f0bec50ddbba Mon Sep 17 00:00:00 2001 From: John ODonnell Date: Fri, 15 Dec 2023 13:54:14 -0500 Subject: [PATCH 3/9] Dump CF app logs --- tests/integration/features/integration.feature | 18 ------------------ .../step_definitions/integration_steps.rb | 2 +- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/tests/integration/features/integration.feature b/tests/integration/features/integration.feature index d27e8c8..c35025f 100644 --- a/tests/integration/features/integration.feature +++ b/tests/integration/features/integration.feature @@ -14,28 +14,10 @@ Feature: Integrations Tests for remote TAS foundation Given I create an org and space And I install the buildpack - Scenario: Python offline buildpack integration - When I push a "python" app with the "offline" buildpack - Then the secrets.yml values are available in the app - Scenario: Ruby offline buildpack integration When I push a "ruby" app with the "offline" buildpack Then the secrets.yml values are available in the app - Scenario: Java offline buildpack integration - When I push a "java" app with the "offline" buildpack - Then the secrets.yml values are available in the app - -# # The online buildpack tests are only valid if the latest commits -# # are push to the Github remote branch. - Scenario: Python online buildpack integration - When I push a "python" app with the "online" buildpack - Then the secrets.yml values are available in the app - Scenario: Ruby online buildpack integration When I push a "ruby" app with the "online" buildpack Then the secrets.yml values are available in the app - - Scenario: Java online buildpack integration - When I push a "java" app with the "online" buildpack - Then the secrets.yml values are available in the app diff --git a/tests/integration/features/step_definitions/integration_steps.rb b/tests/integration/features/step_definitions/integration_steps.rb index 4358880..75f7271 100644 --- a/tests/integration/features/step_definitions/integration_steps.rb +++ b/tests/integration/features/step_definitions/integration_steps.rb @@ -24,7 +24,7 @@ else create_offline_app_manifest end - ShellSession.execute("cf push #{@app_name} --random-route") + ShellSession.execute("cf push #{@app_name} --random-route; cf logs #{@app_name}") end end From 73005b0e9f2aa7b256fc75a7a8bd87b929348fa5 Mon Sep 17 00:00:00 2001 From: Kumbirai Tanekha Date: Wed, 27 Mar 2024 17:38:30 +0000 Subject: [PATCH 4/9] Explicitly specify docker-compose build context for conjur-env-builder --- conjur-env/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conjur-env/docker-compose.yml b/conjur-env/docker-compose.yml index f39bc71..26e5ee7 100644 --- a/conjur-env/docker-compose.yml +++ b/conjur-env/docker-compose.yml @@ -1,7 +1,8 @@ version: '2.1' services: conjur-env-builder: - build: "" + build: + context: . volumes: - .:/conjur-env - ../vendor:/pkg From 57add60280b9ba2cdab4764eda275b446124799a Mon Sep 17 00:00:00 2001 From: Shlomo Heigh Date: Wed, 27 Mar 2024 14:55:47 -0400 Subject: [PATCH 5/9] Specify docker compose build context for mock-conjur-env-builder --- tests/retrieve-secrets/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/retrieve-secrets/docker-compose.yml b/tests/retrieve-secrets/docker-compose.yml index cef7d8b..33e3fbd 100644 --- a/tests/retrieve-secrets/docker-compose.yml +++ b/tests/retrieve-secrets/docker-compose.yml @@ -1,7 +1,8 @@ version: '2.1' services: mock-conjur-env-builder: - build: "" + build: + context: . volumes: - .:/mock-conjur-env - ../../tmp/vendor:/pkg From a4849be55a76e0e735e438f46f10572d8fc83a80 Mon Sep 17 00:00:00 2001 From: Shlomo Heigh Date: Wed, 27 Mar 2024 14:58:30 -0400 Subject: [PATCH 6/9] Remove obsolete 'version' from docker-compose.yml files --- conjur-env/docker-compose.yml | 1 - tests/docker-compose.yml | 1 - tests/retrieve-secrets/docker-compose.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/conjur-env/docker-compose.yml b/conjur-env/docker-compose.yml index 26e5ee7..b101a55 100644 --- a/conjur-env/docker-compose.yml +++ b/conjur-env/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.1' services: conjur-env-builder: build: diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index e6e2c43..a81b22f 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2' services: tester: build: . diff --git a/tests/retrieve-secrets/docker-compose.yml b/tests/retrieve-secrets/docker-compose.yml index 33e3fbd..d236183 100644 --- a/tests/retrieve-secrets/docker-compose.yml +++ b/tests/retrieve-secrets/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.1' services: mock-conjur-env-builder: build: From 035f267aef674111759db07fa47065155f47d1c4 Mon Sep 17 00:00:00 2001 From: Shlomo Heigh Date: Thu, 9 May 2024 10:41:58 -0400 Subject: [PATCH 7/9] Fix port number with Puma --- tests/integration/apps/ruby/Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/apps/ruby/Procfile b/tests/integration/apps/ruby/Procfile index 8136a6f..5d48018 100644 --- a/tests/integration/apps/ruby/Procfile +++ b/tests/integration/apps/ruby/Procfile @@ -1 +1 @@ -web: bundle exec puma \ No newline at end of file +web: bundle exec puma -b tcp://0.0.0.0:8080 \ No newline at end of file From 81412389168ad9223b68f2512ea757728ebb12dd Mon Sep 17 00:00:00 2001 From: Shlomo Heigh Date: Thu, 9 May 2024 14:13:54 -0400 Subject: [PATCH 8/9] Fix app install hanging --- .../integration/features/step_definitions/integration_steps.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/features/step_definitions/integration_steps.rb b/tests/integration/features/step_definitions/integration_steps.rb index 75f7271..538a2df 100644 --- a/tests/integration/features/step_definitions/integration_steps.rb +++ b/tests/integration/features/step_definitions/integration_steps.rb @@ -24,7 +24,7 @@ else create_offline_app_manifest end - ShellSession.execute("cf push #{@app_name} --random-route; cf logs #{@app_name}") + ShellSession.execute("cf push #{@app_name} --random-route; cf logs #{@app_name} --recent") end end From bba8b1e43b3a05a3155cbe2ee55ac026eed1d77d Mon Sep 17 00:00:00 2001 From: Shlomo Heigh Date: Thu, 9 May 2024 14:40:15 -0400 Subject: [PATCH 9/9] Add back all integration test cases --- tests/integration/features/integration.feature | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/features/integration.feature b/tests/integration/features/integration.feature index c35025f..9a905ad 100644 --- a/tests/integration/features/integration.feature +++ b/tests/integration/features/integration.feature @@ -14,10 +14,28 @@ Feature: Integrations Tests for remote TAS foundation Given I create an org and space And I install the buildpack + Scenario: Python offline buildpack integration + When I push a "python" app with the "offline" buildpack + Then the secrets.yml values are available in the app + Scenario: Ruby offline buildpack integration When I push a "ruby" app with the "offline" buildpack Then the secrets.yml values are available in the app + Scenario: Java offline buildpack integration + When I push a "java" app with the "offline" buildpack + Then the secrets.yml values are available in the app + + # The online buildpack tests are only valid if the latest commits + # are push to the Github remote branch. + Scenario: Python online buildpack integration + When I push a "python" app with the "online" buildpack + Then the secrets.yml values are available in the app + Scenario: Ruby online buildpack integration When I push a "ruby" app with the "online" buildpack Then the secrets.yml values are available in the app + + Scenario: Java online buildpack integration + When I push a "java" app with the "online" buildpack + Then the secrets.yml values are available in the app