From 8c362f76fa6ae88f75a47320dfb610e8cb66ae6f Mon Sep 17 00:00:00 2001 From: Justin Harringa Date: Mon, 22 Jan 2018 16:54:01 -0800 Subject: [PATCH 1/4] Abort on container exit for docker-compose --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07f3ef7d..b03c9748 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ script: - docker build -t salesforce/dockerfile-image-update . - echo git_api_token=${ITEST_GH_TOKEN} > `pwd`/itest.env - export user_itest_secrets_file_secret=`pwd`/itest.env -- docker-compose up +- docker-compose up --abort-on-container-exit after_success: - mvn coveralls:report deploy: From 62d681ff0384d74c140aa3949e108c78ba3b758c Mon Sep 17 00:00:00 2001 From: Justin Harringa Date: Wed, 21 Feb 2018 21:09:32 -0800 Subject: [PATCH 2/4] Fix integration test jar location --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index de26908f..558cadde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: # This allows us to get our test results back to workspace for the pipeline to pick up - ./test-results:/tmp/test-results # Place integration test into the component so that we don't have to ship integration tests inside the component - - ./dockerfile-image-update-itest/target/integration-test-1.0-SNAPSHOT.jar:/tmp/integration-test.jar + - ./dockerfile-image-update-itest/target/dockerfile-image-update-itest-1.0-SNAPSHOT.jar:/tmp/integration-test.jar # If you have a secret file, you can volume mount the file to gain access to it # export user_itest_secrets_file_secret=/path/to/secretFile in shell # For CI purposes, the secret file can be added to Jenkins Credential Store with the id user_itest_secrets_file From 9e4a0e48deb921e321ddbb7f34b8e935e6a56362 Mon Sep 17 00:00:00 2001 From: Andrey Falko Date: Thu, 26 Jul 2018 16:12:28 -0700 Subject: [PATCH 3/4] Improve itest readme and don't fail on clean up fail --- README.md | 31 +++++++++++++------ .../itest/tests/AllCommandTest.java | 2 +- .../itest/tests/ChildCommandTest.java | 2 +- .../itest/tests/ParentCommandTest.java | 2 +- .../itest/tests/TestCommon.java | 25 ++++++++++----- .../utils/DockerfileGitHubUtil.java | 1 + 6 files changed, 43 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4abf5935..b7efe28b 100644 --- a/README.md +++ b/README.md @@ -150,18 +150,31 @@ java -jar dockerfile-image-update-1.0-SNAPSHOT.jar ``` ### Creating a new feature -Under dockerfile-image-update/src/main/java/com/salesforce/dva/dockerfileimageupdate/subcommands/impl, create a new class `YOUR_FEATURE.java`. Make sure it implements `ExecutableWithNamespace` and has the `SubCommand` annotation with a `help`, `requiredParams`, and `optionalParams`. Then, under the `execute` method, code what you want this tool to do. +Under [dockerfile-image-update/src/main/java/com/salesforce/dva/dockerfileimageupdate/subcommands/impl](https://github.com/salesforce/dockerfile-image-update/tree/master/dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/subcommands/impl), +create a new class `YOUR_FEATURE.java`. Make sure it implements `ExecutableWithNamespace` and has the `SubCommand` +annotation with a `help`, `requiredParams`, and `optionalParams`. Then, under the `execute` method, code what you want this tool to do. ### Running unit tests Run unit tests by running `mvn test`. ### Running integration tests Before you run the integration tests (locally): - * Make sure that you have access to the github orgs specified in TestCommon.ORGS. You likely will need to change it to three - orgs where you have permissions to create repositories. - * Make sure you have `git_api_url=https://api.github.com` in `/dockerfile-image-update-itest/itest.env`. - * Make sure you have a secret file which contains the `git_api_token`. This needs access to CRUD repositories and - github statuses. - * Export the following environment variable: `export user_itest_secrets_file_secret=/path/to/secretFile` - * Run integration tests by running `make itest-local-changes`. - + 1. Make sure that you have access to the github orgs specified in [TestCommon.ORGS](https://github.com/salesforce/dockerfile-image-update/blob/master/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java#L33). You likely will need to change it to three + orgs where you have permissions to create repositories. + 2. Make sure you have `git_api_url=https://api.github.com` in `/dockerfile-image-update-itest/itest.env`, + or set it to your internal GitHub Enterprise. + 3. Make sure you have a secret file which contains the `git_api_token`. + The token needs to have `delete_repo, repo` permissions. + You can generate your token by going to [personal access tokens](https://github.com/settings/tokens/new) in GitHub. + Once you have your token place it in a file: + ``` + echo git_api_token=[copy personal access token here] > ${HOME}/.dfiu-itest-token + ``` + 4. Export the following environment variable to point to the file: + ``` + export user_itest_secrets_file_secret=${HOME}/.dfiu-itest-token + ``` + 5. Run integration tests by running + ``` + make itest-local-changes + ``` diff --git a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/AllCommandTest.java b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/AllCommandTest.java index c79a7311..f3f44c52 100644 --- a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/AllCommandTest.java +++ b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/AllCommandTest.java @@ -118,7 +118,7 @@ public void testIdempotency() throws Exception{ @AfterClass public void cleanUp() throws Exception { addVersionStoreRepo(github, createdRepos, STORE_NAME); - cleanAllRepos(createdRepos); + cleanAllRepos(createdRepos, false); } } diff --git a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ChildCommandTest.java b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ChildCommandTest.java index 79450e4f..e92c4723 100644 --- a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ChildCommandTest.java +++ b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ChildCommandTest.java @@ -158,6 +158,6 @@ private void cleanBefore() throws Exception { @AfterClass public void cleanUp() throws Exception { addVersionStoreRepo(github, createdRepos, STORE_NAME); - TestCommon.cleanAllRepos(createdRepos); + TestCommon.cleanAllRepos(createdRepos, false); } } diff --git a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ParentCommandTest.java b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ParentCommandTest.java index e0cda0bd..f18fb08f 100644 --- a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ParentCommandTest.java +++ b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ParentCommandTest.java @@ -145,6 +145,6 @@ public void testStoreUpdate() throws Exception { @AfterClass public void cleanUp() throws Exception { addVersionStoreRepo(github, createdRepos, STORE_NAME); - TestCommon.cleanAllRepos(createdRepos); + TestCommon.cleanAllRepos(createdRepos, false); } } diff --git a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java index 4342cd88..31abff96 100644 --- a/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java +++ b/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java @@ -12,6 +12,7 @@ import org.kohsuke.github.GHOrganization; import org.kohsuke.github.GHRepository; import org.kohsuke.github.GitHub; +import org.kohsuke.github.PagedIterable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,21 +58,21 @@ public static void initializeRepos(GHOrganization org, List repos, Strin } } - public static void printCollectedExceptionsAndFail(List exceptions) { + public static void printCollectedExceptionsAndFail(List exceptions, boolean exitWithFail) { for (int i = 0; i < exceptions.size(); i++) { log.error("Hit exception {}/{} while cleaning up.", i+1, exceptions.size()); log.error("", exceptions.get(i)); } - if (exceptions.size() > 0) { + if (exitWithFail && exceptions.size() > 0) { throw new RuntimeException(exceptions.get(0)); } } - public static void cleanAllRepos(List createdRepos) throws Exception { + public static void cleanAllRepos(List createdRepos, boolean exitWithFail) throws Exception { List exceptions = new ArrayList<>(); exceptions.addAll(checkAndDelete(createdRepos)); - TestCommon.printCollectedExceptionsAndFail(exceptions); + TestCommon.printCollectedExceptionsAndFail(exceptions, false); } private static Exception checkAndDelete(GHRepository repo) { @@ -87,11 +88,19 @@ private static Exception checkAndDelete(GHRepository repo) { private static List checkAndDelete(List repos) throws IOException { List exceptions = new ArrayList<>(); for (GHRepository repo : repos) { - for (GHRepository fork : repo.listForks()) { - Exception forkDeleteException = checkAndDelete(fork); - if (forkDeleteException != null) { - exceptions.add(forkDeleteException); + + PagedIterable forks; + try { + forks = repo.listForks(); + for (GHRepository fork : forks) { + Exception forkDeleteException = checkAndDelete(fork); + if (forkDeleteException != null) { + exceptions.add(forkDeleteException); + } } + } catch (Exception getForksException) { + log.error("Could not get forks for repo: ", repo.getFullName()); + exceptions.add(getForksException); } Exception repoDeleteException = checkAndDelete(repo); if (repoDeleteException != null) { diff --git a/dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/utils/DockerfileGitHubUtil.java b/dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/utils/DockerfileGitHubUtil.java index 021b87a2..7e399e0e 100644 --- a/dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/utils/DockerfileGitHubUtil.java +++ b/dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/utils/DockerfileGitHubUtil.java @@ -199,6 +199,7 @@ public void updateStore(String store, String img, String tag) throws IOException log.info("Image tag store cannot be null. Skipping store update..."); return; } + log.info("Updating store: {} with image: {} tag: {}...", store, img, tag); GHRepository storeRepo; try { GHMyself myself = gitHubUtil.getMyself(); From 6fedc1f657f971eaef70a5d60295aa6005382e8a Mon Sep 17 00:00:00 2001 From: Andrey Falko Date: Thu, 26 Jul 2018 17:53:10 -0700 Subject: [PATCH 4/4] Add comment about itests flapping --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ff80e619..f711e4fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ script: - docker build -t salesforce/dockerfile-image-update . - echo git_api_token=${ITEST_GH_TOKEN} > `pwd`/itest.env - export user_itest_secrets_file_secret=`pwd`/itest.env -- docker-compose up --abort-on-container-exit +- docker-compose up #--abort-on-container-exit TODO: remove this once itests can be made not to flap see issue #21 - rm itest.env cache: directories: @@ -31,4 +31,4 @@ deploy: script: sh $TRAVIS_BUILD_DIR/.travis.deploy.sh skip_cleanup: true on: - branch: master \ No newline at end of file + branch: master