Abort on container exit for docker-compose

.travis.yml

@@ -19,7 +19,7 @@ script:
 - docker build -t salesforce/dockerfile-image-update .
 - echo git_api_token=${ITEST_GH_TOKEN} > `pwd`/itest.env
 - export user_itest_secrets_file_secret=`pwd`/itest.env
-- docker-compose up
+- docker-compose up #--abort-on-container-exit TODO: remove this once itests can be made not to flap see issue #21
 - rm itest.env
 cache:
   directories:
@@ -31,4 +31,4 @@ deploy:
   script: sh $TRAVIS_BUILD_DIR/.travis.deploy.sh
   skip_cleanup: true
   on:
-    branch: master
+    branch: master

README.md

@@ -150,18 +150,31 @@ java -jar dockerfile-image-update-1.0-SNAPSHOT.jar <COMMAND> <PARAMETERS>
 ```
 
 ### Creating a new feature
-Under dockerfile-image-update/src/main/java/com/salesforce/dva/dockerfileimageupdate/subcommands/impl, create a new class `YOUR_FEATURE.java`. Make sure it implements `ExecutableWithNamespace` and has the `SubCommand` annotation with a `help`, `requiredParams`, and `optionalParams`. Then, under the `execute` method, code what you want this tool to do.
+Under [dockerfile-image-update/src/main/java/com/salesforce/dva/dockerfileimageupdate/subcommands/impl](https://github.com/salesforce/dockerfile-image-update/tree/master/dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/subcommands/impl), 
+create a new class `YOUR_FEATURE.java`. Make sure it implements `ExecutableWithNamespace` and has the `SubCommand` 
+annotation with a `help`, `requiredParams`, and `optionalParams`. Then, under the `execute` method, code what you want this tool to do.
 
 ### Running unit tests
 Run unit tests by running `mvn test`. 
 
 ### Running integration tests
 Before you run the integration tests (locally):
- * Make sure that you have access to the github orgs specified in TestCommon.ORGS. You likely will need to change it to three
-   orgs where you have permissions to create repositories. 
- * Make sure you have `git_api_url=https://api.github.com` in `/dockerfile-image-update-itest/itest.env`.
- * Make sure you have a secret file which contains the `git_api_token`. This needs access to CRUD repositories and
-   github statuses. 
- * Export the following environment variable: `export user_itest_secrets_file_secret=/path/to/secretFile`
- * Run integration tests by running `make itest-local-changes`.
-
+ 1. Make sure that you have access to the github orgs specified in [TestCommon.ORGS](https://github.com/salesforce/dockerfile-image-update/blob/master/dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java#L33). You likely will need to change it to three
+    orgs where you have permissions to create repositories. 
+ 2. Make sure you have `git_api_url=https://api.github.com` in `/dockerfile-image-update-itest/itest.env`, 
+    or set it to your internal GitHub Enterprise.
+ 3. Make sure you have a secret file which contains the `git_api_token`. 
+    The token needs to have `delete_repo, repo` permissions. 
+    You can generate your token by going to [personal access tokens](https://github.com/settings/tokens/new) in GitHub. 
+    Once you have your token place it in a file: 
+    ```
+    echo git_api_token=[copy personal access token here] > ${HOME}/.dfiu-itest-token
+    ```   
+ 4. Export the following environment variable to point to the file: 
+    ```
+    export user_itest_secrets_file_secret=${HOME}/.dfiu-itest-token
+    ```
+ 5. Run integration tests by running 
+    ```
+    make itest-local-changes
+    ```

docker-compose.yml

@@ -14,7 +14,7 @@ services:
             # This allows us to get our test results back to workspace for the pipeline to pick up
             - ./test-results:/tmp/test-results
             # Place integration test into the component so that we don't have to ship integration tests inside the component
-            - ./dockerfile-image-update-itest/target/integration-test-1.0-SNAPSHOT.jar:/tmp/integration-test.jar
+            - ./dockerfile-image-update-itest/target/dockerfile-image-update-itest-1.0-SNAPSHOT.jar:/tmp/integration-test.jar
             # If you have a secret file, you can volume mount the file to gain access to it
             # export user_itest_secrets_file_secret=/path/to/secretFile in shell
             # For CI purposes, the secret file can be added to Jenkins Credential Store with the id user_itest_secrets_file

dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/AllCommandTest.java

@@ -118,7 +118,7 @@ public void testIdempotency() throws Exception{
     @AfterClass
     public void cleanUp() throws Exception {
         addVersionStoreRepo(github, createdRepos, STORE_NAME);
-        cleanAllRepos(createdRepos);
+        cleanAllRepos(createdRepos, false);
     }
 
 }

dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ChildCommandTest.java

@@ -158,6 +158,6 @@ private void cleanBefore() throws Exception {
     @AfterClass
     public void cleanUp() throws Exception {
         addVersionStoreRepo(github, createdRepos, STORE_NAME);
-        TestCommon.cleanAllRepos(createdRepos);
+        TestCommon.cleanAllRepos(createdRepos, false);
     }
 }

dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/ParentCommandTest.java

@@ -145,6 +145,6 @@ public void testStoreUpdate() throws Exception {
     @AfterClass
     public void cleanUp() throws Exception {
         addVersionStoreRepo(github, createdRepos, STORE_NAME);
-        TestCommon.cleanAllRepos(createdRepos);
+        TestCommon.cleanAllRepos(createdRepos, false);
     }
 }

dockerfile-image-update-itest/src/main/java/com/salesforce/dockerfileimageupdate/itest/tests/TestCommon.java

@@ -12,6 +12,7 @@
 import org.kohsuke.github.GHOrganization;
 import org.kohsuke.github.GHRepository;
 import org.kohsuke.github.GitHub;
+import org.kohsuke.github.PagedIterable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,21 +58,21 @@ public static void initializeRepos(GHOrganization org, List<String> repos, Strin
         }
     }
 
-    public static void printCollectedExceptionsAndFail(List<Exception> exceptions) {
+    public static void printCollectedExceptionsAndFail(List<Exception> exceptions, boolean exitWithFail) {
         for (int i = 0; i < exceptions.size(); i++) {
             log.error("Hit exception {}/{} while cleaning up.", i+1, exceptions.size());
             log.error("", exceptions.get(i));
         }
-        if (exceptions.size() > 0) {
+        if (exitWithFail && exceptions.size() > 0) {
             throw new RuntimeException(exceptions.get(0));
         }
     }
 
-    public static void cleanAllRepos(List<GHRepository> createdRepos) throws Exception {
+    public static void cleanAllRepos(List<GHRepository> createdRepos, boolean exitWithFail) throws Exception {
         List<Exception> exceptions = new ArrayList<>();
         exceptions.addAll(checkAndDelete(createdRepos));
 
-        TestCommon.printCollectedExceptionsAndFail(exceptions);
+        TestCommon.printCollectedExceptionsAndFail(exceptions, false);
     }
 
     private static Exception checkAndDelete(GHRepository repo) {
@@ -87,11 +88,19 @@ private static Exception checkAndDelete(GHRepository repo) {
     private static List<Exception> checkAndDelete(List<GHRepository> repos) throws IOException {
         List<Exception> exceptions = new ArrayList<>();
         for (GHRepository repo : repos) {
-            for (GHRepository fork : repo.listForks()) {
-                Exception forkDeleteException = checkAndDelete(fork);
-                if (forkDeleteException != null) {
-                    exceptions.add(forkDeleteException);
+
+            PagedIterable<GHRepository> forks;
+            try {
+                forks = repo.listForks();
+                for (GHRepository fork : forks) {
+                    Exception forkDeleteException = checkAndDelete(fork);
+                    if (forkDeleteException != null) {
+                        exceptions.add(forkDeleteException);
+                    }
                 }
+            } catch (Exception getForksException) {
+                log.error("Could not get forks for repo: ", repo.getFullName());
+                exceptions.add(getForksException);
             }
             Exception repoDeleteException = checkAndDelete(repo);
             if (repoDeleteException != null) {

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/utils/DockerfileGitHubUtil.java

@@ -199,6 +199,7 @@ public void updateStore(String store, String img, String tag) throws IOException
             log.info("Image tag store cannot be null. Skipping store update...");
             return;
         }
+        log.info("Updating store: {} with image: {} tag: {}...", store, img, tag);
         GHRepository storeRepo;
         try {
             GHMyself myself = gitHubUtil.getMyself();