Generate temporary repos to test against (hail-is#97)

* wip

* a little closer

* bugs & posix

* fixes

* setup endpoints

* give notice of setup-endpoints.sh`

* wait more than one minute

when there are no k8s nodes available it takes about 60 seconds to start one

* bad rebase, should have deleted this line

* pretty sure ssl is not supported in ci2

* stop using ci-test repo and use better temp dir

* hide token in test-locally

* be professional

* address a few comments

* hide TOKEN

README.md

@@ -62,6 +62,16 @@ And if you want to cleanly restart fresh:
 HAIL_CI_REMOTE_PORT=3001 make update-conda-env restart-all-proxies run
 ```
 
+Setting up a New Repo
+---
+
+Make sure you have a `hail-ci-build.sh` and a `hail-ci-build-image` (containing
+the name of a docker image that the CI has privileges to access).
+
+Execute `setup-endpoints.sh` to configure GitHub to send notifications of GitHub
+behavior to the CI system.
+
+
 Testing
 ---
 

ci/build_state.py

@@ -1,8 +1,7 @@
 from batch.client import Job
 from batch_helper import try_to_cancel_job
 from ci_logging import log
-from constants import CONTEXT
-from environment import batch_client
+from environment import batch_client, CONTEXT
 import json
 import re
 

ci/constants.py

@@ -1,7 +1,6 @@
 GITHUB_API_URL = 'https://api.github.com/'
 GITHUB_CLONE_URL = 'https://github.com/'
 VERSION = '0-1'
-CONTEXT = 'hail-ci-0-1'
 BUILD_JOB_PREFIX = 'hail-ci-0-2-1'
 BUILD_JOB_TYPE = BUILD_JOB_PREFIX + '-build'
 DEPLOY_JOB_TYPE = BUILD_JOB_PREFIX + '-deploy'

ci/environment.py

@@ -3,6 +3,9 @@
 from batch.client import BatchClient
 import json
 import os
+import uuid
+
+INSTANCE_ID = uuid.uuid4().hex
 
 SELF_HOSTNAME = os.environ.get('SELF_HOSTNAME',
                                'http://set_the_SELF_HOSTNAME/')
@@ -11,6 +14,8 @@
 REFRESH_INTERVAL_IN_SECONDS = \
     int(os.environ.get('REFRESH_INTERVAL_IN_SECONDS', 60))
 
+CONTEXT = f'hail-ci-0-1-{INSTANCE_ID}'
+
 try:
     WATCHED_TARGETS = [
         (FQRef.from_short_str(ref), deployable)
@@ -47,5 +52,8 @@
 log.info(f'SELF_HOSTNAME {SELF_HOSTNAME}')
 log.info(f'REFRESH_INTERVAL_IN_SECONDS {REFRESH_INTERVAL_IN_SECONDS}')
 log.info(f'WATCHED_TARGETS {[(ref.short_str(), deployable) for (ref, deployable) in WATCHED_TARGETS]}')
+log.info(f'INSTANCE_ID = {INSTANCE_ID}')
+log.info(f'CONTEXT = {CONTEXT}')
+
 
 batch_client = BatchClient(url=BATCH_SERVER_URL)

ci/github.py

@@ -46,5 +46,4 @@ def overall_review_state(reviews):
             break
         elif (state == 'APPROVED'):
             total_state = 'approved'
-
     return {'state': total_state, 'reviews': latest_state_by_login}

ci/pr.py

@@ -4,8 +4,8 @@
     Failure, Mergeable, Unknown, NoImage, Building, Buildable, Merged, \
     build_state_from_json
 from ci_logging import log
-from constants import CONTEXT, BUILD_JOB_TYPE, VERSION, GCS_BUCKET, SHA_LENGTH
-from environment import PR_BUILD_SCRIPT, SELF_HOSTNAME, batch_client
+from constants import BUILD_JOB_TYPE, VERSION, GCS_BUCKET, SHA_LENGTH
+from environment import PR_BUILD_SCRIPT, SELF_HOSTNAME, batch_client, CONTEXT
 from git_state import FQSHA, FQRef
 from github import latest_sha_for_ref
 from http_helper import get_repo, post_repo, BadStatus

ci/prs.py

@@ -154,13 +154,12 @@ def build_next(self, target):
                     x for x in self.for_target(target) if x.is_pending_build()
                 ]
                 to_build = all_pending_prs
-        log.info(f'next to build for {target.short_str()}: {[str(x) for x in to_build]}')
+        log.info(f'next to build for {target.short_str()}: {[x.short_str() for x in to_build]}')
         for pr in to_build:
             self._set(pr.source.ref, pr.target.ref, pr.build_it())
 
     _deploy_secrets = {
         Repo('hail-is', 'hail'): f'ci-deploy-{VERSION}--hail-is-hail-service-account-key',
-        Repo('hail-is', 'ci-test'): f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key',
         Repo('Nealelab', 'cloudtools'): f'ci-deploy-{VERSION}--nealelab-cloudtools',
         Repo('hail-is', 'batch'): 'gcr-push-service-account-key'
     }
@@ -202,7 +201,11 @@ def try_deploy(self, target_ref):
                     'name': 'docker-sock-volume'
                 }
             }]
-            deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None)
+            if target_ref.repo.owner == "hail-ci-test":
+                # special case for test repos
+                deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key'
+            else:
+                deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None)
             if deploy_secret:
                 volumes.append({
                     'volume': {

setup-endpoints.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+set +x
+
+FULLY_QUALIFIED_REPO_NAME=$1
+TOKEN=$2
+CALLBACK_URL=$3
+
+if [[ -z "${FULLY_QUALIFIED_REPO_NAME}" ]] || [[ -z "${TOKEN}" ]] || [[ -z "${CALLBACK_URL}" ]]
+then
+    echo "USAGE: ./setup-endpoints.sh FULLY_QUALIFIED_REPO_NAME GITHUB_OAUTH_TOKEN CALLBACK_URL"
+    echo "    e.g.: ./setup-endpoints.sh hail-ci-test/foo abcdef http://ci2.hail.is"
+    exit 1
+fi
+
+for ENDPOINT in push pull_request pull_request_review
+do
+    echo "creating endpoint ${ENDPOINT}"
+    curl -XPOST \
+         https://api.github.com/repos/${FULLY_QUALIFIED_REPO_NAME}/hooks \
+         -H "Authorization: token ${TOKEN}" \
+         -d '{ "name": "web"
+             , "config": {
+                 "url": "'${CALLBACK_URL}/${ENDPOINT}'"
+               , "content_type": "json"
+               }
+             , "events": ["'${ENDPOINT}'"]
+             }'
+done
+

test-locally.sh

@@ -1,16 +1,62 @@
 #!/bin/bash
 set -ex
 
-function cleanup {
+export REPO_NAME=ci-test-$(LC_CTYPE=C LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)
+export WATCHED_TARGETS='[["hail-ci-test/'${REPO_NAME}':master", true]]'
+
+set +x
+TOKEN=$(cat github-tokens/user1)
+set -x
+
+cleanup() {
+    set - INT TERM
+    set +e
     kill $(cat ci.pid)
     rm -rf ci.pid
+    set +x
+    curl -XDELETE \
+         https://api.github.com/repos/hail-ci-test/${REPO_NAME} \
+         -H "Authorization: token ${TOKEN}"
+    set -x
 }
 trap cleanup EXIT
 
-export WATCHED_TARGETS='[["hail-is/ci-test:master", true]]'
+trap "exit 24" INT TERM
+
+# create the temp repo
+set +x
+curl -XPOST \
+     https://api.github.com/orgs/hail-ci-test/repos \
+     -H "Authorization: token ${TOKEN}" \
+     -d "{ \"name\" : \"${REPO_NAME}\" }"
+set -x
 
+# start CI system
 source activate hail-ci
 python ci/ci.py & echo $! > ci.pid
-sleep 5
-PYTHONPATH=$PYTHONPATH:${PWD}/ci pytest -vv test/test-ci.py
+sleep 10
+
+# upload files to temp repo
+# https://unix.stackexchange.com/questions/30091/fix-or-alternative-for-mktemp-in-os-x
+REPO_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+cp test-repo/* ${REPO_DIR}
+pushd ${REPO_DIR}
+git init
+git config user.email 'ci-automated-tests@broadinstitute.org'
+git config user.name 'ci-automated-tests'
+set +x
+git remote add origin \
+    https://${TOKEN}@github.com/hail-ci-test/${REPO_NAME}.git
+set -x
+git add *
+git commit -m 'inital commit'
+git push origin master:master
+popd
+
+# setup webhooks for temp repo
+set +x
+./setup-endpoints.sh hail-ci-test/${REPO_NAME} ${TOKEN} ${SELF_HOSTNAME}
+set -x
 
+export PYTHONPATH=$PYTHONPATH:${PWD}/ci
+pytest -vv test/test-ci.py

test-repo/hail-ci-build-image

@@ -0,0 +1 @@
+google/cloud-sdk:alpine

test-repo/hail-ci-build.sh

@@ -0,0 +1,15 @@
+uname -a
+mkdir -p artifacts/foo/bar
+cat <<EOF > artifacts/index.html
+<html>
+<body>
+<h1>Hello World!</h1>
+<a href='foo/bar/thing'>foo/bar/thing!</a>
+</body>
+</html>
+EOF
+cat <<EOF > artifacts/foo/bar/thing
+this is a thing
+thing thing thing
+thing a ding
+EOF

test-repo/hail-ci-deploy.sh

@@ -0,0 +1,10 @@
+set +ex
+
+gcloud auth activate-service-account \
+  --key-file=/secrets/ci-deploy-0-1--hail-is-ci-test.json
+
+DEPLOY_SHA=$(git rev-parse HEAD)
+gsutil ls gs://hail-ci-test/${DEPLOY_SHA} || (
+    git show HEAD > foo
+    gsutil cp foo gs://hail-ci-test/$(git rev-parse HEAD)
+)