diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98967f2f5f..2d4e5019a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -398,31 +398,22 @@ mesos: - make test threads="${TEST_THREADS}" src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest # Cactus-on-Kubernetes integration (as a script and not a pytest test) -#cactus_integration: -# stage: integration -# script: -# - set -e -# - ${MAIN_PYTHON_PKG} -m virtualenv --system-site-packages venv -# - . venv/bin/activate -# - pip install -U pip wheel -# - pip install .[aws] -# - export TOIL_KUBERNETES_OWNER=toiltest -# - export TOIL_AWS_SECRET_NAME=shared-s3-credentials -# - export TOIL_KUBERNETES_HOST_PATH=/data/scratch -# - export TOIL_WORKDIR=/var/lib/toil -# - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache -# - mkdir -p ${TOIL_WORKDIR} -# - BUCKET_NAME=toil-test-$RANDOM-$RANDOM-$RANDOM -# - cd -# - git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive -# - cd cactus -# - git fetch origin -# - git checkout f5adf4013326322ae58ef1eccb8409b71d761583 -# - git submodule update --init --recursive -# # We can't use setuptools 66 on Ubuntu due to https://github.com/pypa/setuptools/issues/3772 -# - pip install --upgrade 'setuptools<66' pip -# - pip install --upgrade . -# - pip install --upgrade numpy psutil # Cactus installs an old psutil that Toil isn't compatible with. TODO: Do we really need Numpy? -# - if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi -# - toil clean aws:us-west-2:${BUCKET_NAME} -# - time cactus --setEnv SINGULARITY_DOCKER_HUB_MIRROR --batchSystem kubernetes --retryCount=3 --consCores 2 --binariesMode singularity --clean always aws:us-west-2:${BUCKET_NAME} examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk "8G" --logDebug +cactus_integration: + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH + changes: + compare_to: 'refs/heads/master' + paths: + - 'src/toil/test/cactus/test_cactus_integration.py' + stage: integration + script: + - export CACTUS_COMMIT_SHA=f5adf4013326322ae58ef1eccb8409b71d761583 + - set -e + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws] + - python setup_gitlab_docker.py # login to increase the docker.io rate limit + # This reads GITLAB_SECRET_FILE_SSH_KEYS + - python setup_gitlab_ssh.py + - chmod 400 /root/.ssh/id_rsa + - make test tests=src/toil/test/cactus/test_cactus_integration.py diff --git a/src/toil/test/cactus/test_cactus_integration.py b/src/toil/test/cactus/test_cactus_integration.py new file mode 100644 index 0000000000..079317f8c2 --- /dev/null +++ b/src/toil/test/cactus/test_cactus_integration.py @@ -0,0 +1,58 @@ +import os +import uuid + +from toil.provisioners import cluster_factory +from toil.test.provisioners.clusterTest import AbstractClusterTest + + +class CactusIntegrationTest(AbstractClusterTest): + """ + Run the Cactus Integration test on a Kubernetes AWS cluster + """ + + def __init__(self, methodName): + super().__init__(methodName=methodName) + self.clusterName = "cactus-test-" + str(uuid.uuid4()) + self.leaderNodeType = "t2.medium" + self.clusterType = "kubernetes" + + def setUp(self): + super().setUp() + self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}" + + def test_cactus_integration(self): + # Make a cluster with worker nodes + self.createClusterUtil(args=["--nodeTypes=t2.xlarge", "-w=1-3"]) + # get the leader so we know the IP address - we don't need to wait since create cluster + # already ensures the leader is running + self.cluster = cluster_factory( + provisioner="aws", zone=self.zone, clusterName=self.clusterName + ) + self.leader = self.cluster.getLeader() + + CACTUS_COMMIT_SHA = os.environ["CACTUS_COMMIT_SHA"] or "f5adf4013326322ae58ef1eccb8409b71d761583" # default cactus commit + + # command to install and run cactus on the cluster + cactus_command = ("python -m virtualenv --system-site-packages venv && " + ". venv/bin/activate && " + "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && " + "cd cactus && " + "git fetch origin && " + f"git checkout {CACTUS_COMMIT_SHA} && " + "git submodule update --init --recursive && " + "pip install --upgrade 'setuptools<66' pip && " + "pip install --upgrade . && " + "pip install --upgrade numpy psutil && " + "time cactus --batchSystem kubernetes --retryCount=3 " + f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} " + "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug") + + # run cactus + self.sshUtil( + [ + "bash", + "-c", + cactus_command + ] + ) +