diff --git a/.github/workflows/build-chimbuko-spack-dev-ddb.yml b/.github/workflows/build-chimbuko-spack-dev-ddb.yml new file mode 100644 index 0000000..7c1cc95 --- /dev/null +++ b/.github/workflows/build-chimbuko-spack-dev-ddb.yml @@ -0,0 +1,66 @@ +name: Build and push chimbuko-spack-dev-ddb image + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + steps: + + - name: Check out this repo + uses: actions/checkout@v5 + with: + path: 'ChimbukoVisualizationII' + ref: 'actions_unstable' + + - name: Check out the PerformanceAnalysis repo + uses: actions/checkout@v5 + with: + path: 'PerformanceAnalysis' + ref: 'ckelly_develop_ddb' + repository: 'CODARcode/PerformanceAnalysis' + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + # note that the calling workflow must set `secrets: inherit` + username: '${{ github.actor }}' + password: '${{ secrets.GITHUB_TOKEN }}' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Pull chimbuko/chimbuko-spack-env docker image + shell: bash + run: | + docker pull ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-env:ubuntu18.04 + docker image tag \ + ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-env:ubuntu18.04 \ + chimbuko/chimbuko-spack-env:ubuntu18.04 + + - name: Build test environment image + shell: bash + run: | + cd PerformanceAnalysis/ + docker build \ + -f docker/ubuntu18.04/openmpi4.0.4/all_spack/Dockerfile.cv2 \ + -t chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 \ + . + + - name: Push chimbuko-spack-dev-ddb image + shell: bash + run: | + docker image tag chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 \ + ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 + docker image push ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 diff --git a/.github/workflows/build-chimbuko-spack-env.yml b/.github/workflows/build-chimbuko-spack-env.yml new file mode 100644 index 0000000..94e5f56 --- /dev/null +++ b/.github/workflows/build-chimbuko-spack-env.yml @@ -0,0 +1,58 @@ +name: Build and push chimbuko-spack-env image + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + steps: + + - name: Check out this repo + uses: actions/checkout@v5 + with: + path: 'ChimbukoVisualizationII' + ref: 'actions' + + - name: Check out the PerformanceAnalysis repo + uses: actions/checkout@v5 + with: + path: 'PerformanceAnalysis' + ref: 'ckelly_develop' + repository: 'CODARcode/PerformanceAnalysis' + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + # note that the calling workflow must set `secrets: inherit` + username: '${{ github.actor }}' + password: '${{ secrets.GITHUB_TOKEN }}' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build PerformanceAnalysis image + shell: bash + run: | + cd PerformanceAnalysis/docker/ubuntu18.04/openmpi4.0.4/all_spack + docker build \ + -f Dockerfile.spack_env \ + -t chimbuko/chimbuko-spack-env:ubuntu18.04 \ + . + + - name: Push PerformanceAnalysis image + shell: bash + run: | + docker image tag chimbuko/chimbuko-spack-env:ubuntu18.04 \ + ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-env:ubuntu18.04 + docker image push ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-env:ubuntu18.04 diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml new file mode 100644 index 0000000..000c5e8 --- /dev/null +++ b/.github/workflows/end-to-end.yml @@ -0,0 +1,51 @@ +name: Chimbuko Visualization II End-to-end Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + end_to_end_tests: + runs-on: ubuntu-latest + steps: + + # Check out ChimubkoVisualizationII (CV2) repo. + - name: Check out the CV2 repo + uses: actions/checkout@v4 + with: + ref: 'actions_unstable' + + - name: Check out the PerformanceAnalysis repo + uses: actions/checkout@v4 + with: + path: 'PerformanceAnalysis' + ref: 'ckelly_develop' + repository: 'CODARcode/PerformanceAnalysis' + + - name: Pull chimbuko/chimbuko-spack-dev-ddb docker image + shell: bash + run: | + docker pull ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 + docker image tag \ + ghcr.io/${{ github.repository_owner }}/chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 \ + chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 + + - name: Run tests in container + shell: bash + run: | + docker run --rm \ + --privileged \ + --entrypoint /bin/bash \ + --env NGROK_TOKEN=${{ secrets.NGROK_TOKEN }} \ + --env RUN_DIR=$(pwd) \ + --publish 5002:5002 \ + --volume $(pwd):$(pwd) \ + chimbuko/chimbuko-spack-dev-ddb:ubuntu18.04 \ + -c "$(pwd)/docker/e2e-test.sh" + + diff --git a/docker/e2e-test.sh b/docker/e2e-test.sh new file mode 100755 index 0000000..5adfbf6 --- /dev/null +++ b/docker/e2e-test.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +set -eux pipefail + +env + +ls -halF $RUN_DIR + +pushd $RUN_DIR +git config --global --add safe.directory $RUN_DIR +git checkout actions_unstable + +mkdir -p data/grid +ln -s /Downloads/repeat_1rank data/grid/ + +# Install Redis + +./webserver/run-redis.sh & + +pushd /opt/spack-environment/ && \ + source /spack/spack/share/spack/setup-env.sh && \ + spack env activate . +popd + +python -m pip install -r requirements.large.txt +python -m pip install -r requirements.txt + +REDIS_CLI="./redis-stable/src/redis-cli" +function hold_for_redis () { + while [[ ! -x $REDIS_CLI ]]; do + echo wait for redis-cli: hold on 10 s + sleep 10 + done + OUTPUT=""; + while [[ "$OUTPUT" != "PONG" ]]; do + OUTPUT=`$REDIS_CLI PING`; + echo wait for redis: hold on 1 sec + sleep 1 + done +} + +set +e + +hold_for_redis + +set -e + +# Fix configuration +CONF="./redis-stable/redis.conf" +sed -i '422s/^/# /' $CONF +sed -i '2017s/^/# /' $CONF +sed -i '2018s/^/# /' $CONF +sed -i "s/^protected-mode yes/protected-mode no/" $CONF +sed -i "s/^bind 127.0.0.1/bind 0.0.0.0/" $CONF +sed -i "s/^daemonize no/daemonize yes/" $CONF +sed -i "s|^dir ./|dir $RUN_DIR/|" $CONF +sed -i "s|^pidfile /var/run/redis_6379.pid|pidfile $RUN_DIR/redis.pid|" $CONF +killall -r '.*redis.*' + +popd + +# Install ngrok +curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc \ + | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null \ + && echo "deb https://ngrok-agent.s3.amazonaws.com bookworm main" \ + | tee /etc/apt/sources.list.d/ngrok.list \ + && apt update \ + && apt install -y ngrok +ngrok config add-authtoken $NGROK_TOKEN + +RUN_SCRIPT="./run.sh" +sed -i "s/^cycles=200/cycles=100/" $RUN_SCRIPT +sed -i "s/mpirun/mpirun --oversubscribe/" $RUN_SCRIPT + +ngrok http 5002 & + +CHIMBUKO_VIZ_ROOT=$RUN_DIR $RUN_SCRIPT + +set +ex diff --git a/server/server.py b/server/server.py index f12744f..58bc1fd 100644 --- a/server/server.py +++ b/server/server.py @@ -80,6 +80,7 @@ def get_inspect(): if 'pdb' in globals(): del pdb print("Shutting down provdb!") + return jsonify({}) @main.route('/')