Github workflows for building environments

.github/README.md

@@ -0,0 +1 @@
+See the readme in `testing/development` for more information about these workflows and how to develop for them. 

.github/workflows/cluster.yml

@@ -2,10 +2,259 @@ name: Cluster Run
 
 on:
   workflow_dispatch:
+  # pull_request:
+  #  branches:
+  #    - '*'
+
 jobs:
-  manual-job:
-    runs-on: ubuntu-latest
+  build-and-test-cluster:
+    runs-on: self-hosted
+    env:
+      UNIQUE_ID: 
+      IP_ADDRESS: 
+      LS1_IP:
+      BRANCH_NAME:
+      elastic:
 
     steps:
-      - name: Greet
-        run: echo "This is a manually triggered workflow."
+      - name: Checkout repository
+        uses: actions/checkout@v4.1.1
+
+      - name: Setup environment variables
+        run: |
+          PUBLIC_IP=$(curl -s https://api.ipify.org)
+          echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV
+          echo "UNIQUE_ID=$(openssl rand -hex 3 | head -c 6)" >> $GITHUB_ENV
+
+      - name: Get branch name
+        shell: bash
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(uname -s)-$(uname -m)" \
+          -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Set the environment for docker-compose 
+        run: |
+          cd testing/development
+          # Get the UID and GID of the current user
+          echo "HOST_UID=$(id -u)" > .env
+          echo "HOST_GID=$(id -g)" >> .env
+
+      # - name: Run Docker Compose Build to fix a user id issue in a prebuilt container
+      #   run: |
+      #     cd testing/development
+      #     docker compose -p ${{ env.UNIQUE_ID }} build --no-cache 
+
+      - name: Run Docker Compose
+        run: docker compose -p ${{ env.UNIQUE_ID }} -f testing/development/docker-compose.yml up -d
+
+      - name: List docker containers to wait for them to start
+        run: |
+          docker ps 
+
+      - name: List files in home directory
+        run: |
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme bash -c "pwd && ls -la"
+
+      - name:  Check powershell environment 
+        run: |
+          set +e  
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme pwsh -Command "& { 
+            cd /home/admin.ackbar/LME; \
+            ls -la; \
+            exit \$LASTEXITCODE;
+          }"
+          EXIT_CODE=$?
+          echo "Exit code: $EXIT_CODE"
+          set -e  
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            exit $EXIT_CODE
+          fi
+
+      - name:  Build the cluster 
+        run: |
+          set +e  
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme pwsh -Command "& { 
+            cd /home/admin.ackbar/LME/testing; \
+            \$env:AZURE_CLIENT_ID='${{ secrets.AZURE_CLIENT_ID }}'; \
+            \$env:AZURE_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_CLIENT_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_TENANT='${{ secrets.AZURE_TENANT }}'; \
+            \$env:UNIQUE_ID='${{ env.UNIQUE_ID }}'; \
+            \$env:RESOURCE_GROUP='LME-pipe-${{ env.UNIQUE_ID }}'; \
+            \$env:IP_ADDRESS='${{ env.IP_ADDRESS }}'; \
+            ./development/build_cluster.ps1 -IPAddress \$env:IP_ADDRESS; \
+            exit \$LASTEXITCODE;
+          }"
+          EXIT_CODE=$?
+          echo "Exit code: $EXIT_CODE"
+          set -e  
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            exit $EXIT_CODE
+          fi
+          cd .. 
+          . configure/lib/functions.sh
+          extract_ls1_ip 'LME-pipe-${{ env.UNIQUE_ID }}.cluster.output.log'
+          echo "LS1_IP=$LS1_IP" >> $GITHUB_ENV
+
+      - name:  Install lme on cluster 
+        run: |
+          set +e  
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme pwsh -Command "& { 
+            cd /home/admin.ackbar/LME/testing; \
+            \$env:AZURE_CLIENT_ID='${{ secrets.AZURE_CLIENT_ID }}'; \
+            \$env:AZURE_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_CLIENT_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_TENANT='${{ secrets.AZURE_TENANT }}'; \
+            \$env:UNIQUE_ID='${{ env.UNIQUE_ID }}'; \
+            \$env:RESOURCE_GROUP='LME-pipe-${{ env.UNIQUE_ID }}'; \
+            ./development/install_lme.ps1 -b '${{ env.BRANCH_NAME }}'; \
+            exit \$LASTEXITCODE;
+          }"
+          EXIT_CODE=$?
+          echo "Exit code: $EXIT_CODE"
+          set -e  
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            exit $EXIT_CODE
+          fi
+
+      - name:  Set the environment passwords for other steps 
+        run: |
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme bash -c "
+            cd /home/admin.ackbar/LME/testing \
+            && . configure/lib/functions.sh \
+            && extract_credentials 'LME-pipe-${{ env.UNIQUE_ID }}.password.txt' \
+            && write_credentials_to_file '${{ env.UNIQUE_ID }}.github_env.sh' \
+            "
+            . ../${{ env.UNIQUE_ID }}.github_env.sh
+            rm ../${{ env.UNIQUE_ID }}.github_env.sh
+            echo "elastic=$elastic" >> $GITHUB_ENV
+            echo "kibana=$kibana" >> $GITHUB_ENV
+            echo "logstash_system=$logstash_system" >> $GITHUB_ENV
+            echo "logstash_writer=$logstash_writer" >> $GITHUB_ENV
+            echo "dashboard_update=$dashboard_update" >> $GITHUB_ENV
+
+      - name:  Check that the environment variables are set 
+        run: |
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme bash -c "
+            if [ -z \"${{ env.elastic }}\" ]; then
+              echo 'Error: env.elastic variable is not set' >&2
+              exit 1
+            else
+              echo 'Elastic password is set'
+            fi
+          "
+
+      # - name: Run a command on the domain controller 
+      #   run: |
+      #     set +e  
+      #     cd testing/development
+      #     docker compose -p ${{ env.UNIQUE_ID }} exec -T lme pwsh -Command "& { 
+      #       cd /home/admin.ackbar/LME/testing; \
+      #       \$env:AZURE_CLIENT_ID='${{ secrets.AZURE_CLIENT_ID }}'; \
+      #       \$env:AZURE_SECRET='${{ secrets.AZURE_SECRET }}'; \
+      #       \$env:AZURE_CLIENT_SECRET='${{ secrets.AZURE_SECRET }}'; \
+      #       \$env:AZURE_TENANT='${{ secrets.AZURE_TENANT }}'; \
+      #       \$env:UNIQUE_ID='${{ env.UNIQUE_ID }}'; \
+      #       \$env:RESOURCE_GROUP='LME-pipe-${{ env.UNIQUE_ID }}'; \
+      #       az login --service-principal -u \$env:AZURE_CLIENT_ID -p \$env:AZURE_SECRET --tenant \$env:AZURE_TENANT; \
+      #       az vm run-command invoke  \
+      #         --command-id RunPowerShellScript  \
+      #         --name DC1  \
+      #         --resource-group \$env:RESOURCE_GROUP \
+      #         --scripts 'ls C:\'; \
+      #       exit \$LASTEXITCODE;
+      #     }"
+      #     EXIT_CODE=$?
+      #     echo "Exit code: $EXIT_CODE"
+      #     set -e  
+      #     if [ "$EXIT_CODE" -ne 0 ]; then
+      #       exit $EXIT_CODE
+      #     fi
+
+      - name: Run a command on the linux machine 
+        run: |
+          set +e  
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme pwsh -Command "& { 
+            cd /home/admin.ackbar/LME/testing; \
+            \$env:AZURE_CLIENT_ID='${{ secrets.AZURE_CLIENT_ID }}'; \
+            \$env:AZURE_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_CLIENT_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_TENANT='${{ secrets.AZURE_TENANT }}'; \
+            \$env:UNIQUE_ID='${{ env.UNIQUE_ID }}'; \
+            \$env:RESOURCE_GROUP='LME-pipe-${{ env.UNIQUE_ID }}'; \
+            az login --service-principal -u \$env:AZURE_CLIENT_ID -p \$env:AZURE_SECRET --tenant \$env:AZURE_TENANT; \
+            az vm run-command invoke  \
+              --command-id RunShellScript  \
+              --name LS1  \
+              --resource-group \$env:RESOURCE_GROUP \
+              --scripts 'ls -lan'; \
+            exit \$LASTEXITCODE;
+          }"
+          EXIT_CODE=$?
+          echo "Exit code: $EXIT_CODE"
+          set -e  
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            exit $EXIT_CODE
+          fi
+
+      # This only passes when you do a full install
+      - name: Run api tests in container
+        run: |
+          set +e
+          cd testing/development
+          docker-compose -p ${{ env.UNIQUE_ID }} exec -T -u admin.ackbar lme bash -c " cd testing/tests \
+          && echo export elastic=${{ env.elastic }} > .env \
+          && echo export ES_HOST=${{ env.LS1_IP }} >> .env \
+          && cat .env \
+          && python3 -m venv /home/admin.ackbar/venv_test \
+          && . /home/admin.ackbar/venv_test/bin/activate \
+          && pip install -r requirements.txt \
+          && sudo chmod ugo+w /home/admin.ackbar/LME/ -R \
+          && pytest -v api_tests/"
+
+      - name: Run selenium tests in container
+        run: |
+          set +e
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T -u admin.ackbar lme bash -c " cd testing/tests \
+          && echo export ELASTIC_PASSWORD=${{ env.elastic }} > .env \
+          && . .env \
+          && python3 -m venv /home/admin.ackbar/venv_test \
+          && . /home/admin.ackbar/venv_test/bin/activate \
+          && pip install -r requirements.txt \
+          && sudo chmod ugo+w /home/admin.ackbar/LME/ -R \
+          && python selenium_tests.py --domain ${{ env.LS1_IP }} -v"
+
+      - name: Cleanup environment 
+        if: always()
+        run: |
+          cd testing/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T lme pwsh -Command "& { 
+            cd /home/admin.ackbar/LME/testing; \
+            \$env:AZURE_CLIENT_ID='${{ secrets.AZURE_CLIENT_ID }}'; \
+            \$env:AZURE_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_CLIENT_SECRET='${{ secrets.AZURE_SECRET }}'; \
+            \$env:AZURE_TENANT='${{ secrets.AZURE_TENANT }}'; \
+            \$env:UNIQUE_ID='${{ env.UNIQUE_ID }}'; \
+            \$env:RESOURCE_GROUP='LME-pipe-${{ env.UNIQUE_ID }}'; \
+            ./development/destroy_cluster.ps1; \
+            exit \$LASTEXITCODE;
+          }"
+          docker compose -p ${{ env.UNIQUE_ID }} down
+          docker system prune --force

.github/workflows/linux_only.yml

@@ -1,11 +1,101 @@
-name: Linux Only 
+name: Linux Only
 
 on:
   workflow_dispatch:
-jobs:
-  manual-job:
-    runs-on: ubuntu-latest
+  pull_request:
+    branches:
+      - '*'
 
+jobs:
+  build-and-test-linux-only:
+    # runs-on: ubuntu-latest
+    runs-on: self-hosted
+
+    env:
+      UNIQUE_ID:
+      BRANCH_NAME:
+
     steps:
-      - name: Greet
-        run: echo "This is a manually triggered workflow."
+    - name: Checkout repository
+      uses: actions/checkout@v4.1.1
+
+    - name: Setup environment variables
+      run: |
+        echo "UNIQUE_ID=$(openssl rand -hex 3 | head -c 6)" >> $GITHUB_ENV
+
+    - name: Setup environment variables
+      run: |
+        echo "AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV
+        echo "AZURE_SECRET=${{ secrets.AZURE_SECRET }}" >> $GITHUB_ENV
+        echo "AZURE_CLIENT_SECRET=${{ secrets.AZURE_SECRET }}" >> $GITHUB_ENV
+        echo "AZURE_TENANT=${{ secrets.AZURE_TENANT }}" >> $GITHUB_ENV
+        echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV
+
+    - name: Set Branch Name
+      shell: bash
+      env:
+        EVENT_NAME: ${{ github.event_name }}
+        HEAD_REF: ${{ github.head_ref }}
+        GITHUB_REF: ${{ github.ref }}
+      run: |
+        if [ "$EVENT_NAME" == "pull_request" ]; then
+          echo "BRANCH_NAME=$HEAD_REF" >> $GITHUB_ENV
+        else
+          BRANCH_REF="${GITHUB_REF##*/}"
+          echo "BRANCH_NAME=$BRANCH_REF" >> $GITHUB_ENV
+        fi
+
+    - name: Set up Docker Compose
+      run: |
+        sudo curl -L "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(uname -s)-$(uname -m)" \
+          -o /usr/local/bin/docker-compose
+        sudo chmod +x /usr/local/bin/docker-compose
+
+    - name: Set the environment for docker-compose
+      run: |
+        cd testing/development
+        # Get the UID and GID of the current user
+        echo "HOST_UID=$(id -u)" > .env
+        echo "HOST_GID=$(id -g)" >> .env
+
+    - name: Run Docker Build
+      run: docker compose -p ${{ env.UNIQUE_ID }} -f testing/development/docker-compose.yml build lme --no-cache
+
+    - name: Run Docker Compose
+      run: docker compose -p ${{ env.UNIQUE_ID }} -f testing/development/docker-compose.yml up -d
+
+    - name: List docker containers to wait for them to start
+      run: |
+        docker ps
+
+    - name: Execute commands inside ubuntu container
+      run: |
+        cd testing/development
+        docker compose -p ${{ env.UNIQUE_ID }} exec -T ubuntu bash -c "echo 'Ubuntu container built'"
+
+    - name: Install LME in container
+      run: |
+        set -x
+        cd testing/development
+        docker compose -p ${{ env.UNIQUE_ID }} exec -T lme bash -c "./testing/development/build_docker_lme_install.sh -b ${{ env.BRANCH_NAME }} \
+          && sudo chmod go+r /opt/lme/Chapter\ 3\ Files/output.log"
+
+    - name: Run api tests in container
+      run: |
+        cd testing/development
+        docker compose -p ${{ env.UNIQUE_ID }} exec -T -u admin.ackbar lme bash -c ". testing/configure/lib/functions.sh \
+          && sudo cp /opt/lme/Chapter\ 3\ Files/output.log . \
+          && extract_credentials output.log \
+          && sudo rm output.log \
+          && sudo docker ps \
+          && . /home/admin.ackbar/venv_test/bin/activate \
+          && sudo chmod ugo+w /home/admin.ackbar/LME/ \
+          && pytest testing/tests/api_tests/linux_only/ "
+
+    - name: Cleanup Docker Compose
+      if: always()
+      run: |
+        cd testing/development
+        docker compose -p ${{ env.UNIQUE_ID }} exec -T -u root lme bash -c "rm -rf /home/admin.ackbar/LME/.pytest_cache"
+        docker compose -p ${{ env.UNIQUE_ID }} down
+        docker system prune --force