Merge branch 'RESTAPI-957-create-examples-dir' into 'master'

Create examples dir

See merge request firecrest/firecrest!280

.gitlab-ci.yml

@@ -35,10 +35,6 @@ build_images:
 
         if [ "$CI_COMMIT_TAG" != "" ]; then exit 0; fi
 
-        # build web client
-        /kaniko/executor --context src/tests/template_client --dockerfile ./Dockerfile \
-        --destination ${CI_REGISTRY_PREFIX}/client:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} --cleanup --single-snapshot
-
         # build tester
         /kaniko/executor --context ./ --dockerfile deploy/docker/tester/Dockerfile \
         --destination ${CI_REGISTRY_PREFIX}/tester:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} --cleanup --single-snapshot
@@ -85,7 +81,7 @@ deploy_dev:
       # link API specification inside chart
       ln -s ../../../../doc/openapi/firecrest-api.yaml openapi/files/firecrest-api.yaml
 
-      for app in config client certificator compute jaeger keycloak kong minio openapi reservations status storage tasks utilities; do
+      for app in config certificator compute jaeger keycloak kong minio openapi reservations status storage tasks utilities; do
         helm uninstall -n ${CI_NAMESPACE_DEV} --kube-token=${CI_K8S_TOKEN} "$app" || true
         helm install --wait --wait-for-jobs --timeout 240s -n ${CI_NAMESPACE_DEV} -f values-dev.yaml --kube-token=${CI_K8S_TOKEN} "$app" $app;
       done
@@ -166,7 +162,7 @@ cleanup_dev_deployment:
     - CI_REGISTRY_GROUP="$(vault kv get -field=REGISTRY_GROUP firecrest/dev)"
     - CI_REGISTRY_PREFIX="$(vault kv get -field=REPO_PREFIX firecrest/dev)"
     - >
-      for app in config certificator client compute jaeger keycloak kong minio openapi reservations status storage tasks utilities cluster tester; do
+      for app in config certificator compute jaeger keycloak kong minio openapi reservations status storage tasks utilities cluster tester; do
         helm uninstall -n ${CI_NAMESPACE_DEV} --kube-token=${CI_K8S_TOKEN} "$app" || true
       done
   only:
@@ -332,7 +328,7 @@ cleanup_dev_images:
     - CI_REGISTRY_GROUP="$(vault kv get -field=REGISTRY_GROUP firecrest/dev)"
     - CI_REGISTRY_PREFIX="$(vault kv get -field=REPO_PREFIX firecrest/dev)"
     - >
-      for image in certificator client compute reservations status storage tasks tester utilities; do
+      for image in certificator compute reservations status storage tasks tester utilities; do
 
         jfrog rt del --recursive --quiet --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$image/tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID}/"
 

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add `examples` directory for practical use cases of FirecREST.
+
 ### Changed
 
 - Environment variable names
@@ -50,6 +52,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Improved retrieval of tasks from persistence storage.
+- Upgraded `kong` gateway to [v3.6.0](https://docs.konghq.com/gateway/changelog/#3600)
+- Upgraded `cryptography`package to [v42.0.4](https://cryptography.io/en/latest/changelog/#v42-0-4)
+- Upgraded `paramiko` package to [v3.4.0](https://github.com/paramiko/paramiko/tree/3.4.0)
 
 ### Fixed
 
@@ -59,12 +64,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed SSH connection error catching
 - Fixed secured "ssh-keygen" command execution
 
-### Changed
-
-- Upgraded `kong` gateway to [v3.6.0](https://docs.konghq.com/gateway/changelog/#3600)
-- Upgraded `cryptography`package to [v42.0.4](https://cryptography.io/en/latest/changelog/#v42-0-4)
-- Upgraded `paramiko` package to [v3.4.0](https://github.com/paramiko/paramiko/tree/3.4.0)
-
 ## [1.14.0]
 
 ### Added

doc/source/index.rst

@@ -12,6 +12,7 @@ Among the most prominent services that FirecREST exposes we find authentication
 
    Overview <overview>
    Hands On <tutorial>
+   Use-case examples <usecases>
    Reference Guide <reference>
 ..   A python client setup <client>
 

doc/source/usecases.rst

@@ -0,0 +1,22 @@
+=================
+Use-case examples
+=================
+
+In the code's repository you can find examples of use cases for FirecREST where `PyFirecREST <https://github.com/eth-cscs/pyfirecrest/>`__ is used as backend for web user interfaces and workflow automation tools that interact with remote high performance computing (HPC) facilities.
+
+
+CI/CD pipeline
+^^^^^^^^^^^^^^
+
+In this `example <https://github.com/eth-cscs/firecrest/examples/CI-pipeline>`__ we create a GitHub CI/CD pipeline that will run in a HPC system through FirecREST. 
+
+Web User Interfaces
+^^^^^^^^^^^^^^^^^^^
+
+We show examples of Web Graphic User Interface applications in Python that interact with HCP services using FirecREST.
+Two workflows are used for the authentication with an identity provider: `the Client Credential workflow <https://github.com/eth-cscs/firecrest/examples/UI-client-credentials>`__ and `the Authorization Code workflow <https://github.com/eth-cscs/firecrest/examples/UI-code-flow>`__.
+
+FirecREST Operators for Airflow
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this `example <https://github.com/eth-cscs/firecrest/examples/UI-code-flow>`__ we define an Airflow graph combining small tasks which run localy in a laptop with compute-intensive tasks that must run on an HPC system. The idea is to add in Airflow the support for executing the compute-intensive tasks in a supercomputer via FirecREST. For that we are going to write custom Airflow operators that will use FirecREST to access the HPC system.

examples/CI-pipeline/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "test_CSCS"
+  test_CSCS:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        system_name: ["daint"]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+
+      - name: install python packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r use-case-CI-pipeline/ci-requirements.txt
+
+      - name: Run testing script
+        env:
+          FIRECREST_CLIENT_ID: ${{ secrets.F7T_CLIENT_ID }}
+          FIRECREST_CLIENT_SECRET: ${{ secrets.F7T_CLIENT_SECRET }}
+          FIRECREST_URL: ${{ secrets.F7T_URL }}
+          AUTH_TOKEN_URL: ${{ secrets.F7T_TOKEN_URL }}
+        run: python use-case-CI-pipeline/ci/ci_script.py --system=${{ matrix.system_name }} --branch=${{ github.ref_name }} --repo=${{ github.server_url }}/${{ github.repository }}.git --account=your_project

examples/CI-pipeline/README.md

@@ -0,0 +1,42 @@
+# CI/CD pipeline with FirecREST
+
+Creating a CI/CD pipeline that will run in Piz Daint through FirecREST.
+
+## Prerequisites
+
+- **Basic python and git knowledge**: The task involves very basic Python.
+Even if you have experience with another programming language, you'll likely find the task manageable.
+- **CSCS user account**: The pipeline is alredy configured for access to Piz Daint but it requires minimal changes to customize for a different machine.
+- **Github account**: The CI will utilize resources from your GitHub account, so make sure you have one.
+- **Basic CI/CD understanding**: Familiarity with basic concepts of Continuous Integration and Continuous Deployment processes is recommended.
+
+## Getting Started
+
+1. **Create an OIDC client, if you haven't already.**
+
+1. **Create a GitHub repository**
+    - Copy all the files of this folder in the root folder of your repo.
+    - The workflows will be disabled by default in your repo so go ahead and enable them in the "Actions" tab of your repository.
+
+1. **Inspect the code that will be tested:**
+    Take a moment to review the code in the `dist` folder. This is the code that will be tested in the CI/CD pipeline.
+
+    Right now there is nothing meaningful there, but you can add your own tests.
+
+1. **Configure CI/CD Pipeline:**
+    - Open the CI configuration file (`.github/workflows/ci.yml`) and, with the help of the comments, try to understand the different steps that are already configured. The only change is the last line of and change it to your project on the machine ` --account=your_project`.
+    - Set up the secrets that are used in the pipeline in your account. The variables are needed are `FIRECREST_CLIENT_ID`, `FIRECREST_CLIENT_SECRET`, `FIRECREST_URL` and `AUTH_TOKEN_URL`.
+
+1. **Review Results:**
+    Once you've configured the pipeline, commit your changes and push them to your GitHub repository.
+    You can follow the progress of the workflow in the "Actions" tab and ensure that the tests ran successfully, and the job was submitted to Piz Daint without issues.
+
+1. **[Optional] Apply this to your own codes:**
+    If you are familiar with another CI platform and you have code that you would like to test on Piz Daint we can help you set up the CI.
+
+## Additional Resources
+
+- [OIDC Dashboard](https://oidc-dashboard-prod.cscs.ch/)
+- [pyFirecrest documentation](https://pyfirecrest.readthedocs.io)
+- [How to set up secrets in Github Actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions)
+- [FirecREST documentation](https://firecrest.readthedocs.io)

examples/CI-pipeline/ci-requirements.txt

@@ -0,0 +1 @@
+pyfirecrest==1.5.1

examples/CI-pipeline/ci/ci_script.py

@@ -0,0 +1,84 @@
+import firecrest as fc
+import os
+import time
+import argparse
+import utilities as util
+
+
+final_slurm_states = {
+    'BOOT_FAIL',
+    'CANCELLED',
+    'COMPLETED',
+    'DEADLINE',
+    'FAILED',
+    'NODE_FAIL',
+    'OUT_OF_MEMORY',
+    'PREEMPTED',
+    'TIMEOUT',
+}
+
+# Setup variables of the client
+CLIENT_ID = os.environ.get("FIRECREST_CLIENT_ID")
+CLIENT_SECRET = os.environ.get("FIRECREST_CLIENT_SECRET")
+FIRECREST_URL = os.environ.get("FIRECREST_URL")
+AUTH_TOKEN_URL = os.environ.get("AUTH_TOKEN_URL")
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--system", default=os.environ.get('MACHINE'), help="choose system to run")
+parser.add_argument("--branch", default="main", help="branch to be tested")
+parser.add_argument("--account", default="csstaff", help="branch to be tested")
+parser.add_argument("--repo", help="repository to be tested")
+
+args = parser.parse_args()
+system_name = args.system
+ref = args.branch
+print(f"Will try to run the ci in system {system_name} on branch {ref}")
+
+keycloak = fc.ClientCredentialsAuth(CLIENT_ID, CLIENT_SECRET, AUTH_TOKEN_URL)
+client = fc.Firecrest(firecrest_url=FIRECREST_URL, authorization=keycloak)
+
+print(client.all_systems())
+script_content = util.create_batch_script(repo=args.repo, constraint='gpu', num_nodes=2, account=args.account, custom_modules=['cray-python'], branch=ref)
+with open("submission_script.sh", "w") as fp:
+    fp.write(script_content)
+
+system_state = client.system(system_name)
+print(f'Status of system is: {system_state["status"]}')
+
+if system_state["status"] == "available":
+    job = client.submit(system_name, "submission_script.sh")
+    print(f"Submitted job: {job['jobid']}")
+    poll_result = client.poll_active(system_name, jobs=[job["jobid"]])
+    while poll_result:
+        state = poll_result[0]["state"]
+        if state in final_slurm_states:
+            print(f"Job is in final state: {state}")
+            break
+
+        print(f"Status of the job is {poll_result[0]['state']}, will try again in 10 seconds")
+        time.sleep(10)
+        poll_result = client.poll_active(system_name, jobs=[job["jobid"]])
+
+    if not poll_result:
+        print("The job is no longer active")
+
+    print(f"\nSTDOUT in {job['job_file_out']}")
+    stdout_content = client.head(system_name, job['job_file_out'], lines=100)
+    print(stdout_content)
+
+    print(f"\nSTDERR in {job['job_file_err']}")
+    stderr_content = client.head(system_name, job['job_file_err'], lines=100)
+    print(stderr_content)
+
+    # Some sanity checks:
+    poll_result = client.poll(system_name, jobs=[job["jobid"]])
+    if poll_result[0]["state"] != "COMPLETED":
+        print(f"Job was not successful, status: {poll_result[0]['state']}")
+        exit(1)
+
+    util.check_output(stdout_content)
+
+else:
+    print("System {system_name} is not available")
+    exit(1)

examples/CI-pipeline/ci/utilities.py

@@ -0,0 +1,44 @@
+
+
+def create_batch_script(repo, constraint, num_nodes=1, account=None, custom_modules=None, branch="main"):
+    script =f"""#!/bin/bash -l
+#SBATCH --job-name="ci_job"
+#SBATCH --output=job.out
+#SBATCH --error=job.err
+#SBATCH --time=0:10:0
+#SBATCH --nodes={num_nodes}
+"""
+
+    if constraint:
+        script += f"#SBATCH --constraint={constraint}\n"
+
+    if account:
+        script += f"#SBATCH --account={account}\n"
+
+    script += f"""
+
+git clone -b {branch} {repo} firecrest-ci
+cd firecrest-ci/use-case-CI-pipeline
+"""
+
+    if custom_modules:
+        script += f"module load {' '.join(custom_modules)}\n"
+
+    script += """
+python -m venv testing-venv
+. ./testing-venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install -r requirements.txt
+
+python --version
+
+srun python -m timeit --setup='import dist; import numpy as np; \
+    p = np.arange(1000); q = np.arange(1000) + 2' \
+    'dist.simple_numpy_dist(p, q)'
+"""
+
+    return script
+
+
+def check_output(file_content):
+    assert "loops, best of" in file_content

examples/CI-pipeline/dist/__init__.py

@@ -0,0 +1,11 @@
+import numpy as np
+
+
+def naive_dist(p, q):
+    square_distance = 0
+    for p_i, q_i in zip(p, q):
+        square_distance += (p_i - q_i) ** 2
+    return square_distance ** 0.5
+
+def simple_numpy_dist(p, q):
+    return (np.sum((p - q) ** 2)) ** 0.5

examples/CI-pipeline/requirements.txt

@@ -0,0 +1,2 @@
+pytest
+numpy

examples/UI-client-credentials/Makefile

@@ -0,0 +1,9 @@
+current_dir := $(shell pwd)
+
+build:
+	docker image rm -f firecrest-live
+	docker build -f ./docker/Dockerfile -t firecrest-live .
+run:
+	docker run --rm -p 9090:9090 -v ${current_dir}/log:/var/log  --name firecrest-live firecrest-live
+stop:
+	docker stop firecrest-live