From 6276fca4dc184110e53be1a00dd0d2990d299915 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Fri, 21 Jun 2024 17:15:29 +0100
Subject: [PATCH 01/14] mcb-update-actions - Update matplotlib to latest
 version

---
 .github/scripts/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt
index 5360f418..ca1bfe69 100644
--- a/.github/scripts/requirements.txt
+++ b/.github/scripts/requirements.txt
@@ -2,7 +2,7 @@ argparse
 typing
 requests==2.26.0
 pandas==1.5.3
-matplotlib==3.4.3
+matplotlib==3.9.0
 jsonpath_ng
 jsonref
 jsonschema==3.2.0

From 83b4741dc5cf7956be7b15eacb4cc253c5872b27 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 12:04:48 +0100
Subject: [PATCH 02/14] mcb-update-actions - Correct typo in documentation

---
 .github/scripts/update_version_manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/update_version_manifest.py b/.github/scripts/update_version_manifest.py
index 2b9d34b3..f26a3eb1 100644
--- a/.github/scripts/update_version_manifest.py
+++ b/.github/scripts/update_version_manifest.py
@@ -14,7 +14,7 @@
 from utils.json_validation import validate_json
 
 # Script used to update the version manifest (./docs/releases) with the versions of the JSON schemas. For further details, please check:
-#   https://github.com/EbiEga/ega-metadata-schema/tree/main/docs/biovalidator_benchmarks
+#   https://github.com/EbiEga/ega-metadata-schema/tree/main/docs/releases
 
 # --------- #
 # Handling arguments

From 433f37cc4030492638179bfe63be6c724bf9c68d Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 12:29:34 +0100
Subject: [PATCH 03/14] mcb-update-actions - Give more time for server start

---
 ...json_validation_deploying_biovalidator.yml | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 4c8b6c53..ed6572e8 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -45,6 +45,7 @@ jobs:
           # See https://github.com/elixir-europe/biovalidator#installation
           # We want to test the "main" branch of the project
         run: git clone https://github.com/elixir-europe/biovalidator.git
+
       - name: Install Biovalidator
           # We want to test with the latest Node.js version, but if not, we 
           # could make use of action: https://github.com/actions/setup-node
@@ -59,11 +60,22 @@ jobs:
           #   changes. Therefore we specify the schemas at deployment level (-r)
         run: |
           schemas_dir="./schemas"
+          node biovalidator/src/biovalidator -r "$schemas_dir/*.json" > biovalidator.log 2>&1 &
           
-          node biovalidator/src/biovalidator -r "$schemas_dir/*.json" &
-          
-          # We stop for a few seconds to give the server time
-          sleep 3
+          # Check if the server is up and running, and wait for a few seconds if not
+          for i in {1..10}; do
+            if curl --output /dev/null --silent --head --fail "http://localhost:3020/validate"; then
+              echo "Biovalidator server is up!"
+              break
+            fi
+            echo "Waiting ($i time(s)) for Biovalidator server to start..."
+            sleep 5
+          done
+          if ! curl --output /dev/null --silent --head --fail "http://localhost:3020/validate"; then
+            echo "Biovalidator server failed to start"
+            cat biovalidator.log
+            exit 1
+          fi
 
       - uses: actions/setup-python@v4
         with:
@@ -86,5 +98,4 @@ jobs:
           # The following URL points to the locally deployed server of
           #   Biovalidator
           url="http://localhost:3020/validate"
-
-          python3 ./.github/scripts/request_validation.py "$json_ex_dir" "$url"
\ No newline at end of file
+          python3 ./.github/scripts/request_validation.py "$json_ex_dir" "$url"

From a7c360b84f0180b7f71b30007c89ba08bd6eba67 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 12:49:55 +0100
Subject: [PATCH 04/14] mcb-update-actions - Add cache of dependencies and
 modify pip install

---
 .../workflows/check_project_version_change.yml    | 13 ++++++++++++-
 .../workflows/json_validation_against_EGA_API.yml | 13 ++++++++++++-
 .../json_validation_deploying_biovalidator.yml    | 15 ++++++++++++---
 .github/workflows/update_version_manifest.yml     | 13 ++++++++++++-
 4 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/check_project_version_change.yml b/.github/workflows/check_project_version_change.yml
index cb581fd2..31b95067 100644
--- a/.github/workflows/check_project_version_change.yml
+++ b/.github/workflows/check_project_version_change.yml
@@ -43,11 +43,22 @@ jobs:
           # This bit is needed for running the following python script
           python-version: '3.x'
 
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
+          #   the dependencies are restored from the cache instead of being downloaded and installed again.
+          path: ~/.cache/pip
+          # combines the OS type and a hash of the requirements.txt file: the cache is specific to the dependencies listed
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           pip install --upgrade pip
           requirements_f="./.github/scripts/requirements.txt"
-          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f"; fi
+          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f" --prefer-binary --verbose; fi
 
       - name: Run version modification check
         id: run-version-modification-check
diff --git a/.github/workflows/json_validation_against_EGA_API.yml b/.github/workflows/json_validation_against_EGA_API.yml
index 53e783cd..e06e7b3a 100644
--- a/.github/workflows/json_validation_against_EGA_API.yml
+++ b/.github/workflows/json_validation_against_EGA_API.yml
@@ -39,11 +39,22 @@ jobs:
           # This bit is needed for running the following python script
           python-version: '3.x'
 
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
+          #   the dependencies are restored from the cache instead of being downloaded and installed again.
+          path: ~/.cache/pip
+          # combines the OS type and a hash of the requirements.txt file: the cache is specific to the dependencies listed
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           pip install --upgrade pip
           requirements_f="./.github/scripts/requirements.txt"
-          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f"; fi
+          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f" --prefer-binary --verbose; fi
 
       - name: Validate JSON examples
           # Validate all JSON documents against their corresponding schemas
diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index ed6572e8..6df4079b 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -27,7 +27,6 @@ name: |
   (json_validation_deploying_biovalidator.yml)
 
 on:
-  # Executes on any commit to a PR to the "main" branch
   pull_request:
     branches: [main]
 
@@ -38,7 +37,6 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
-          # Full git history
           fetch-depth: 0
 
       - name: Clone Biovalidator
@@ -83,11 +81,22 @@ jobs:
           # This bit is needed for running the following python script
           python-version: '3.x'
 
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
+          #   the dependencies are restored from the cache instead of being downloaded and installed again.
+          path: ~/.cache/pip
+          # combines the OS type and a hash of the requirements.txt file: the cache is specific to the dependencies listed
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           pip install --upgrade pip
           requirements_f="./.github/scripts/requirements.txt"
-          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f"; fi
+          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f" --prefer-binary --verbose; fi
 
       - name: Validate JSON examples
           # Validate all JSON documents against their corresponding schemas
diff --git a/.github/workflows/update_version_manifest.yml b/.github/workflows/update_version_manifest.yml
index 0482662d..ad1bee0d 100644
--- a/.github/workflows/update_version_manifest.yml
+++ b/.github/workflows/update_version_manifest.yml
@@ -40,11 +40,22 @@ jobs:
           # This bit is needed for running the following python script
           python-version: '3.x'
 
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
+          #   the dependencies are restored from the cache instead of being downloaded and installed again.
+          path: ~/.cache/pip
+          # combines the OS type and a hash of the requirements.txt file: the cache is specific to the dependencies listed
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           pip install --upgrade pip
           requirements_f="./.github/scripts/requirements.txt"
-          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f"; fi
+          if [ -f "$requirements_f" ]; then pip install -r "$requirements_f" --prefer-binary --verbose; fi
 
       - name: Update version manifest
         run: |

From 9b08f7bb4d9d16fe4eb4438680f286ce28a7c285 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 13:15:52 +0100
Subject: [PATCH 05/14] mcb-update-actions - Test Biovalidator server

---
 .../workflows/json_validation_deploying_biovalidator.yml   | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 6df4079b..6a36ed36 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -103,6 +103,13 @@ jobs:
           # Check:
           #   github.com/EbiEga/ega-metadata-schema/tree/main/.github/scripts
         run: |
+          #!
+          if curl --output /dev/null --silent --head --fail "http://localhost:3020/validate"; then
+            echo "Biovalidator server is up!"
+          else
+            echo "Biovalidator went down"
+          fi
+          #!
           json_ex_dir="./examples/json_validation_tests"
           # The following URL points to the locally deployed server of
           #   Biovalidator

From ac83ed18f9129cd0005527809a1f2120f95b6ca8 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 14:05:57 +0100
Subject: [PATCH 06/14] mcb-update-actions - Increase verbosity of error
 function

---
 .github/scripts/utils/json_validation.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/utils/json_validation.py b/.github/scripts/utils/json_validation.py
index 5b79c6c0..d832d2fd 100644
--- a/.github/scripts/utils/json_validation.py
+++ b/.github/scripts/utils/json_validation.py
@@ -49,11 +49,15 @@ def get_errors_response(
     if not response.status_code == requests.codes.ok:
         error_message = (
             f"The POST response was not successful: instead of {requests.codes.ok},"
-            f" the status code was '{response.status_code}' when validating file '{filename}'"
+            f" the status code was '{response.status_code}' when validating file '{filename}'.\n"
+            f"Response content: {response.content.decode('utf-8')}"
         )
         return error_message
 
-    val_response_list = response.json()
+    try:
+        val_response_list = response.json()
+    except ValueError:
+        return f"Invalid JSON response when validating file '{filename}'. Response content: {response.content.decode('utf-8')}"
 
     # If the list is empty "[]", the validation found no errors
     if not len(val_response_list) == 0:

From fdb9de50993acb276fda6e90a0d62640f00cf68f Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 14:06:17 +0100
Subject: [PATCH 07/14] mcb-update-actions - Test new cache v4

---
 .github/workflows/json_validation_deploying_biovalidator.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 6a36ed36..909e3f6e 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -82,7 +82,7 @@ jobs:
           python-version: '3.x'
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
           #   the dependencies are restored from the cache instead of being downloaded and installed again.
@@ -91,6 +91,8 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+          # If there is no cache for this key, we create it at the end of the run, even if there was an error mid-way
+          save-always: true
 
       - name: Install dependencies
         run: |

From e5811d2948e9cc2e74e90c8d5033946d2026b809 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 14:08:03 +0100
Subject: [PATCH 08/14] mcb-update-actions - Continue on error added

---
 .github/workflows/json_validation_deploying_biovalidator.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 909e3f6e..8e14e93b 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -101,6 +101,7 @@ jobs:
           if [ -f "$requirements_f" ]; then pip install -r "$requirements_f" --prefer-binary --verbose; fi
 
       - name: Validate JSON examples
+        continue-on-error: true
           # Validate all JSON documents against their corresponding schemas
           # Check:
           #   github.com/EbiEga/ega-metadata-schema/tree/main/.github/scripts

From 96b93c1f36e0b44db64fa866a6e257f9f858d8b5 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 14:27:38 +0100
Subject: [PATCH 09/14] mcb-update-actions - Reorganise cache action

---
 .../check_project_version_change.yml          |  4 +-
 .../json_validation_against_EGA_API.yml       |  4 +-
 ...json_validation_deploying_biovalidator.yml | 45 ++++++++-----------
 .github/workflows/update_version_manifest.yml |  4 +-
 4 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/check_project_version_change.yml b/.github/workflows/check_project_version_change.yml
index 31b95067..ca27e93f 100644
--- a/.github/workflows/check_project_version_change.yml
+++ b/.github/workflows/check_project_version_change.yml
@@ -44,7 +44,7 @@ jobs:
           python-version: '3.x'
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
           #   the dependencies are restored from the cache instead of being downloaded and installed again.
@@ -53,6 +53,8 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+          # If there is no cache for this key, we create it at the end of the run, even if there was an error mid-way
+          save-always: true
 
       - name: Install dependencies
         run: |
diff --git a/.github/workflows/json_validation_against_EGA_API.yml b/.github/workflows/json_validation_against_EGA_API.yml
index e06e7b3a..30efc17f 100644
--- a/.github/workflows/json_validation_against_EGA_API.yml
+++ b/.github/workflows/json_validation_against_EGA_API.yml
@@ -40,7 +40,7 @@ jobs:
           python-version: '3.x'
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
           #   the dependencies are restored from the cache instead of being downloaded and installed again.
@@ -49,6 +49,8 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+          # If there is no cache for this key, we create it at the end of the run, even if there was an error mid-way
+          save-always: true
 
       - name: Install dependencies
         run: |
diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 8e14e93b..956bcbf3 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -39,6 +39,25 @@ jobs:
         with:
           fetch-depth: 0
 
+      - uses: actions/setup-python@v4
+        with:
+          # We'll use the latest version of python from major 3 release
+          # This bit is needed for running the following python script
+          python-version: '3.x'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
+          #   the dependencies are restored from the cache instead of being downloaded and installed again.
+          path: ~/.cache/pip
+          # combines the OS type and a hash of the requirements.txt file: the cache is specific to the dependencies listed
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+          # If there is no cache for this key, we create it at the end of the run, even if there was an error mid-way
+          save-always: true
+
       - name: Clone Biovalidator
           # See https://github.com/elixir-europe/biovalidator#installation
           # We want to test the "main" branch of the project
@@ -75,25 +94,6 @@ jobs:
             exit 1
           fi
 
-      - uses: actions/setup-python@v4
-        with:
-          # We'll use the latest version of python from major 3 release
-          # This bit is needed for running the following python script
-          python-version: '3.x'
-
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
-          #   the dependencies are restored from the cache instead of being downloaded and installed again.
-          path: ~/.cache/pip
-          # combines the OS type and a hash of the requirements.txt file: the cache is specific to the dependencies listed
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-          # If there is no cache for this key, we create it at the end of the run, even if there was an error mid-way
-          save-always: true
-
       - name: Install dependencies
         run: |
           pip install --upgrade pip
@@ -106,13 +106,6 @@ jobs:
           # Check:
           #   github.com/EbiEga/ega-metadata-schema/tree/main/.github/scripts
         run: |
-          #!
-          if curl --output /dev/null --silent --head --fail "http://localhost:3020/validate"; then
-            echo "Biovalidator server is up!"
-          else
-            echo "Biovalidator went down"
-          fi
-          #!
           json_ex_dir="./examples/json_validation_tests"
           # The following URL points to the locally deployed server of
           #   Biovalidator
diff --git a/.github/workflows/update_version_manifest.yml b/.github/workflows/update_version_manifest.yml
index ad1bee0d..f56ab51e 100644
--- a/.github/workflows/update_version_manifest.yml
+++ b/.github/workflows/update_version_manifest.yml
@@ -41,7 +41,7 @@ jobs:
           python-version: '3.x'
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           # On subsequent runs, if the cache key matches (i.e., operating system and the hash of the requirements.txt file),
           #   the dependencies are restored from the cache instead of being downloaded and installed again.
@@ -50,6 +50,8 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+          # If there is no cache for this key, we create it at the end of the run, even if there was an error mid-way
+          save-always: true
 
       - name: Install dependencies
         run: |

From e84577fe52b171217e073ed988ce68cd032da74d Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 14:31:09 +0100
Subject: [PATCH 10/14] mcb-update-actions - Test biovalidator log

---
 .github/workflows/json_validation_deploying_biovalidator.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 956bcbf3..aab53ba1 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -111,3 +111,6 @@ jobs:
           #   Biovalidator
           url="http://localhost:3020/validate"
           python3 ./.github/scripts/request_validation.py "$json_ex_dir" "$url"
+          #!
+          cat biovalidator.log
+          #!

From 0022059fe291ff141c018c0d03d6e4c4a6252108 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 14:47:42 +0100
Subject: [PATCH 11/14] mcb-update-actions - Remove continue on errors

---
 .github/workflows/json_validation_deploying_biovalidator.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index aab53ba1..773a8c72 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -101,7 +101,6 @@ jobs:
           if [ -f "$requirements_f" ]; then pip install -r "$requirements_f" --prefer-binary --verbose; fi
 
       - name: Validate JSON examples
-        continue-on-error: true
           # Validate all JSON documents against their corresponding schemas
           # Check:
           #   github.com/EbiEga/ega-metadata-schema/tree/main/.github/scripts

From 0c2edb8e830abc31202c8b7ed486d5fe5611ed37 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 18:08:48 +0100
Subject: [PATCH 12/14] mcb-update-actions - Test missing schemas with prints

---
 .github/scripts/request_validation.py                      | 4 ++++
 .../workflows/json_validation_deploying_biovalidator.yml   | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/.github/scripts/request_validation.py b/.github/scripts/request_validation.py
index 7817de02..703d54e1 100644
--- a/.github/scripts/request_validation.py
+++ b/.github/scripts/request_validation.py
@@ -25,6 +25,10 @@
     if not file.path.endswith(extension):
         continue
 
+    #!
+    print(f"----Validation attempt for file '{file}'")
+    #!
+
     request = request_validation(
         data_filepath=file,
         validator_url=curl_URL
diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 773a8c72..3dd7cce8 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -76,6 +76,12 @@ jobs:
           # We want to specify the JSON schemas and documents with the new 
           #   changes. Therefore we specify the schemas at deployment level (-r)
         run: |
+          #!
+          echo "--- Listing files:"
+          pwd
+          ll
+          ll ./schemas
+          #!
           schemas_dir="./schemas"
           node biovalidator/src/biovalidator -r "$schemas_dir/*.json" > biovalidator.log 2>&1 &
           
@@ -111,5 +117,6 @@ jobs:
           url="http://localhost:3020/validate"
           python3 ./.github/scripts/request_validation.py "$json_ex_dir" "$url"
           #!
+          echo "Biovalidator log:\n"
           cat biovalidator.log
           #!

From 3af4c3e89ca1d942c68cdf014e62843a07cec930 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 18:10:09 +0100
Subject: [PATCH 13/14] mcb-update-actions - Solve ll to ls

---
 .github/workflows/json_validation_deploying_biovalidator.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index 3dd7cce8..c81115af 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -79,8 +79,8 @@ jobs:
           #!
           echo "--- Listing files:"
           pwd
-          ll
-          ll ./schemas
+          ls
+          ls ./schemas
           #!
           schemas_dir="./schemas"
           node biovalidator/src/biovalidator -r "$schemas_dir/*.json" > biovalidator.log 2>&1 &

From 41ea3d32cdc09633e3260d34f11c26c80941f443 Mon Sep 17 00:00:00 2001
From: M-casado <mcasado@ebi.ac.uk>
Date: Mon, 24 Jun 2024 18:35:11 +0100
Subject: [PATCH 14/14] mcb-update-actions - Improve error handling

---
 .github/scripts/request_validation.py         | 34 +++++++++++--------
 .github/scripts/utils/json_validation.py      |  4 +--
 ...json_validation_deploying_biovalidator.yml | 10 ------
 3 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/.github/scripts/request_validation.py b/.github/scripts/request_validation.py
index 703d54e1..30b54f99 100644
--- a/.github/scripts/request_validation.py
+++ b/.github/scripts/request_validation.py
@@ -2,33 +2,29 @@
 import os
 import sys
 
-from utils.json_validation import request_validation, \
-    get_errors_response
+from utils.json_validation import request_validation, get_errors_response
 
-# --------- #
 # Handling arguments
-# --------- #
 dirname = sys.argv[1]
 curl_URL = sys.argv[2]
 
-# - #
 # Hardcoded values
-# - #
 extension = ".json"
 
-# --------- #
 # Running validations
-# --------- #
 error_dict = {}
-# We iterate over the JSON documents to validate
+validated_files = []
+total_files = 0
+files_with_errors = 0
+
+# Iterate over the JSON documents to validate
 for file in os.scandir(dirname):
     if not file.path.endswith(extension):
         continue
 
-    #!
-    print(f"----Validation attempt for file '{file}'")
-    #!
-
+    total_files += 1
+    validated_files.append(file.name)
+    
     request = request_validation(
         data_filepath=file,
         validator_url=curl_URL
@@ -40,7 +36,17 @@
     )
     if val_error:
         error_dict[file.name] = val_error
+        files_with_errors += 1
 
 # If the dictionary is not empty, fail the script returning it
+summary = {
+    "n_total_files": total_files,
+    "n_files_with_errors": files_with_errors,
+    "error_files": error_dict,
+    "all_files": validated_files
+}
+
 if error_dict:
-    sys.exit(json.dumps(error_dict, sort_keys=True, indent=4))
+    sys.exit(json.dumps(summary, sort_keys=True, indent=4))
+else:
+    print(json.dumps(summary, sort_keys=True, indent=4))
diff --git a/.github/scripts/utils/json_validation.py b/.github/scripts/utils/json_validation.py
index d832d2fd..ced52da7 100644
--- a/.github/scripts/utils/json_validation.py
+++ b/.github/scripts/utils/json_validation.py
@@ -49,8 +49,8 @@ def get_errors_response(
     if not response.status_code == requests.codes.ok:
         error_message = (
             f"The POST response was not successful: instead of {requests.codes.ok},"
-            f" the status code was '{response.status_code}' when validating file '{filename}'.\n"
-            f"Response content: {response.content.decode('utf-8')}"
+            f" the status code was '{response.status_code}' when validating file '{filename}'."
+            f" Response content: {response.content.decode('utf-8')}"
         )
         return error_message
 
diff --git a/.github/workflows/json_validation_deploying_biovalidator.yml b/.github/workflows/json_validation_deploying_biovalidator.yml
index c81115af..ba9e396a 100644
--- a/.github/workflows/json_validation_deploying_biovalidator.yml
+++ b/.github/workflows/json_validation_deploying_biovalidator.yml
@@ -76,12 +76,6 @@ jobs:
           # We want to specify the JSON schemas and documents with the new 
           #   changes. Therefore we specify the schemas at deployment level (-r)
         run: |
-          #!
-          echo "--- Listing files:"
-          pwd
-          ls
-          ls ./schemas
-          #!
           schemas_dir="./schemas"
           node biovalidator/src/biovalidator -r "$schemas_dir/*.json" > biovalidator.log 2>&1 &
           
@@ -116,7 +110,3 @@ jobs:
           #   Biovalidator
           url="http://localhost:3020/validate"
           python3 ./.github/scripts/request_validation.py "$json_ex_dir" "$url"
-          #!
-          echo "Biovalidator log:\n"
-          cat biovalidator.log
-          #!