CU-869b5ncen: fix bugs, beginnings of a test for api backend

.github/workflows/medcat-trainer_ci.yml

@@ -3,8 +3,8 @@
 on:
   pull_request:
     paths:
-      - 'medcat-trainer/**'
-      - '.github/workflows/medcat-trainer**'
+      - "medcat-trainer/**"
+      - ".github/workflows/medcat-trainer**"
 
 defaults:
   run:
@@ -23,7 +23,7 @@
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |
@@ -45,7 +45,6 @@
           cd client
           python -m build
 
-
   test-frontend:
     runs-on: ubuntu-latest
     steps:
@@ -55,25 +54,67 @@
       - name: Set up Node.js
         uses: actions/setup-node@v6
         with:
-          node-version: '20'      
+          node-version: "20"
 
       - name: Install dependencies
         run: |
-          cd webapp/frontend      
-          npm ci                  
+          cd webapp/frontend
+          npm ci
 
       - name: Run frontend tests
         run: |
           cd webapp/frontend
           npm run test:unit
 
-
+  test-backend:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd webapp
+          pip install -r requirements.txt
+
+      - name: Download spaCy model
+        run: |
+          python -m spacy download en_core_web_md
+
+      - name: Run Django tests
+        env:
+          DB_ENGINE: sqlite3
+          SECRET_KEY: test-secret-key
+          DEBUG: 1
+        run: |
+          cd webapp/api
+          python manage.py test
+
   # Build and test webapp container
   build-and-push:
     runs-on: ubuntu-latest
-    needs: 
+    needs:
       - test-client
       - test-frontend
+      - test-backend
     steps:
       - name: Checkout main
         uses: actions/checkout@v5

.github/workflows/medcat-trainer_qa.yml

@@ -2,7 +2,7 @@ name: medcat-trainer qa-build
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
 
 permissions:
   id-token: write
@@ -19,12 +19,12 @@ jobs:
       - name: Checkout main
         uses: actions/checkout@v5
         with:
-          ref: 'main'
+          ref: "main"
 
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |
@@ -64,29 +64,72 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v6
         with:
-          node-version: '20'      
+          node-version: "20"
 
       - name: Install dependencies
         run: |
-          cd webapp/frontend      
-          npm ci                  
+          cd webapp/frontend
+          npm ci
 
       - name: Run frontend tests
         run: |
           cd webapp/frontend
           npm run test:unit
 
+  test-backend:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v5
+        with:
+          ref: "main"
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd webapp
+          pip install -r requirements.txt
+
+      - name: Download spaCy model
+        run: |
+          python -m spacy download en_core_web_md
+
+      - name: Run Django tests
+        env:
+          DB_ENGINE: sqlite3
+          SECRET_KEY: test-secret-key
+          DEBUG: 1
+        run: |
+          cd webapp/api
+          python manage.py test
+
   # Build and test webapp container
   build-and-push:
     runs-on: ubuntu-latest
-    needs: 
+    needs:
       - test-client
       - test-frontend
+      - test-backend
     steps:
       - name: Checkout main
         uses: actions/checkout@v5
         with:
-          ref: 'main'
+          ref: "main"
 
       - name: Build
         run: |

.github/workflows/medcat-trainer_release.yml

@@ -3,7 +3,7 @@ name: medcat-trainer release-build
 on:
   push:
     tags:
-      - 'medcat-trainer/v*.*.*'  
+      - "medcat-trainer/v*.*.*"
 
 permissions:
   id-token: write
@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |
@@ -57,10 +57,78 @@ jobs:
         with:
           packages_dir: medcat-trainer/client/dist
 
+  test-frontend:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v5
+        with:
+          ref: "main"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "20"
+
+      - name: Install dependencies
+        run: |
+          cd webapp/frontend
+          npm ci
+
+      - name: Run frontend tests
+        run: |
+          cd webapp/frontend
+          npm run test:unit
+
+  test-backend:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v5
+        with:
+          ref: "main"
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd webapp
+          pip install -r requirements.txt
+
+      - name: Download spaCy model
+        run: |
+          python -m spacy download en_core_web_md
+
+      - name: Run Django tests
+        env:
+          DB_ENGINE: sqlite3
+          SECRET_KEY: test-secret-key
+          DEBUG: 1
+        run: |
+          cd webapp/api
+          python manage.py test
+
   # Build and test webapp container
   build-and-push:
     runs-on: ubuntu-latest
-    needs: test-and-publish-client
+    needs:
+      - test-and-publish-client
+      - test-frontend
+      - test-backend
     steps:
       - name: Checkout main
         uses: actions/checkout@v5

medcat-trainer/webapp/api/api/data_utils.py

@@ -1,10 +1,12 @@
 import json
 import logging
+import os
 import re
 from collections import defaultdict
 from datetime import datetime
 from typing import Dict, List
 
+import pandas as pd
 from django.contrib.auth.models import User
 from django.db import transaction
 from django.db.models import Q
@@ -19,6 +21,11 @@
 logger = logging.getLogger(__name__)
 
 
+class InvalidParameterError(Exception):
+    """Exception raised when invalid parameters are provided"""
+    pass
+
+
 def dataset_from_file(dataset: Dataset):
     if '.csv' in dataset.original_file.path:
         df = pd.read_csv(dataset.original_file.path, on_bad_lines='error')
@@ -210,8 +217,10 @@ def upload_projects_export(
                     m_a = MetaAnnotation()
                     m_a.annotated_entity = a
                     # there will be at least one or more of these available.
-                    m_a = MetaTask.objects.filter(name=task_name).first()
+                    m_a.meta_task = MetaTask.objects.filter(name=task_name).first()
                     m_a.validated = meta_anno['validated']
+                    m_a.acc = meta_anno['acc']
+                    m_a.meta_task_value = MetaTaskValue.objects.filter(name=meta_anno['value']).first()
                     m_a.save()
                     # missing acc on the model
             anno_to_doc_ind = {a.start_ind: a for a in annos}