From 73e8f390a86ce4880e38c72659ed079e2d7371df Mon Sep 17 00:00:00 2001 From: Kerry Chu <21276247+kerrychu@users.noreply.github.com> Date: Wed, 24 Jan 2024 08:59:31 +1000 Subject: [PATCH] Realiability/pytest (#6) --- .github/workflows/code-quality.yml | 62 +++++++++++++++++++ .github/workflows/pylint.yml | 23 ------- requirements.txt | 3 + gpu_jobs.py => src/gpu_jobs.py | 4 +- __init__.py => src/hooks/__init__.py | 0 {hooks => src/hooks}/slack.py | 0 my_jobs.py => src/my_jobs.py | 6 +- quota.py => src/quota.py | 4 +- {hooks => src/utils}/__init__.py | 0 {utils => src/utils}/data_serialization.py | 0 {utils => src/utils}/subprocess_operations.py | 8 +-- {utils => tests}/__init__.py | 0 tests/test_gpu_jobs.py | 0 tests/test_hooks/__init__.py | 0 tests/test_my_jobs.py | 0 tests/test_quota.py | 0 tests/test_utils/__init__.py | 0 .../test_utils/test_subprocess_operations.py | 17 +++++ 18 files changed, 93 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/pylint.yml rename gpu_jobs.py => src/gpu_jobs.py (90%) rename __init__.py => src/hooks/__init__.py (100%) rename {hooks => src/hooks}/slack.py (100%) rename my_jobs.py => src/my_jobs.py (95%) rename quota.py => src/quota.py (88%) rename {hooks => src/utils}/__init__.py (100%) rename {utils => src/utils}/data_serialization.py (100%) rename {utils => src/utils}/subprocess_operations.py (91%) rename {utils => tests}/__init__.py (100%) create mode 100644 tests/test_gpu_jobs.py create mode 100644 tests/test_hooks/__init__.py create mode 100644 tests/test_my_jobs.py create mode 100644 tests/test_quota.py create mode 100644 tests/test_utils/__init__.py create mode 100644 tests/test_utils/test_subprocess_operations.py diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..51a952c --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,62 @@ +name: Code Quality + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + linting: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + + - name: Run Pylint + run: | + python -m pylint $(git ls-files '*.py') + + testing: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Pytest + run: | + python -m pytest tests -vv + + diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 0f0e51e..0000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Pylint - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') diff --git a/requirements.txt b/requirements.txt index ab279be..5ffd540 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ exceptiongroup==1.2.0 executing==2.0.1 idna==3.6 importlib-metadata==7.0.1 +iniconfig==2.0.0 ipython==8.18.1 isort==5.13.2 jaraco.classes==3.3.0 @@ -32,12 +33,14 @@ pathspec==0.12.1 pexpect==4.9.0 pkginfo==1.9.6 platformdirs==4.1.0 +pluggy==1.3.0 prompt-toolkit==3.0.43 ptyprocess==0.7.0 pure-eval==0.2.2 Pygments==2.17.2 pylint==3.0.3 pyproject_hooks==1.0.0 +pytest==7.4.4 python-dateutil==2.8.2 python-dotenv==1.0.0 pytz==2023.3.post1 diff --git a/gpu_jobs.py b/src/gpu_jobs.py similarity index 90% rename from gpu_jobs.py rename to src/gpu_jobs.py index 97d7737..271d523 100644 --- a/gpu_jobs.py +++ b/src/gpu_jobs.py @@ -1,11 +1,11 @@ import os from dotenv import load_dotenv -from utils.subprocess_operations import ( +from src.utils.subprocess_operations import ( get_piped_stdout, stdout_to_gpu_records, job_records_to_slack_message, ) -from hooks.slack import send_slack_message +from src.hooks.slack import send_slack_message load_dotenv() SLACK_WEBHOOK = os.getenv("SLACK_GPU_JOBS_WEBHOOK") diff --git a/__init__.py b/src/hooks/__init__.py similarity index 100% rename from __init__.py rename to src/hooks/__init__.py diff --git a/hooks/slack.py b/src/hooks/slack.py similarity index 100% rename from hooks/slack.py rename to src/hooks/slack.py diff --git a/my_jobs.py b/src/my_jobs.py similarity index 95% rename from my_jobs.py rename to src/my_jobs.py index 417df4b..ad0792c 100644 --- a/my_jobs.py +++ b/src/my_jobs.py @@ -2,12 +2,12 @@ from typing import Optional from dotenv import load_dotenv -from hooks.slack import send_slack_message -from utils.data_serialization import ( +from src.hooks.slack import send_slack_message +from src.utils.data_serialization import ( read_json_as_job_records, write_job_records_to_json, ) -from utils.subprocess_operations import ( +from src.utils.subprocess_operations import ( stdout_to_job_records, JOB_RECORDS, job_records_to_slack_message, diff --git a/quota.py b/src/quota.py similarity index 88% rename from quota.py rename to src/quota.py index 93d2e48..66b8b7f 100644 --- a/quota.py +++ b/src/quota.py @@ -1,7 +1,7 @@ import os from dotenv import load_dotenv -from utils.subprocess_operations import get_piped_stdout, stdout_to_quota_records -from hooks.slack import send_slack_message +from src.utils.subprocess_operations import get_piped_stdout, stdout_to_quota_records +from src.hooks.slack import send_slack_message load_dotenv() diff --git a/hooks/__init__.py b/src/utils/__init__.py similarity index 100% rename from hooks/__init__.py rename to src/utils/__init__.py diff --git a/utils/data_serialization.py b/src/utils/data_serialization.py similarity index 100% rename from utils/data_serialization.py rename to src/utils/data_serialization.py diff --git a/utils/subprocess_operations.py b/src/utils/subprocess_operations.py similarity index 91% rename from utils/subprocess_operations.py rename to src/utils/subprocess_operations.py index a7046ed..7048bea 100644 --- a/utils/subprocess_operations.py +++ b/src/utils/subprocess_operations.py @@ -40,15 +40,15 @@ def get_piped_stdout(main_command: str, piped_command: str) -> Optional[str]: ) -def strip_spaces(l: list[str]) -> list[str]: +def strip_empty_string(l: list[str]) -> list[str]: return [x for x in l if x != ""] def stdout_to_job_records(s: str) -> JOB_RECORDS: s = s.strip() s_list = s.split("\n") - headers = strip_spaces(s_list[0].split(" ")) - data = [strip_spaces(element.split(" ")) for element in s_list[1:]] + headers = strip_empty_string(s_list[0].split(" ")) + data = [strip_empty_string(element.split(" ")) for element in s_list[1:]] job_records = [] for l in data: d = {} @@ -82,7 +82,7 @@ def stdout_to_gpu_records(s: str) -> JOB_RECORDS: s = s.strip() s_list = s.split("\n") headers = ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES"] - data = [strip_spaces(element.split(" ")) for element in s_list] + data = [strip_empty_string(element.split(" ")) for element in s_list] job_records = [] for l in data: d = {} diff --git a/utils/__init__.py b/tests/__init__.py similarity index 100% rename from utils/__init__.py rename to tests/__init__.py diff --git a/tests/test_gpu_jobs.py b/tests/test_gpu_jobs.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_hooks/__init__.py b/tests/test_hooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_my_jobs.py b/tests/test_my_jobs.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_quota.py b/tests/test_quota.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_utils/test_subprocess_operations.py b/tests/test_utils/test_subprocess_operations.py new file mode 100644 index 0000000..460faf2 --- /dev/null +++ b/tests/test_utils/test_subprocess_operations.py @@ -0,0 +1,17 @@ +from src.utils.subprocess_operations import ( + strip_empty_string, + job_records_to_slack_message, +) + + +def test_strip_empty_string(): + assert strip_empty_string(["hello", "world", ""]) == ["hello", "world"] + + +def test_job_records_to_slack_message(): + header = "header" + job_records = [{"key1": "value1", "key2": "value2", "key3": "value3"}] + assert ( + job_records_to_slack_message(header, job_records) + == "header\n\t⦿ key1: value1\n\t⦿ key2: value2\n\t⦿ key3: value3\n" + )