Dev readme pipeline update

.github/workflows/python-app.yml → .github/workflows/development.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: master
+name: development
 
 on:
   push:

.github/workflows/master.yml

@@ -0,0 +1,80 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: master
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-install:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pre-commit
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+  test:
+    needs: dependency-install
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest pytest-mock
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Test with pytest
+      run: |
+        python -m pytest
+
+  devops:
+    needs: test
+    environment: master
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pre-commit
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with pylint
+      run: |
+        pip install pylint
+        pylint --fail-under=9.7 --rcfile=.pylintrc utilities
+    - name: code coverage
+      run: |
+        mkdir -p ./coverage
+        pip install pytest-cov pytest-mock
+        python -m pytest --cov --cov-report=xml:./coverage/coverage.xml
+
+    - name: Upload coverage
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+        files: ./coverage/coverage.xml # optional

.github/workflows/staging.yml

@@ -0,0 +1,80 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: staging
+
+on:
+  push:
+    branches: [ "staging" ]
+  pull_request:
+    branches: [ "staging" ]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-install:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pre-commit
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+  test:
+    needs: dependency-install
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest pytest-mock
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Test with pytest
+      run: |
+        python -m pytest
+
+  devops:
+    needs: test
+    environment: staging
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pre-commit
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with pylint
+      run: |
+        pip install pylint
+        pylint --fail-under=9.7 --rcfile=.pylintrc utilities
+    - name: code coverage
+      run: |
+        mkdir -p ./coverage
+        pip install pytest-cov pytest-mock
+        python -m pytest --cov --cov-report=xml:./coverage/coverage.xml
+
+    - name: Upload coverage
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+        files: ./coverage/coverage.xml # optional

README.md

@@ -1,7 +1,10 @@
 ![example branch parameter](https://github.com/guorbit/utilities/actions/workflows/python-app.yml/badge.svg?branch=main) [![codecov](https://codecov.io/github/guorbit/utilities/branch/main/graph/badge.svg?token=3RVZAHQ4W2)](https://codecov.io/github/guorbit/utilities)
 
+![example branch parameter](https://github.com/guorbit/utilities/actions/workflows/python-app.yml/badge.svg?branch=development)
+
 Note before installation: None of these commands have been properly tested. Make sure you installed the package in a virtual environment.
 
+## Installation
 For installing the utilities repo as a package use the following commands in the terminal:
 Note: you need to have a working ssh key to access github from your current machine.
 you need to have wheel installed.
@@ -15,3 +18,73 @@ After build run the following command to install, the built package.
 pip install .
 
 ```
+## Preparing for development
+### Prerequisites
+It is recommended to use a virtual environment for development. To create a virtual environment, with python 3.10 as of now. To create a virtual environment run the following command in the terminal with anaconda:
+```
+conda create -n <env_name> python=3.10
+```
+Then activate the virtual environment by running the following command:
+```
+conda activate <env_name>
+```
+
+The rest of the dependencies are in the pyproject.toml file. To install those dependencies run the following command in the terminal:
+```
+pip install .[dev]
+```
+You might potentially have to build the project as well. If yes in that case run the following command:
+```
+pip wheel . --no-deps --wheel-dir=dist
+```
+Additionally, it is recommended to use certain extensions for development(in case you are using vs code). These extensions are listed as recommended in the utilities.code-workspace file. To install these extensions, open the utilities.code-workspace file in vs code and click on the install button when prompted. This also enables other checking tools like linting and type checking.
+### Project branching strategy
+The project is using a staging branching combined with feature branching strategy. The main branch is the production branch. The staging branch is the staging branch meant for advanced longer testing. The development branch is the development branch with experimental features and less reliability. The feature branches are used for new feature development. The feature branches are branched off from the development branch. The development branch is branched off from the staging branch. 
+At release the development branch is merged into the staging branch. The staging branch is merged into the main branch, if the branches pass sucsessfully the CI pipeline. - Automated build testing is still under considerationd
+
+The following diagram shows the branching strategy:
+
+```
+Main ------------------------------------------> 
+                                             ^  
+                                            /
+Staging -----------------------------------/--->
+                                        ^
+                                       /
+Development --------------------------/-------->
+                \                    /
+                 \-Feature branches-/
+
+```
+### Testing
+To run the tests run the following command in the terminal:
+```
+pytest
+```
+If the above command doesn't work, run the following command:
+```
+python -m pytest
+```
+In case you want to run a coverage test, run the following command:
+```
+python -m pytest --cov --cov-report term
+```
+
+### Linting
+To run the linting tests run the following command in the terminal (this also loads in the linting configuration stored in the project root):
+```
+pylint --rcfile=.pylintrc utilities
+```
+
+## Future plans (Ever expanding list)
+- [ ] Docker build and deployment for the package, to docker hub.
+- [ ] PyPi build and deployment for the package.
+- [ ] Additional CI pipeline for staging environment.
+- [ ] Mutation testing.
+- [ ] Tensorboard building package
+- [ ] Scalable ML data reading pipeline
+- [ ] Model conversion pipelines
+- [ ] Image manupulation tools
+- [ ] Image augmentation tools
+
+Submit a suggestion for a new feature by creating an issue, or if you already have it done by creating a pull request to development.

pyproject.toml

@@ -1,21 +1,19 @@
 [build-system]
-requires = ["setuptools", "wheel"]
+requires = ["setuptools", "wheel", "setuptools_scm"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "utilities"
 version = "0.1.0"
-authors = [
-    {name = "GU Orbit Software Team", email = "uog.orbit@gmail.com"}
-]
+authors = [{ name = "GU Orbit Software Team", email = "uog.orbit@gmail.com" }]
 requires-python = ">=3.10"
 description = "A package containing utilities for GU Orbit Software"
 readme = "README.md"
-license = {text = "MIT"}
+license = { text = "MIT" }
 classifiers = [
     "Programming Language :: Python :: 3.10",
     "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent"
+    "Operating System :: OS Independent",
 ]
 dependencies = [
     "numpy >= 1.24.0",
@@ -28,13 +26,13 @@ dependencies = [
 [tool.setuptools]
 packages = ["utilities"]
 
+
 [project.optional-dependencies]
 dev = [
     "pytest >= 7.2.2",
     "pytest-cov >= 4.0.0",
     "pytest-mock >= 3.10.0",
     "twine >= 4.0.0",
-
     "Sphinx >= 6.1.3",
     "furo >= 2022.12.7",
 ]