Merge pull request #14 from scratchrealm/switch-to-flake8

Switch to flake8

.flake8

@@ -0,0 +1,9 @@
+[flake8]
+extend-ignore = E124,E128,E301,E302,E305,E501,E261
+# E124: closing bracket does not match visual indentation
+# E128: continuation line under-indented for visual indent
+# E301: expected 1 blank line, found 0
+# E302: expected 2 blank lines, found 1
+# E305: expected 2 blank lines after class or function definition, found 1
+# E501: line too long (82 > 79 characters)
+# E261: at least two spaces before inline comment

.github/workflows/linter_checks.yml

@@ -0,0 +1,18 @@
+name: testing
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Linter checks
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install
+      run: cd python && pip install -e .
+    - name: Install packages needed for tests
+      run: pip install pytest pytest-cov pyright boto3 kachery_cloud flake8
+    - name: Install additional packages used by api_helpers
+      run: pip install -r requirements.txt
+    - name: Run linter checks
+      run: cd python && flake8 --config ../.flake8 && pyright

.github/workflows/testing.yml → .github/workflows/tests.yml

@@ -5,15 +5,15 @@ on: [push, pull_request]
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Test
+    name: Tests
     steps:
     - uses: actions/checkout@v1
     - name: Install
       run: cd python && pip install -e .
     - name: Install packages needed for tests
-      run: pip install pytest pytest-cov pyright boto3 kachery_cloud autopep8
+      run: pip install pytest pytest-cov boto3 kachery_cloud
     - name: Run tests and collect coverage
-      run: devel/check_formatting.sh && cd python && pyright && pytest --cov protocaas --cov-report=xml --cov-report=term tests/
+      run: cd python && pytest --cov protocaas --cov-report=xml --cov-report=term tests/
     - uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}

api/index.py

@@ -1,13 +1,13 @@
 from fastapi import FastAPI
 
-# Here's the reason that all the other Python files are in ../api_helpers
+# Here's the reason that all the other Python files are in ../python/protocaas/api_helpers
 # I was noticing very long build times (~15 minutes)...
 # Apparently, vercel treats every .py file in /api as a lambda function.
 # So it was building each and every one of them, even though index.py should be the only one.
 # See https://github.com/orgs/vercel/discussions/46
 
 import sys
-sys.path.append("..")
+sys.path.append("../python/protocaas")
 from api_helpers.routers.processor.router import router as processor_router
 from api_helpers.routers.compute_resource.router import router as compute_resource_router
 from api_helpers.routers.client.router import router as client_router

doc/for_developers.md

@@ -1,22 +1,14 @@
 ## Running the tests (vscode action)
 
 ```bash
-pip install pytest pytest-cov pyright autopep8
+pip install pytest pytest-cov pyright flake8
 ```
 
 In vscode:
 
 Command palette => Run Task => Test
 
-This will test formatting (see autopep8 below), pyright, and will use pytest to run the tests in python/tests.
-
-## Running autopep8 (vscode action)
-
-This project is configured to run a very limited auto-formatting. In vscode:
-
-Command palette => Run Task => autopep8
-
-This will only apply the rules defined in python/setup.cfg.
+This will check formatting using flake8 (see below), will check typing using pyright (see below), and will use pytest to run the tests in python/tests.
 
 ## Using pyright vscode extension
 
@@ -33,8 +25,11 @@ def test2():
 
 So you should install the pyright vscode extension (ms-pyright).
 
-## Running pylint (vscode action)
+## Using flake8 vscode extension
+
+Install the flake8 vscode extension.
+
+The rules being ignored are in .flake8
 
-Although not all these rules are enforced it may be useful to run the pylint vscode task:
+You can run `cd python && flake8 --config ../.flake8` to see the errors.
 
-Command palette => Run Task => pylint

api_helpers/clients/db.py → python/protocaas/api_helpers/clients/db.py

@@ -212,7 +212,7 @@ async def set_compute_resource_spec(compute_resource_id: str, spec: ComputeResou
         }
     })
 
-async def fetch_job(job_id: str, *, include_dandi_api_key: bool=False, include_secret_params: bool=False):
+async def fetch_job(job_id: str, *, include_dandi_api_key: bool = False, include_secret_params: bool = False):
     client = _get_mongo_client()
     jobs_collection = client['protocaas']['jobs']
     job = await jobs_collection.find_one({'jobId': job_id})

api_helpers/clients/pubsub.py → python/protocaas/api_helpers/clients/pubsub.py

@@ -19,7 +19,7 @@ async def publish_pubsub_message(*, channel: str, message: dict):
     url = f"https://ps.pndsn.com/publish/{pub_key}/{sub_key}/0/{channel}/0/{payload}?uuid={uuid}"
 
     headers = {
-    'Accept': 'application/json'
+        'Accept': 'application/json'
     }
 
     # async http get request

api_helpers/core/protocaas_types.py → python/protocaas/api_helpers/core/protocaas_types.py

@@ -15,7 +15,7 @@ class ProtocaasProject(BaseModel):
     tags: List[str]
     timestampCreated: float
     timestampModified: float
-    computeResourceId: Union[str, None]=None # it seems this needs to go at the end, otherwise it will be required by pydantic - not sure why
+    computeResourceId: Union[str, None] = None # it seems this needs to go at the end, otherwise it will be required by pydantic - not sure why
 
 class ProtocaasJobInputFile(BaseModel):
     name: str
@@ -24,26 +24,26 @@ class ProtocaasJobInputFile(BaseModel):
 
 class ProtocaasJobInputParameter(BaseModel):
     name: str
-    value: Union[Any, None]=None
-    secret: Union[bool, None]=None
+    value: Union[Any, None] = None
+    secret: Union[bool, None] = None
 
 class ProtocaasJobOutputFile(BaseModel):
     name: str
     fileName: str
-    fileId: Union[str, None]=None
+    fileId: Union[str, None] = None
 
 class ComputeResourceSpecProcessorParameter(BaseModel):
     name: str
     help: str
     type: str
-    default: Union[Any, None]=None
-    options: Union[List[str], List[int], None]=None
-    secret: bool=False
+    default: Union[Any, None] = None
+    options: Union[List[str], List[int], None] = None
+    secret: bool = False
 
 class ComputeResourceSpecProcessorInput(BaseModel):
     name: str
     help: str
-    list: bool=False
+    list: bool = False
 
 class ComputeResourceSpecProcessorOutput(BaseModel):
     name: str
@@ -78,19 +78,19 @@ class ProtocaasJob(BaseModel):
     timestampCreated: float
     computeResourceId: str
     status: str # 'pending' | 'queued' | 'starting' | 'running' | 'completed' | 'failed'
-    batchId: Union[str, None]=None
-    error: Union[str, None]=None
-    processorVersion: Union[str, None]=None
-    computeResourceNodeId: Union[str, None]=None
-    computeResourceNodeName: Union[str, None]=None
-    consoleOutputUrl: Union[str, None]=None
-    timestampQueued: Union[float, None]=None
-    timestampStarting: Union[float, None]=None
-    timestampStarted: Union[float, None]=None
-    timestampFinished: Union[float, None]=None
-    outputFileIds: Union[List[str], None]=None
+    batchId: Union[str, None] = None
+    error: Union[str, None] = None
+    processorVersion: Union[str, None] = None
+    computeResourceNodeId: Union[str, None] = None
+    computeResourceNodeName: Union[str, None] = None
+    consoleOutputUrl: Union[str, None] = None
+    timestampQueued: Union[float, None] = None
+    timestampStarting: Union[float, None] = None
+    timestampStarted: Union[float, None] = None
+    timestampFinished: Union[float, None] = None
+    outputFileIds: Union[List[str], None] = None
     processorSpec: ComputeResourceSpecProcessor
-    dandiApiKey: Union[str, None]=None
+    dandiApiKey: Union[str, None] = None
 
 class ProtocaasFile(BaseModel):
     projectId: str
@@ -101,32 +101,32 @@ class ProtocaasFile(BaseModel):
     timestampCreated: float
     content: str # e.g., 'url:https://...'
     metadata: dict
-    jobId: Union[str, None]=None # the job that produced this file
+    jobId: Union[str, None] = None # the job that produced this file
 
 class ComputeResourceAwsBatchOpts(BaseModel):
     jobQueue: str
     jobDefinition: str
 
 class ComputeResourceSlurmOpts(BaseModel):
-    partition: Union[str, None]=None
-    time: Union[str, None]=None
-    cpusPerTask: Union[int, None]=None
-    otherOpts: Union[str, None]=None
+    partition: Union[str, None] = None
+    time: Union[str, None] = None
+    cpusPerTask: Union[int, None] = None
+    otherOpts: Union[str, None] = None
 
 class ProtocaasComputeResourceApp(BaseModel):
     name: str
     specUri: str
-    executablePath: Union[str, None]=None # to be removed (once database has been cleared)
-    container: Union[str, None]=None # to be removed (once database has been cleared)
-    awsBatch: Union[ComputeResourceAwsBatchOpts, None]=None
-    slurm: Union[ComputeResourceSlurmOpts, None]=None
+    executablePath: Union[str, None] = None # to be removed (once database has been cleared)
+    container: Union[str, None] = None # to be removed (once database has been cleared)
+    awsBatch: Union[ComputeResourceAwsBatchOpts, None] = None
+    slurm: Union[ComputeResourceSlurmOpts, None] = None
 
 class ComputeResourceSpecApp(BaseModel):
     name: str
     help: str
     processors: List[ComputeResourceSpecProcessor]
-    appImage: Union[str, None]=None
-    appExecutable: Union[str, None]=None
+    appImage: Union[str, None] = None
+    appExecutable: Union[str, None] = None
 
 class ComputeResourceSpec(BaseModel):
     apps: List[ComputeResourceSpecApp]
@@ -137,7 +137,7 @@ class ProtocaasComputeResource(BaseModel):
     name: str
     timestampCreated: float
     apps: List[ProtocaasComputeResourceApp]
-    spec: Union[ComputeResourceSpec, None]=None
+    spec: Union[ComputeResourceSpec, None] = None
 
 class PubsubSubscription(BaseModel):
     pubnubSubscribeKey: str

api_helpers/routers/gui/compute_resource_routes.py → python/protocaas/api_helpers/routers/gui/compute_resource_routes.py

@@ -41,7 +41,7 @@ class AuthException(Exception):
     pass
 
 @router.get("")
-async def get_compute_resources(github_access_token: str=Header(...)):
+async def get_compute_resources(github_access_token: str = Header(...)):
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)
@@ -63,7 +63,7 @@ class SetComputeResourceAppsResponse(BaseModel):
     success: bool
 
 @router.put("/{compute_resource_id}/apps")
-async def set_compute_resource_apps(compute_resource_id, data: SetComputeResourceAppsRequest, github_access_token: str=Header(...)) -> SetComputeResourceAppsResponse:
+async def set_compute_resource_apps(compute_resource_id, data: SetComputeResourceAppsRequest, github_access_token: str = Header(...)) -> SetComputeResourceAppsResponse:
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)
@@ -95,7 +95,7 @@ class DeleteComputeResourceResponse(BaseModel):
     success: bool
 
 @router.delete("/{compute_resource_id}")
-async def delete_compute_resource(compute_resource_id, github_access_token: str=Header(...)) -> DeleteComputeResourceResponse:
+async def delete_compute_resource(compute_resource_id, github_access_token: str = Header(...)) -> DeleteComputeResourceResponse:
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)
@@ -150,7 +150,7 @@ class RegisterComputeResourceResponse(BaseModel):
     success: bool
 
 @router.post("/register")
-async def register_compute_resource(data: RegisterComputeResourceRequest, github_access_token: str=Header(...)) -> RegisterComputeResourceResponse:
+async def register_compute_resource(data: RegisterComputeResourceRequest, github_access_token: str = Header(...)) -> RegisterComputeResourceResponse:
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)
@@ -179,7 +179,7 @@ class GetJobsForComputeResourceResponse(BaseModel):
     success: bool
 
 @router.get("/{compute_resource_id}/jobs")
-async def get_jobs_for_compute_resource(compute_resource_id, github_access_token: str=Header(...)) -> GetJobsForComputeResourceResponse:
+async def get_jobs_for_compute_resource(compute_resource_id, github_access_token: str = Header(...)) -> GetJobsForComputeResourceResponse:
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)

api_helpers/routers/gui/create_job_route.py → python/protocaas/api_helpers/routers/gui/create_job_route.py

@@ -1,4 +1,4 @@
-from typing import Union, List, Any
+from typing import Union, List
 import traceback
 from pydantic import BaseModel
 from fastapi import APIRouter, HTTPException, Header
@@ -29,7 +29,7 @@ class AuthException(Exception):
     pass
 
 @router.post("/jobs")
-async def create_job_handler(data: CreateJobRequest, github_access_token: str=Header(...)) -> CreateJobResponse:
+async def create_job_handler(data: CreateJobRequest, github_access_token: str = Header(...)) -> CreateJobResponse:
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)

api_helpers/routers/gui/file_routes.py → python/protocaas/api_helpers/routers/gui/file_routes.py

@@ -55,7 +55,7 @@ class AuthException(Exception):
     pass
 
 @router.put("/projects/{project_id}/files/{file_name:path}")
-async def set_file(project_id, file_name, data: SetFileRequest, github_access_token: str=Header(...)):
+async def set_file(project_id, file_name, data: SetFileRequest, github_access_token: str = Header(...)):
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)
@@ -88,7 +88,7 @@ class DeleteFileResponse(BaseModel):
     success: bool
 
 @router.delete("/projects/{project_id}/files/{file_name:path}")
-async def delete_file(project_id, file_name, github_access_token: str=Header(...)):
+async def delete_file(project_id, file_name, github_access_token: str = Header(...)):
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)

api_helpers/routers/gui/github_auth_routes.py → python/protocaas/api_helpers/routers/gui/github_auth_routes.py

@@ -1,4 +1,3 @@
-import os
 from pydantic import BaseModel
 from fastapi import APIRouter
 import aiohttp
@@ -14,7 +13,7 @@ class GithubAuthResponse(BaseModel):
 @router.get("/github_auth/{code}")
 async def github_auth(code) -> GithubAuthResponse:
     settings = get_settings()
-    GITHUB_CLIENT_ID =settings.GITHUB_CLIENT_ID
+    GITHUB_CLIENT_ID = settings.GITHUB_CLIENT_ID
     GITHUB_CLIENT_SECRET = settings.GITHUB_CLIENT_SECRET
     if GITHUB_CLIENT_ID is None:
         raise Exception('Env var not set: VITE_GITHUB_CLIENT_ID')

api_helpers/routers/gui/job_routes.py → python/protocaas/api_helpers/routers/gui/job_routes.py

@@ -37,7 +37,7 @@ class DeleteJobResponse(BaseModel):
     success: bool
 
 @router.delete("/{job_id}")
-async def delete_job(job_id, github_access_token: str=Header(...)) -> DeleteJobResponse:
+async def delete_job(job_id, github_access_token: str = Header(...)) -> DeleteJobResponse:
     try:
         # authenticate the request
         user_id = await _authenticate_gui_request(github_access_token)