Merge pull request #39 from omnivector-solutions/tucker/JOB-86--add-a…

…uth0-token-support-to-jobbergate-cli

Tucker/job 86  add auth0 token support to jobbergate cli

docker-compose.yml

@@ -7,6 +7,8 @@ services:
       dockerfile: Dockerfile-dev
       args:
         - PYPI_PASSWORD=${PYPI_PASSWORD}
+    networks:
+      - jobbergate-net
     volumes:
       - ./jobbergate-api/jobbergateapi2:/app/jobbergateapi2
     command: uvicorn jobbergateapi2.main:app --reload --workers 1 --host 0.0.0.0 --port 8000
@@ -30,6 +32,8 @@ services:
   nextgen-db:
     image: postgres
     restart: always
+    networks:
+      - jobbergate-net
     volumes:
       - postgres_data:/var/lib/postgresql/data/
     environment:
@@ -41,6 +45,8 @@ services:
 
   minio:
     image: minio/minio
+    networks:
+      - jobbergate-net
     volumes:
       - minio_data:/data
     ports:
@@ -53,10 +59,16 @@ services:
 
   adminer:
     image: adminer
+    networks:
+      - jobbergate-net
     restart: always
     ports:
       - 8080:8080
 
 volumes:
     postgres_data:
     minio_data:
+
+networks:
+  jobbergate-net:
+    driver: bridge

jobbergate-api/jobbergateapi2/main.py

@@ -34,7 +34,7 @@ async def health_check():
 
 
 app = FastAPI()
-
+app.mount("/jobbergate", subapp)
 
 @app.on_event("startup")
 async def init_database():
@@ -53,6 +53,3 @@ async def disconnect_database():
     """
     logger.debug("Disconnecting database")
     await storage.database.disconnect()
-
-
-app.mount("/jobbergate", subapp)

jobbergate-cli/jobbergate_cli/client.py

@@ -7,6 +7,8 @@
 # import locally so we can patch them when tracing
 from requests import Session, delete, get, post, put
 
+from jobbergate_cli import constants
+
 
 (
     get,
@@ -19,9 +21,6 @@
 urllib3_logger = logging.getLogger("requests.packages.urllib3")
 
 
-DEFAULT_MAX_BYTES_DEBUG = 1000
-
-
 def debug_body_printer(max_bytes):
     """
     -> function which can print the response body, for debugging, as a hook
@@ -34,7 +33,7 @@ def debug_body_print(response, *a, **kw):
     return debug_body_print
 
 
-def debug_requests_on(max_bytes=DEFAULT_MAX_BYTES_DEBUG):
+def debug_requests_on(max_bytes=constants.DEFAULT_MAX_BYTES_DEBUG):
     """Switches on logging of the requests module.
 
     Response body will be printed as well, up to max_bytes

jobbergate-cli/jobbergate_cli/config.py

@@ -0,0 +1,70 @@
+"""
+Configuration file, sets all the necessary environment variables.
+Can load configuration from a dotenv file if supplied.
+"""
+from pathlib import Path
+from typing import Optional
+
+from pydantic import AnyHttpUrl, BaseSettings, Field, root_validator
+import urllib3
+
+from jobbergate_cli import constants
+
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+class Settings(BaseSettings):
+
+    JOBBERGATE_CACHE_DIR: Path = Field(Path.home() / ".local/share/jobbergate")
+    JOBBERGATE_API_ENDPOINT: AnyHttpUrl = Field("https://jobbergateapi2-staging.omnivector.solutions")
+
+    # enable http tracing
+    JOBBERGATE_DEBUG: bool = Field(False)
+
+    SBATCH_PATH: Path = Field("/usr/bin/sbatch")
+    SENTRY_DSN: Optional[str]
+
+    # Settings for log uploads
+    JOBBERGATE_AWS_ACCESS_KEY_ID: Optional[str]
+    JOBBERGATE_AWS_SECRET_ACCESS_KEY: Optional[str]
+    JOBBERGATE_S3_LOG_BUCKET: str = Field("jobbergate-cli-logs")
+
+    # Computed values. Listed as Optional, but will *always* be set (or overridden) based on other values
+    JOBBERGATE_APPLICATION_MODULE_PATH: Optional[Path]
+    JOBBERGATE_APPLICATION_CONFIG_PATH: Optional[Path]
+    JOBBERGATE_LOG_PATH: Optional[Path]
+    JOBBERGATE_USER_TOKEN_DIR: Optional[Path]
+    JOBBERGATE_API_JWT_PATH: Optional[Path]
+
+    @root_validator
+    def compute_extra_settings(cls, values):
+        cache_dir = values["JOBBERGATE_CACHE_DIR"]
+        cache_dir.mkdir(exist_ok=True, parents=True)
+
+        values["JOBBERGATE_APPLICATION_MODULE_PATH"] = (
+            cache_dir / constants.JOBBERGATE_APPLICATION_MODULE_FILE_NAME
+        )
+        values["JOBBERGATE_APPLICATION_CONFIG_PATH"] = (
+            cache_dir / constants.JOBBERGATE_APPLICATION_CONFIG_FILE_NAME
+        )
+
+        log_dir = cache_dir / "logs"
+        log_dir.mkdir(exist_ok=True, parents=True)
+        values["JOBBERGATE_LOG_PATH"] = log_dir / "jobbergate-cli.log"
+
+        token_dir = cache_dir / "token"
+        token_dir.mkdir(exist_ok=True, parents=True)
+        values["JOBBERGATE_USER_TOKEN_DIR"] = token_dir
+        values["JOBBERGATE_API_JWT_PATH"] = token_dir / "jobbergate.token"
+
+        return values
+
+    class Config:
+        if constants.JOBBERGATE_DEFAULT_DOTENV_PATH.is_file():
+            env_file = constants.JOBBERGATE_DEFAULT_DOTENV_PATH
+        else:
+            env_file = ".env"
+
+
+settings = Settings()

jobbergate-cli/jobbergate_cli/constants.py

@@ -0,0 +1,35 @@
+"""
+Constants used throughout the tool
+"""
+from pathlib import Path
+
+
+JOBBERGATE_APPLICATION_CONFIG = {
+    "application_name": "",
+    "application_description": "",
+    "application_file": "",
+    "application_config": "",
+}
+
+JOBBERGATE_JOB_SCRIPT_CONFIG = {
+    "job_script_name": "",
+    "job_script_description": "TEST_DESC",
+    "job_script_owner": "",
+    "application": "",
+}
+
+JOBBERGATE_JOB_SUBMISSION_CONFIG = {
+    "job_submission_name": "",
+    "job_submission_description": "TEST_DESC",
+    "job_submission_owner": "",
+    "job_script": "",
+}
+
+JOBBERGATE_DEFAULT_DOTENV_PATH = Path("/etc/default/jobbergate-cli")
+JOBBERGATE_APPLICATION_MODULE_FILE_NAME = "jobbergate.py"
+JOBBERGATE_APPLICATION_CONFIG_FILE_NAME = "jobbergate.yaml"
+TAR_NAME = "jobbergate.tar.gz"
+
+ARMADA_CLAIMS_KEY = "https://www.armada-hpc.com"
+
+DEFAULT_MAX_BYTES_DEBUG = 1000