diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f2dac4..3579dad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,37 @@ on: repository_dispatch: types: [notebooks-updated] jobs: + commit-checks: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: YAML Validity check + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: . + config_data: | + extends: relaxed + rules: + line-length: disable + trailing-spaces: disable + new-line-at-end-of-file: disable + document-start: disable + indentation: disable + truthy: disable + + - name: Run GitLeaks secret scanner + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run Ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./src" + args: "format --check" + build: name: Build Docker Image runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eb00360..03c7a36 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,9 +4,39 @@ on: workflow_dispatch: jobs: - test: + commit-checks: runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: YAML Validity check + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: . + config_data: | + extends: relaxed + rules: + line-length: disable + trailing-spaces: disable + new-line-at-end-of-file: disable + document-start: disable + indentation: disable + truthy: disable + + - name: Run GitLeaks secret scanner + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run Ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./src" + args: "format --check" + + test: + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/src/generated_api_template.py b/src/generated_api_template.py index 16221e1..fd7f5df 100644 --- a/src/generated_api_template.py +++ b/src/generated_api_template.py @@ -11,7 +11,7 @@ import logging import sys -#Setup logging to console +# Setup logging to console logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) console_handler = logging.StreamHandler(sys.stdout) @@ -38,7 +38,8 @@ logger.info(f"Route path set as: {route_path}") parameterHandler = NotebookParameters(notebook_path) -PostModel = create_model("post_model",**parameterHandler.dynamic_model) +PostModel = create_model("post_model", **parameterHandler.dynamic_model) + def parse_output(output): """ @@ -47,13 +48,14 @@ def parse_output(output): match output.get("output_type"): case "stream": return handle_streams(output) - + case "execute_result" | "display_data": return handle_rich_outputs(output) - + case "error": return handle_errors(output) + def execute_notebook(notebook, params=None): logger.info(f"Executing notebook: {notebook}") @@ -62,67 +64,56 @@ def execute_notebook(notebook, params=None): with tempfile.NamedTemporaryFile(suffix=".ipynb") as tmp: try: if params is None: - pm.execute_notebook( - notebook, - tmp.name, - kernel_name="global_venv" - ) + pm.execute_notebook(notebook, tmp.name, kernel_name="global_venv") else: - pm.execute_notebook( - notebook, - tmp.name, - params, - kernel_name="global_venv" - ) - except PapermillExecutionError as e: + pm.execute_notebook(notebook, tmp.name, params, kernel_name="global_venv") + except PapermillExecutionError as e: pass # Read outputs nb = nbformat.read(tmp.name, as_version=4) for cell in nb.cells: - if cell.cell_type != "code": - continue + if cell.cell_type != "code": + continue - for output in cell.get("outputs", []): - results.append(parse_output(output)) + for output in cell.get("outputs", []): + results.append(parse_output(output)) return results except ValueError as e: return f"Error= {e}" + ############## # API Routes # ############## API_PREFIX = os.environ.get("API_PREFIX", "") -app = FastAPI(docs_url=f"{API_PREFIX}/docs",openapi_url=f"{API_PREFIX}/openapi.json") +app = FastAPI(docs_url=f"{API_PREFIX}/docs", openapi_url=f"{API_PREFIX}/openapi.json") + @app.get(f"{API_PREFIX}/") async def root(): return RedirectResponse(url=f"{API_PREFIX}{route_path}") + @app.get(f"{API_PREFIX}/getParameters") async def getParameters(): return parameterHandler.readable_json -@app.get(f"{API_PREFIX}{route_path}",response_model=NotebookResponse) + +@app.get(f"{API_PREFIX}{route_path}", response_model=NotebookResponse) async def endpoint(): - output = await asyncio.to_thread( - execute_notebook, - notebook_path, - params=None - ) + output = await asyncio.to_thread(execute_notebook, notebook_path, params=None) return NotebookResponse(outputs=output) -@app.post(f"{API_PREFIX}{route_path}",response_model=NotebookResponse) -async def executeWithParams(params:PostModel): - output = await asyncio.to_thread( - execute_notebook, - notebook_path, - params= params.model_dump() - ) + +@app.post(f"{API_PREFIX}{route_path}", response_model=NotebookResponse) +async def executeWithParams(params: PostModel): + output = await asyncio.to_thread(execute_notebook, notebook_path, params=params.model_dump()) return NotebookResponse(outputs=output) + @app.get(f"/health") async def health(): - return {"status": "ok"} \ No newline at end of file + return {"status": "ok"} diff --git a/src/generated_helper.py b/src/generated_helper.py index b61c3ac..80baf43 100644 --- a/src/generated_helper.py +++ b/src/generated_helper.py @@ -1,9 +1,11 @@ -from io import StringIO -from typing import Literal, Any, Annotated, Union -from pydantic import BaseModel, Field import ast -import nbformat import re +from io import StringIO +from typing import Annotated, Any, Literal, Union + +import nbformat +from pydantic import BaseModel, Field + # Pydantic API response model class StreamOutput(BaseModel): @@ -11,50 +13,61 @@ class StreamOutput(BaseModel): stream: Literal["stdout", "stderr"] content: str + class ImageOutput(BaseModel): type: Literal["image"] mime: Literal["image/png", "image/jpeg", "image/gif"] content: str # base64 encoded + class SvgOutput(BaseModel): type: Literal["svg"] content: str # raw XML string + class JsonOutput(BaseModel): type: Literal["json"] content: Any + class HtmlOutput(BaseModel): type: Literal["html"] content: str + class MarkdownOutput(BaseModel): type: Literal["markdown"] content: str + class LatexOutput(BaseModel): type: Literal["latex"] content: str + class TextOutput(BaseModel): type: Literal["text"] content: str + class ErrorOutput(BaseModel): type: Literal["error"] ename: str evalue: str traceback: list[str] + class DataframeOutput(BaseModel): type: Literal["dataframe"] content: str json: Any + class UnknownOutput(BaseModel): type: Literal["unknown"] data: dict[str, Any] + CellOutput = Annotated[ Union[ StreamOutput, @@ -67,22 +80,24 @@ class UnknownOutput(BaseModel): TextOutput, ErrorOutput, UnknownOutput, - DataframeOutput + DataframeOutput, ], - Field(discriminator="type") + Field(discriminator="type"), ] + class NotebookResponse(BaseModel): outputs: list[CellOutput] + class NotebookParameters: - def __init__(self,notebook_path): + def __init__(self, notebook_path): self.notebook_path = notebook_path self._parameter_cell = self._extract_parameters() self._parameter_nodes = self._infer_parameter_types() self.readable_json = self._to_readable_json() self.dynamic_model = self._to_dynamic_model() - + def _extract_parameters(self): """ Extracts cells tagged as Parameters from the notebook @@ -102,14 +117,14 @@ def _infer_parameter_types(self): if self._parameter_cell is None: return {} tree = ast.parse(self._parameter_cell) - parameterNodes:list = [] + parameterNodes: list = [] for node in tree.body: # Checks for variable assignment if isinstance(node, ast.Assign): parameterNodes.append(node) return parameterNodes - + def _to_readable_json(self): """ Format parameter type to readable json @@ -125,15 +140,12 @@ def _to_readable_json(self): # Evaluate to get value try: value = ast.literal_eval(node.value) - parameters[param_name] = { - "defaultValue": value, - "type": type(value).__name__ - } + parameters[param_name] = {"defaultValue": value, "type": type(value).__name__} except ValueError: # For complex types or expressions literal_eval can't handle parameters[param_name] = {"value": "Complex Expression", "type": "unknown"} return parameters - + def _to_dynamic_model(self): """ Format parameters types to Pydantic model @@ -148,26 +160,28 @@ def _to_dynamic_model(self): # Evaluate to get value try: value = ast.literal_eval(node.value) - parameters[param_name] = (type(value),value) + parameters[param_name] = (type(value), value) except ValueError: # For complex types or expressions literal_eval can't handle - parameters[param_name] = (type(object),"") + parameters[param_name] = (type(object), "") return parameters - + + def handle_streams(output): return { "type": "stream", "stream": output.get("name", "stdout"), - "content": "".join(output.text) if isinstance(output.text, list) else output.text + "content": "".join(output.text) if isinstance(output.text, list) else output.text, } + def handle_html(content): """ Parse HTML or Dataframe """ if " str: """ Normalizes file names into lower case and replaces spaces, special characters @@ -46,19 +53,15 @@ def extract_imports(notebook_path): imports.add(top_level) return sorted(imports) -def main(base_dir = None): - GENERATED_DIR = "services" - BASE_DIR = (sys.argv[1] if len(sys.argv) > 1 else os.getcwd()) if base_dir is None else base_dir - API_TEMPLATE_PATH = f"{pathlib.Path(__file__).resolve().parent}/generated_api_template.py" - DOCKER_TEMPLATE_PATH = f"{pathlib.Path(__file__).resolve().parent}/generated_dockerfile" - API_RESPONSE_TYPE_PATH = f"{pathlib.Path(__file__).resolve().parent}/generated_helper.py" - EXCLUDED_DIRS = {"venv", ".venv", ".git", "__pycache__", ".ipynb_checkpoints","services", "devops"} - + +def main(base_dir=None): + base_dir = (sys.argv[1] if len(sys.argv) > 1 else os.getcwd()) if base_dir is None else base_dir + if not os.path.exists(GENERATED_DIR): os.makedirs(GENERATED_DIR) detectedNotebooks = [] - for root, dirs, files in os.walk(BASE_DIR): + for root, dirs, files in os.walk(base_dir): dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS] for file in files: if file.endswith(".ipynb"): @@ -82,21 +85,16 @@ def main(base_dir = None): os.makedirs(service_folder) logger.info(f"Copying {full_path} to {service_folder}") - #Copy files to generated directory + # Copy files to generated directory shutil.copy(full_path, service_folder) - shutil.copy(API_TEMPLATE_PATH,f"{service_folder}/main.py") - shutil.copy(DOCKER_TEMPLATE_PATH,f"{service_folder}/dockerfile") - shutil.copy(API_RESPONSE_TYPE_PATH,f"{service_folder}/helper.py") + shutil.copy(API_TEMPLATE_PATH, f"{service_folder}/main.py") + shutil.copy(DOCKER_TEMPLATE_PATH, f"{service_folder}/dockerfile") + shutil.copy(API_RESPONSE_TYPE_PATH, f"{service_folder}/helper.py") # Generate Requirements.txt notebook_imports = extract_imports(full_path) - requirements = "\n".join([ - "fastapi", - "papermill", - "nbformat", - "uvicorn" - ] + notebook_imports) + requirements = "\n".join(["fastapi", "papermill", "nbformat", "uvicorn"] + notebook_imports) # Generate empty requirements.txt with open(os.path.join(service_folder, "requirements.txt"), "w") as f: @@ -104,5 +102,6 @@ def main(base_dir = None): logger.info(f"Generated service for notebook: {file}\n") + if __name__ == "__main__": - main() \ No newline at end of file + main()