From 09df42c6e1de313c441932791c0b3794d701c62d Mon Sep 17 00:00:00 2001 From: Richard Dzurus Date: Wed, 25 Jun 2025 13:45:26 +0200 Subject: [PATCH 1/3] adds a fail safe when generating resources --- polyapi/generate.py | 140 +++++++++++---- polyapi/poly_schemas.py | 164 +++++++++++++----- polyapi/variables.py | 104 +++++++++-- polyapi/webhook.py | 38 ++-- pyproject.toml | 2 +- tests/test_generate.py | 375 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 720 insertions(+), 103 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index ee2836c..6d63449 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -2,6 +2,8 @@ import requests import os import shutil +import logging +import tempfile from typing import List, Optional, Tuple, cast from .auth import render_auth_function @@ -427,48 +429,124 @@ def add_function_file( function_name: str, spec: SpecificationDto, ): - # first lets add the import to the __init__ - init_the_init(full_path) + """ + Atomically add a function file to prevent partial corruption during generation failures. + + This function generates all content first, then writes files atomically using temporary files + to ensure that either the entire operation succeeds or no changes are made to the filesystem. + """ + try: + # first lets add the import to the __init__ + init_the_init(full_path) - func_str, func_type_defs = render_spec(spec) + func_str, func_type_defs = render_spec(spec) - if func_str: - # add function to init - init_path = os.path.join(full_path, "__init__.py") - with open(init_path, "a") as f: - f.write(f"\n\nfrom . import {to_func_namespace(function_name)}\n\n{func_str}") + if not func_str: + # If render_spec failed and returned empty string, don't create any files + raise Exception("Function rendering failed - empty function string returned") - # add type_defs to underscore file - file_path = os.path.join(full_path, f"{to_func_namespace(function_name)}.py") - with open(file_path, "w") as f: - f.write(func_type_defs) + # Prepare all content first before writing any files + func_namespace = to_func_namespace(function_name) + init_path = os.path.join(full_path, "__init__.py") + func_file_path = os.path.join(full_path, f"{func_namespace}.py") + + # Read current __init__.py content if it exists + init_content = "" + if os.path.exists(init_path): + with open(init_path, "r") as f: + init_content = f.read() + + # Prepare new content to append to __init__.py + new_init_content = init_content + f"\n\nfrom . import {func_namespace}\n\n{func_str}" + + # Use temporary files for atomic writes + # Write to __init__.py atomically + with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_init: + temp_init.write(new_init_content) + temp_init_path = temp_init.name + + # Write to function file atomically + with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_func: + temp_func.write(func_type_defs) + temp_func_path = temp_func.name + + # Atomic operations: move temp files to final locations + shutil.move(temp_init_path, init_path) + shutil.move(temp_func_path, func_file_path) + + except Exception as e: + # Clean up any temporary files that might have been created + try: + if 'temp_init_path' in locals() and os.path.exists(temp_init_path): + os.unlink(temp_init_path) + if 'temp_func_path' in locals() and os.path.exists(temp_func_path): + os.unlink(temp_func_path) + except: + pass # Best effort cleanup + + # Re-raise the original exception + raise e def create_function( spec: SpecificationDto ) -> None: + """ + Create a function with atomic directory and file operations. + + Tracks directory creation to enable cleanup on failure. + """ full_path = os.path.dirname(os.path.abspath(__file__)) folders = f"poly.{spec['context']}.{spec['name']}".split(".") - for idx, folder in enumerate(folders): - if idx + 1 == len(folders): - # special handling for final level - add_function_file( - full_path, - folder, - spec, - ) - else: - full_path = os.path.join(full_path, folder) - if not os.path.exists(full_path): - os.makedirs(full_path) - - # append to __init__.py file if nested folders - next = folders[idx + 1] if idx + 2 < len(folders) else "" - if next: - init_the_init(full_path) - add_import_to_init(full_path, next) + created_dirs = [] # Track directories we create for cleanup on failure + + try: + for idx, folder in enumerate(folders): + if idx + 1 == len(folders): + # special handling for final level + add_function_file( + full_path, + folder, + spec, + ) + else: + full_path = os.path.join(full_path, folder) + if not os.path.exists(full_path): + os.makedirs(full_path) + created_dirs.append(full_path) # Track for cleanup + + # append to __init__.py file if nested folders + next = folders[idx + 1] if idx + 2 < len(folders) else "" + if next: + init_the_init(full_path) + add_import_to_init(full_path, next) + + except Exception as e: + # Clean up directories we created (in reverse order) + for dir_path in reversed(created_dirs): + try: + if os.path.exists(dir_path) and not os.listdir(dir_path): # Only remove if empty + os.rmdir(dir_path) + except: + pass # Best effort cleanup + + # Re-raise the original exception + raise e def generate_functions(functions: List[SpecificationDto]) -> None: + failed_functions = [] for func in functions: - create_function(func) + try: + create_function(func) + except Exception as e: + function_path = f"{func.get('context', 'unknown')}.{func.get('name', 'unknown')}" + function_id = func.get('id', 'unknown') + failed_functions.append(f"{function_path} (id: {function_id})") + logging.warning(f"WARNING: Failed to generate function {function_path} (id: {function_id}): {str(e)}") + continue + + if failed_functions: + logging.warning(f"WARNING: {len(failed_functions)} function(s) failed to generate:") + for failed_func in failed_functions: + logging.warning(f" - {failed_func}") diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index 6b42ec7..942ae04 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -1,4 +1,7 @@ import os +import logging +import tempfile +import shutil from typing import Any, Dict, List, Tuple from polyapi.schema import wrapped_generate_schema_types @@ -21,13 +24,33 @@ def generate_schemas(specs: List[SchemaSpecDto], limit_ids: List[str] = None): + failed_schemas = [] if limit_ids: for spec in specs: if spec["id"] in limit_ids: - create_schema(spec) + try: + create_schema(spec) + except Exception as e: + schema_path = f"{spec.get('context', 'unknown')}.{spec.get('name', 'unknown')}" + schema_id = spec.get('id', 'unknown') + failed_schemas.append(f"{schema_path} (id: {schema_id})") + logging.warning(f"WARNING: Failed to generate schema {schema_path} (id: {schema_id}): {str(e)}") + continue else: for spec in specs: - create_schema(spec) + try: + create_schema(spec) + except Exception as e: + schema_path = f"{spec.get('context', 'unknown')}.{spec.get('name', 'unknown')}" + schema_id = spec.get('id', 'unknown') + failed_schemas.append(f"{schema_path} (id: {schema_id})") + logging.warning(f"WARNING: Failed to generate schema {schema_path} (id: {schema_id}): {str(e)}") + continue + + if failed_schemas: + logging.warning(f"WARNING: {len(failed_schemas)} schema(s) failed to generate:") + for failed_schema in failed_schemas: + logging.warning(f" - {failed_schema}") def add_schema_file( @@ -35,51 +58,114 @@ def add_schema_file( schema_name: str, spec: SchemaSpecDto, ): - # first lets add the import to the __init__ - init_the_init(full_path, SCHEMA_CODE_IMPORTS) - - if not spec["definition"].get("title"): - # very empty schemas like mews.Unit are possible - # add a title here to be sure they render - spec["definition"]["title"] = schema_name - - schema_defs = render_poly_schema(spec) - - if schema_defs: - # add function to init + """ + Atomically add a schema file to prevent partial corruption during generation failures. + + This function generates all content first, then writes files atomically using temporary files + to ensure that either the entire operation succeeds or no changes are made to the filesystem. + """ + try: + # first lets add the import to the __init__ + init_the_init(full_path, SCHEMA_CODE_IMPORTS) + + if not spec["definition"].get("title"): + # very empty schemas like mews.Unit are possible + # add a title here to be sure they render + spec["definition"]["title"] = schema_name + + schema_defs = render_poly_schema(spec) + + if not schema_defs: + # If render_poly_schema failed and returned empty string, don't create any files + raise Exception("Schema rendering failed - empty schema content returned") + + # Prepare all content first before writing any files + schema_namespace = to_func_namespace(schema_name) init_path = os.path.join(full_path, "__init__.py") - with open(init_path, "a") as f: - f.write(f"\n\nfrom ._{to_func_namespace(schema_name)} import {schema_name}\n__all__.append('{schema_name}')\n") - - # add type_defs to underscore file - file_path = os.path.join(full_path, f"_{to_func_namespace(schema_name)}.py") - with open(file_path, "w") as f: - f.write(schema_defs) + schema_file_path = os.path.join(full_path, f"_{schema_namespace}.py") + + # Read current __init__.py content if it exists + init_content = "" + if os.path.exists(init_path): + with open(init_path, "r") as f: + init_content = f.read() + + # Prepare new content to append to __init__.py + new_init_content = init_content + f"\n\nfrom ._{schema_namespace} import {schema_name}\n__all__.append('{schema_name}')\n" + + # Use temporary files for atomic writes + # Write to __init__.py atomically + with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_init: + temp_init.write(new_init_content) + temp_init_path = temp_init.name + + # Write to schema file atomically + with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_schema: + temp_schema.write(schema_defs) + temp_schema_path = temp_schema.name + + # Atomic operations: move temp files to final locations + shutil.move(temp_init_path, init_path) + shutil.move(temp_schema_path, schema_file_path) + + except Exception as e: + # Clean up any temporary files that might have been created + try: + if 'temp_init_path' in locals() and os.path.exists(temp_init_path): + os.unlink(temp_init_path) + if 'temp_schema_path' in locals() and os.path.exists(temp_schema_path): + os.unlink(temp_schema_path) + except: + pass # Best effort cleanup + + # Re-raise the original exception + raise e def create_schema( spec: SchemaSpecDto ) -> None: + """ + Create a schema with atomic directory and file operations. + + Tracks directory creation to enable cleanup on failure. + """ full_path = os.path.dirname(os.path.abspath(__file__)) folders = f"schemas.{spec['context']}.{spec['name']}".split(".") - for idx, folder in enumerate(folders): - if idx + 1 == len(folders): - # special handling for final level - add_schema_file( - full_path, - folder, - spec, - ) - else: - full_path = os.path.join(full_path, folder) - if not os.path.exists(full_path): - os.makedirs(full_path) - - # append to __init__.py file if nested folders - next = folders[idx + 1] if idx + 2 < len(folders) else "" - if next: - init_the_init(full_path, SCHEMA_CODE_IMPORTS) - add_import_to_init(full_path, next) + created_dirs = [] # Track directories we create for cleanup on failure + + try: + for idx, folder in enumerate(folders): + if idx + 1 == len(folders): + # special handling for final level + add_schema_file( + full_path, + folder, + spec, + ) + else: + full_path = os.path.join(full_path, folder) + if not os.path.exists(full_path): + os.makedirs(full_path) + created_dirs.append(full_path) # Track for cleanup + + # append to __init__.py file if nested folders + next = folders[idx + 1] if idx + 2 < len(folders) else "" + if next: + init_the_init(full_path, SCHEMA_CODE_IMPORTS) + add_import_to_init(full_path, next) + + except Exception as e: + # Clean up directories we created (in reverse order) + for dir_path in reversed(created_dirs): + try: + if os.path.exists(dir_path) and not os.listdir(dir_path): # Only remove if empty + os.rmdir(dir_path) + except: + pass # Best effort cleanup + + # Re-raise the original exception + raise e def add_schema_to_init(full_path: str, spec: SchemaSpecDto): diff --git a/polyapi/variables.py b/polyapi/variables.py index 673a195..76975cc 100644 --- a/polyapi/variables.py +++ b/polyapi/variables.py @@ -1,4 +1,7 @@ import os +import logging +import tempfile +import shutil from typing import List from polyapi.schema import map_primitive_types @@ -70,8 +73,21 @@ def inject(path=None) -> {variable_type}: def generate_variables(variables: List[VariableSpecDto]): + failed_variables = [] for variable in variables: - create_variable(variable) + try: + create_variable(variable) + except Exception as e: + variable_path = f"{variable.get('context', 'unknown')}.{variable.get('name', 'unknown')}" + variable_id = variable.get('id', 'unknown') + failed_variables.append(f"{variable_path} (id: {variable_id})") + logging.warning(f"WARNING: Failed to generate variable {variable_path} (id: {variable_id}): {str(e)}") + continue + + if failed_variables: + logging.warning(f"WARNING: {len(failed_variables)} variable(s) failed to generate:") + for failed_var in failed_variables: + logging.warning(f" - {failed_var}") def render_variable(variable: VariableSpecDto): @@ -116,26 +132,84 @@ def _get_variable_type(type_spec: PropertyType) -> str: def create_variable(variable: VariableSpecDto) -> None: + """ + Create a variable with atomic directory and file operations. + + Tracks directory creation to enable cleanup on failure. + """ folders = ["vari"] if variable["context"]: folders += variable["context"].split(".") # build up the full_path by adding all the folders full_path = os.path.join(os.path.dirname(os.path.abspath(__file__))) - - for idx, folder in enumerate(folders): - full_path = os.path.join(full_path, folder) - if not os.path.exists(full_path): - os.makedirs(full_path) - next = folders[idx + 1] if idx + 1 < len(folders) else None - if next: - add_import_to_init(full_path, next) - - add_variable_to_init(full_path, variable) + created_dirs = [] # Track directories we create for cleanup on failure + + try: + for idx, folder in enumerate(folders): + full_path = os.path.join(full_path, folder) + if not os.path.exists(full_path): + os.makedirs(full_path) + created_dirs.append(full_path) # Track for cleanup + next = folders[idx + 1] if idx + 1 < len(folders) else None + if next: + add_import_to_init(full_path, next) + + add_variable_to_init(full_path, variable) + + except Exception as e: + # Clean up directories we created (in reverse order) + for dir_path in reversed(created_dirs): + try: + if os.path.exists(dir_path) and not os.listdir(dir_path): # Only remove if empty + os.rmdir(dir_path) + except: + pass # Best effort cleanup + + # Re-raise the original exception + raise e def add_variable_to_init(full_path: str, variable: VariableSpecDto): - init_the_init(full_path) - init_path = os.path.join(full_path, "__init__.py") - with open(init_path, "a") as f: - f.write(render_variable(variable) + "\n\n") + """ + Atomically add a variable to __init__.py to prevent partial corruption during generation failures. + + This function generates all content first, then writes the file atomically using temporary files + to ensure that either the entire operation succeeds or no changes are made to the filesystem. + """ + try: + init_the_init(full_path) + init_path = os.path.join(full_path, "__init__.py") + + # Generate variable content first + variable_content = render_variable(variable) + if not variable_content: + raise Exception("Variable rendering failed - empty content returned") + + # Read current __init__.py content if it exists + init_content = "" + if os.path.exists(init_path): + with open(init_path, "r") as f: + init_content = f.read() + + # Prepare new content to append + new_init_content = init_content + variable_content + "\n\n" + + # Write to temporary file first, then atomic move + with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_file: + temp_file.write(new_init_content) + temp_file_path = temp_file.name + + # Atomic operation: move temp file to final location + shutil.move(temp_file_path, init_path) + + except Exception as e: + # Clean up temporary file if it exists + try: + if 'temp_file_path' in locals() and os.path.exists(temp_file_path): + os.unlink(temp_file_path) + except: + pass # Best effort cleanup + + # Re-raise the original exception + raise e diff --git a/polyapi/webhook.py b/polyapi/webhook.py index b27987d..2f11707 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -2,6 +2,7 @@ import socketio # type: ignore from socketio.exceptions import ConnectionError # type: ignore import uuid +import logging from typing import Any, Dict, List, Tuple from polyapi.config import get_api_key_and_url @@ -121,22 +122,27 @@ def render_webhook_handle( arguments: List[PropertySpecification], return_type: Dict[str, Any], ) -> Tuple[str, str]: - function_args, function_args_def = parse_arguments(function_name, arguments) - - if "WebhookEventType" in function_args: - # let's add the function name import! - function_args = function_args.replace("WebhookEventType", f"{to_func_namespace(function_name)}.WebhookEventType") - - func_str = WEBHOOK_TEMPLATE.format( - description=function_description, - client_id=uuid.uuid4().hex, - function_id=function_id, - function_name=function_name, - function_args=function_args, - function_path=poly_full_path(function_context, function_name), - ) - func_defs = WEBHOOK_DEFS_TEMPLATE.format(function_args_def=function_args_def) - return func_str, func_defs + try: + function_args, function_args_def = parse_arguments(function_name, arguments) + + if "WebhookEventType" in function_args: + # let's add the function name import! + function_args = function_args.replace("WebhookEventType", f"{to_func_namespace(function_name)}.WebhookEventType") + + func_str = WEBHOOK_TEMPLATE.format( + description=function_description, + client_id=uuid.uuid4().hex, + function_id=function_id, + function_name=function_name, + function_args=function_args, + function_path=poly_full_path(function_context, function_name), + ) + func_defs = WEBHOOK_DEFS_TEMPLATE.format(function_args_def=function_args_def) + return func_str, func_defs + except Exception as e: + logging.warning(f"Failed to render webhook handle {function_context}.{function_name} (id: {function_id}): {str(e)}") + # Return empty strings to indicate generation failure - this will be caught by generate_functions error handling + return "", "" def start(*args): diff --git a/pyproject.toml b/pyproject.toml index 497818c..ae2466c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.8.dev3" +version = "0.3.8.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ diff --git a/tests/test_generate.py b/tests/test_generate.py index b738ba6..f6f08fa 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -2,8 +2,11 @@ import os import shutil import importlib.util +from unittest.mock import patch, MagicMock from polyapi.utils import get_type_and_def, rewrite_reserved -from polyapi.generate import render_spec, create_empty_schemas_module +from polyapi.generate import render_spec, create_empty_schemas_module, generate_functions, create_function +from polyapi.poly_schemas import generate_schemas, create_schema +from polyapi.variables import generate_variables, create_variable OPENAPI_FUNCTION = { "kind": "function", @@ -287,3 +290,373 @@ def test_nested_function() -> schemas.api.v1.user.profile: # Clean up schemas directory shutil.rmtree(schemas_path) + + def test_error_handling_generate_functions(self): + """Test that generate_functions handles errors gracefully and continues with other functions""" + # Mock create_function to raise an exception for one function + failing_spec = { + "id": "failing-function-123", + "type": "serverFunction", + "context": "test", + "name": "failingFunction", + "description": "A function that will fail to generate", + } + + working_spec = { + "id": "working-function-456", + "type": "serverFunction", + "context": "test", + "name": "workingFunction", + "description": "A function that will generate successfully", + } + + specs = [failing_spec, working_spec] + + # Mock create_function to fail on the first call and succeed on the second + with patch('polyapi.generate.create_function') as mock_create: + mock_create.side_effect = [Exception("Schema generation failed"), None] + + # Capture logging output + with patch('polyapi.generate.logging.warning') as mock_warning: + generate_functions(specs) + + # Verify that create_function was called twice (once for each spec) + self.assertEqual(mock_create.call_count, 2) + + # Verify that warning messages were logged + mock_warning.assert_any_call("WARNING: Failed to generate function test.failingFunction (id: failing-function-123): Schema generation failed") + mock_warning.assert_any_call("WARNING: 1 function(s) failed to generate:") + mock_warning.assert_any_call(" - test.failingFunction (id: failing-function-123)") + + def test_error_handling_generate_schemas(self): + """Test that generate_schemas handles errors gracefully and continues with other schemas""" + from polyapi.typedefs import SchemaSpecDto + + failing_spec = { + "id": "failing-schema-123", + "type": "schema", + "context": "test", + "name": "failingSchema", + "description": "A schema that will fail to generate", + "definition": {} + } + + working_spec = { + "id": "working-schema-456", + "type": "schema", + "context": "test", + "name": "workingSchema", + "description": "A schema that will generate successfully", + "definition": {} + } + + specs = [failing_spec, working_spec] + + # Mock create_schema to fail on the first call and succeed on the second + with patch('polyapi.poly_schemas.create_schema') as mock_create: + mock_create.side_effect = [Exception("Schema generation failed"), None] + + # Capture logging output + with patch('polyapi.poly_schemas.logging.warning') as mock_warning: + generate_schemas(specs) + + # Verify that create_schema was called twice (once for each spec) + self.assertEqual(mock_create.call_count, 2) + + # Verify that warning messages were logged + mock_warning.assert_any_call("WARNING: Failed to generate schema test.failingSchema (id: failing-schema-123): Schema generation failed") + mock_warning.assert_any_call("WARNING: 1 schema(s) failed to generate:") + mock_warning.assert_any_call(" - test.failingSchema (id: failing-schema-123)") + + def test_error_handling_generate_variables(self): + """Test that generate_variables handles errors gracefully and continues with other variables""" + from polyapi.typedefs import VariableSpecDto + + failing_spec = { + "id": "failing-variable-123", + "type": "serverVariable", + "context": "test", + "name": "failingVariable", + "description": "A variable that will fail to generate", + "variable": { + "valueType": {"kind": "primitive", "type": "string"}, + "secrecy": "PUBLIC" + } + } + + working_spec = { + "id": "working-variable-456", + "type": "serverVariable", + "context": "test", + "name": "workingVariable", + "description": "A variable that will generate successfully", + "variable": { + "valueType": {"kind": "primitive", "type": "string"}, + "secrecy": "PUBLIC" + } + } + + specs = [failing_spec, working_spec] + + # Mock create_variable to fail on the first call and succeed on the second + with patch('polyapi.variables.create_variable') as mock_create: + mock_create.side_effect = [Exception("Variable generation failed"), None] + + # Capture logging output + with patch('polyapi.variables.logging.warning') as mock_warning: + generate_variables(specs) + + # Verify that create_variable was called twice (once for each spec) + self.assertEqual(mock_create.call_count, 2) + + # Verify that warning messages were logged + mock_warning.assert_any_call("WARNING: Failed to generate variable test.failingVariable (id: failing-variable-123): Variable generation failed") + mock_warning.assert_any_call("WARNING: 1 variable(s) failed to generate:") + mock_warning.assert_any_call(" - test.failingVariable (id: failing-variable-123)") + + def test_error_handling_webhook_generation(self): + """Test that render_webhook_handle handles errors gracefully during generation""" + from polyapi.webhook import render_webhook_handle + + # Test with problematic arguments that might cause rendering to fail + with patch('polyapi.webhook.parse_arguments') as mock_parse: + mock_parse.side_effect = Exception("Invalid webhook arguments") + + with patch('polyapi.webhook.logging.warning') as mock_warning: + func_str, func_defs = render_webhook_handle( + function_type="webhookHandle", + function_context="test", + function_name="failingWebhook", + function_id="webhook-123", + function_description="A webhook that fails to generate", + arguments=[], + return_type={} + ) + + # Should return empty strings on failure + self.assertEqual(func_str, "") + self.assertEqual(func_defs, "") + + # Should log a warning + mock_warning.assert_called_once_with("Failed to render webhook handle test.failingWebhook (id: webhook-123): Invalid webhook arguments") + + def test_atomic_function_generation_failure(self): + """Test that function generation failures don't leave partial corrupted files""" + import tempfile + from polyapi.generate import add_function_file + + failing_spec = { + "id": "failing-function-123", + "type": "serverFunction", + "context": "test", + "name": "failingFunction", + "description": "A function that will fail to generate", + } + + # Create a temporary directory for testing + with tempfile.TemporaryDirectory() as temp_dir: + # Mock render_spec to fail after being called + with patch('polyapi.generate.render_spec') as mock_render: + mock_render.side_effect = Exception("Rendering failed") + + # Verify that the function generation fails + with self.assertRaises(Exception): + add_function_file(temp_dir, "failingFunction", failing_spec) + + # Verify no partial files were left behind + files_in_dir = os.listdir(temp_dir) + # Should only have __init__.py from init_the_init, no corrupted function files + self.assertNotIn("failing_function.py", files_in_dir) + self.assertNotIn("failingFunction.py", files_in_dir) + + # If __init__.py exists, it should not contain partial imports + init_path = os.path.join(temp_dir, "__init__.py") + if os.path.exists(init_path): + with open(init_path, "r") as f: + init_content = f.read() + self.assertNotIn("from . import failing_function", init_content) + self.assertNotIn("from . import failingFunction", init_content) + + def test_atomic_variable_generation_failure(self): + """Test that variable generation failures don't leave partial corrupted files""" + import tempfile + from polyapi.variables import add_variable_to_init + + failing_spec = { + "id": "failing-variable-123", + "type": "serverVariable", + "context": "test", + "name": "failingVariable", + "description": "A variable that will fail to generate", + "variable": { + "valueType": {"kind": "primitive", "type": "string"}, + "secrecy": "PUBLIC" + } + } + + # Create a temporary directory for testing + with tempfile.TemporaryDirectory() as temp_dir: + # Mock render_variable to fail + with patch('polyapi.variables.render_variable') as mock_render: + mock_render.side_effect = Exception("Variable rendering failed") + + # Verify that the variable generation fails + with self.assertRaises(Exception): + add_variable_to_init(temp_dir, failing_spec) + + # Verify no partial files were left behind and __init__.py wasn't corrupted + init_path = os.path.join(temp_dir, "__init__.py") + if os.path.exists(init_path): + with open(init_path, "r") as f: + init_content = f.read() + # Should not contain partial variable content or broken imports + self.assertNotIn("failingVariable", init_content) + self.assertNotIn("class failingVariable", init_content) + + def test_atomic_schema_generation_failure(self): + """Test that schema generation failures don't leave partial files or directories""" + with patch('tempfile.TemporaryDirectory') as mock_temp_dir: + mock_temp_dir.return_value.__enter__.return_value = "/tmp/test_dir" + + # Mock the render function to fail + with patch('polyapi.poly_schemas.render_poly_schema', side_effect=Exception("Schema generation failed")): + with patch('logging.warning') as mock_warning: + # This should not crash and should log a warning + schemas = [ + { + "id": "schema1", + "name": "TestSchema", + "context": "", + "type": "schema", + "definition": {"type": "object", "properties": {"test": {"type": "string"}}} + } + ] + generate_schemas(schemas) + + # Should have logged a warning about the failed schema + mock_warning.assert_called() + warning_calls = [call[0][0] for call in mock_warning.call_args_list] + # Check that both the main warning and summary warning are present + self.assertTrue(any("Failed to generate schema" in call for call in warning_calls)) + self.assertTrue(any("TestSchema" in call for call in warning_calls)) + self.assertTrue(any("schema1" in call for call in warning_calls)) + + def test_broken_imports_not_left_on_function_failure(self): + """Test that if a function fails after directories are created, we don't leave broken imports""" + import tempfile + import shutil + import os + from polyapi import generate + + with tempfile.TemporaryDirectory() as temp_dir: + # Create a mock polyapi directory structure + polyapi_dir = os.path.join(temp_dir, "polyapi") + os.makedirs(polyapi_dir) + + # Mock spec that would create a nested structure: poly/context/function_name + spec = { + "id": "test-func-id", + "name": "test_function", + "context": "test_context", + "type": "apiFunction", + "description": "Test function", + "function": { + "arguments": [], + "returnType": {"kind": "any"} + } + } + + # Mock the add_function_file to fail AFTER directories are created + + def failing_add_function_file(*args, **kwargs): + raise Exception("Function file creation failed") + + with patch('polyapi.generate.add_function_file', side_effect=failing_add_function_file): + with patch('os.path.dirname') as mock_dirname: + mock_dirname.return_value = polyapi_dir + with patch('logging.warning') as mock_warning: + + # This should fail gracefully + try: + generate.create_function(spec) + except: + pass # Expected to fail + + # Check that no intermediate directories were left behind + poly_dir = os.path.join(polyapi_dir, "poly") + if os.path.exists(poly_dir): + context_dir = os.path.join(poly_dir, "test_context") + + # If intermediate directories exist, they should not have broken imports + if os.path.exists(context_dir): + init_file = os.path.join(context_dir, "__init__.py") + if os.path.exists(init_file): + with open(init_file, 'r') as f: + content = f.read() + # Should not contain import for the failed function + self.assertNotIn("test_function", content) + + # The function directory should not exist + func_dir = os.path.join(context_dir, "test_function") + self.assertFalse(os.path.exists(func_dir)) + + def test_intermediate_init_files_handle_failure_correctly(self): + """Test that intermediate __init__.py files are handled correctly when function generation fails""" + import tempfile + import os + from polyapi import generate + + with tempfile.TemporaryDirectory() as temp_dir: + polyapi_dir = os.path.join(temp_dir, "polyapi") + os.makedirs(polyapi_dir) + + # Create a poly directory and context directory beforehand + poly_dir = os.path.join(polyapi_dir, "poly") + context_dir = os.path.join(poly_dir, "test_context") + os.makedirs(context_dir) + + # Put some existing content in the context __init__.py + init_file = os.path.join(context_dir, "__init__.py") + with open(init_file, 'w') as f: + f.write("# Existing context init file\nfrom . import existing_function\n") + + spec = { + "id": "test-func-id", + "name": "failing_function", + "context": "test_context", + "type": "apiFunction", + "description": "Test function", + "function": { + "arguments": [], + "returnType": {"kind": "any"} + } + } + + # Mock add_function_file to fail + def failing_add_function_file(full_path, function_name, spec): + # This simulates failure AFTER intermediate directories are processed + # but BEFORE the final function file is written + raise Exception("Function file creation failed") + + with patch('polyapi.generate.add_function_file', side_effect=failing_add_function_file): + with patch('os.path.dirname') as mock_dirname: + mock_dirname.return_value = polyapi_dir + + # This should fail but handle cleanup gracefully + try: + generate.create_function(spec) + except: + pass # Expected to fail + + # The context __init__.py should not contain import for failed function + with open(init_file, 'r') as f: + content = f.read() + + # Should still have existing content + self.assertIn("existing_function", content) + # Should NOT have the failed function + self.assertNotIn("failing_function", content) + + # The failed function directory should not exist + func_dir = os.path.join(context_dir, "failing_function") + self.assertFalse(os.path.exists(func_dir)) From 2b9f98982c4d704e5bb53cbe48b9c1551310a0bd Mon Sep 17 00:00:00 2001 From: Richard Dzurus Date: Wed, 25 Jun 2025 15:23:03 +0200 Subject: [PATCH 2/3] version increase --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ae2466c..e27bb41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.8.dev4" +version = "0.3.8.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From e05fbf8afc21487d0320b8fc2669246e33f69af4 Mon Sep 17 00:00:00 2001 From: Richard Dzurus Date: Fri, 27 Jun 2025 15:21:17 +0200 Subject: [PATCH 3/3] version increment --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 516a121..05cb142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.8.dev7" +version = "0.3.8.dev8" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [