From 2a4b53817da035ba20fa7e98c78b2c85e86978d1 Mon Sep 17 00:00:00 2001 From: Sudipta at TechJays Date: Thu, 31 Oct 2024 20:38:25 +0600 Subject: [PATCH 01/72] # Feature (2970): Update python client to support setup command (#22) * # Feature (2970): Update python client to support setup command - Function add command now support --execution-api-key - Extra Old Function call removed --- polyapi/cli.py | 12 +++++++++++- polyapi/function_cli.py | 5 +++++ pyproject.toml | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 4027463..66dd234 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -28,6 +28,7 @@ def execute_from_cli() -> None: parser.add_argument("--server", action="store_true", help="Pass --server when adding function to add a server function.") parser.add_argument("--logs", action="store_true", help="Pass --logs when adding function if you want to store and see the function logs.") parser.add_argument("--skip-generate", action="store_true", help="Pass --skip-generate to skip generating the library after adding a function.") + parser.add_argument("--execution-api-key", required=False, default="", help="API key for execution (for server functions only).") parser.add_argument("command", choices=CLI_COMMANDS) parser.add_argument("subcommands", nargs="*") args = parser.parse_args() @@ -59,4 +60,13 @@ def execute_from_cli() -> None: if args.subcommands[0] == "execute": print(function_execute(args.context, args.subcommands)) else: - function_add_or_update(args.context, args.description, args.client, args.server, args.logs, args.subcommands, not args.skip_generate) + function_add_or_update( + context=args.context, + description=args.description, + client=args.client, + server=args.server, + logs_enabled=args.logs, + subcommands=args.subcommands, + generate=not args.skip_generate, + execution_api_key=args.execution_api_key + ) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index c6ec54d..15e5f7b 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -208,6 +208,7 @@ def function_add_or_update( logs_enabled: bool, subcommands: List, generate: bool = True, + execution_api_key: str = "" ): parser = argparse.ArgumentParser() parser.add_argument("subcommand", choices=["add"]) @@ -256,6 +257,10 @@ def function_add_or_update( assert api_key if server: url = f"{api_url}/functions/server" + + if execution_api_key: + data["executionApiKey"] = execution_api_key + elif client: url = f"{api_url}/functions/client" else: diff --git a/pyproject.toml b/pyproject.toml index 33a89f6..3969c19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.9" +version = "0.3.0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 87239787efd53a41a4ddec1da93cb9b542009ab1 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 4 Nov 2024 08:02:52 -0800 Subject: [PATCH 02/72] improve polyapi-python setup (#24) * improve polyapi-python setup * # Feature (3019): improve polyapi-python setup (#25) * # Feature (3019): improve polyapi-python setup * # Feature (3019): improve polyapi-python setup - UUID Validation check added --------- Co-authored-by: Sudipta at TechJays --- polyapi/cli.py | 4 ++-- polyapi/config.py | 28 +++++++++++++++++++++++----- polyapi/utils.py | 22 +++++++++++++++++++++- pyproject.toml | 2 +- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 66dd234..af5ab46 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -2,7 +2,7 @@ from polyapi.utils import print_green -from .config import clear_config, set_api_key_and_url +from .config import initialize_config, set_api_key_and_url from .generate import generate, clear from .function_cli import function_add_or_update, function_execute from .rendered_spec import get_and_update_rendered_spec @@ -43,7 +43,7 @@ def execute_from_cli() -> None: elif command == "setup" and len(args.subcommands) == 2: set_api_key_and_url(args.subcommands[1], args.subcommands[0]) elif command == "setup": - clear_config() + initialize_config(force=True) generate() elif command == "update_rendered_spec": assert len(args.subcommands) == 1 diff --git a/polyapi/config.py b/polyapi/config.py index 5e0e143..19016f5 100644 --- a/polyapi/config.py +++ b/polyapi/config.py @@ -3,6 +3,8 @@ import configparser from typing import Tuple +from polyapi.utils import is_valid_polyapi_url, is_valid_uuid, print_green, print_yellow + # cached values API_KEY = None API_URL = None @@ -55,18 +57,34 @@ def set_api_key_and_url(key: str, url: str): config.write(f) -def initialize_config(): +def initialize_config(force=False): key, url = get_api_key_and_url() - if not key or not url: + if force or (not key or not url): + url = url or "https://na1.polyapi.io" print("Please setup your connection to PolyAPI.") - url = input("? Poly API Base URL (https://na1.polyapi.io): ") or "https://na1.polyapi.io" - key = input("? Poly App Key or User Key: ") + url = input(f"? Poly API Base URL ({url}): ").strip() or url + + if not key: + key = input("? Poly App Key or User Key: ").strip() + else: + key_input = input(f"? Poly App Key or User Key ({key}): ").strip() + key = key_input if key_input else key if url and key: + errors = [] + if not is_valid_polyapi_url(url): + errors.append(f"{url} is not a valid Poly API Base URL") + if not is_valid_uuid(key): + errors.append(f"{key} is not a valid Poly App Key or User Key") + if errors: + print_yellow("\n".join(errors)) + sys.exit(1) + set_api_key_and_url(key, url) + print_green(f"Poly setup complete.") if not key or not url: - print("Poly API Key and Poly API Base URL are required.") + print_yellow("Poly API Key and Poly API Base URL are required.") sys.exit(1) return key, url diff --git a/polyapi/utils.py b/polyapi/utils.py index 259e727..a5141a6 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -1,6 +1,7 @@ import keyword import re import os +import uuid from typing import Tuple, List from colorama import Fore, Style from polyapi.constants import BASIC_PYTHON_TYPES @@ -208,4 +209,23 @@ def rewrite_reserved(s: str) -> str: def rewrite_arg_name(s: str): - return rewrite_reserved(camelCase(s)) \ No newline at end of file + return rewrite_reserved(camelCase(s)) + + +valid_subdomains = ["na[1-2]", "eu[1-2]", "dev"] + + +def is_valid_polyapi_url(_url: str): + # Join the subdomains into a pattern + subdomain_pattern = "|".join(valid_subdomains) + pattern = rf"^https://({subdomain_pattern})\.polyapi\.io$" + return re.match(pattern, _url) is not None + + +def is_valid_uuid(uuid_string, version=4): + try: + uuid_obj = uuid.UUID(uuid_string, version=version) + except ValueError: + return False + + return str(uuid_obj) == uuid_string diff --git a/pyproject.toml b/pyproject.toml index 3969c19..3ae5911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.0" +version = "0.3.1.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 7ece4d0121091771a25e243fde73501480ccc31a Mon Sep 17 00:00:00 2001 From: Sudipta at TechJays Date: Mon, 4 Nov 2024 23:49:22 +0600 Subject: [PATCH 03/72] # Feature (3007): Update python -m polyapi function add --logs options (#23) * # Feature (3007): Update python -m polyapi function add --logs options - if --logs added, then value must enabled or disabled - If Nothing passed the value is default disabled - pyproject.toml version updated --- polyapi/cli.py | 5 +++-- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index af5ab46..f450301 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -26,7 +26,7 @@ def execute_from_cli() -> None: parser.add_argument("--description", required=False, default="") parser.add_argument("--client", action="store_true", help="Pass --client when adding function to add a client function.") parser.add_argument("--server", action="store_true", help="Pass --server when adding function to add a server function.") - parser.add_argument("--logs", action="store_true", help="Pass --logs when adding function if you want to store and see the function logs.") + parser.add_argument("--logs", choices=["enabled", "disabled"], default="disabled", help="Enable or disable logs for the function.") parser.add_argument("--skip-generate", action="store_true", help="Pass --skip-generate to skip generating the library after adding a function.") parser.add_argument("--execution-api-key", required=False, default="", help="API key for execution (for server functions only).") parser.add_argument("command", choices=CLI_COMMANDS) @@ -57,6 +57,7 @@ def execute_from_cli() -> None: print("Clearing the generated library...") clear() elif command == "function": + logs_enabled = args.logs == "enabled" if args.subcommands[0] == "execute": print(function_execute(args.context, args.subcommands)) else: @@ -65,7 +66,7 @@ def execute_from_cli() -> None: description=args.description, client=args.client, server=args.server, - logs_enabled=args.logs, + logs_enabled=logs_enabled, subcommands=args.subcommands, generate=not args.skip_generate, execution_api_key=args.execution_api_key diff --git a/pyproject.toml b/pyproject.toml index 3ae5911..6dfc79c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev0" +version = "0.3.1.dev1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From e292e4e27f2e7b3d4bd665b306cf8bddca2a6d06 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Wed, 13 Nov 2024 16:41:37 -0800 Subject: [PATCH 04/72] Project Glide + Refactor main command line args parsing (#26) * Refactor main command line args parsing, adding prepare and sync commands to enable project glide workflows for python * improved tests * updating version --- polyapi/__init__.py | 8 +- polyapi/cli.py | 217 ++++++++++++---- polyapi/deployables.py | 296 +++++++++++++++++++++ polyapi/function_cli.py | 233 ++--------------- polyapi/generate.py | 3 +- polyapi/parser.py | 519 +++++++++++++++++++++++++++++++++++++ polyapi/prepare.py | 135 ++++++++++ polyapi/sync.py | 122 +++++++++ polyapi/typedefs.py | 24 +- pyproject.toml | 2 +- tests/test_deployables.py | 114 ++++++++ tests/test_function_cli.py | 92 ------- tests/test_parser.py | 243 +++++++++++++++++ 13 files changed, 1642 insertions(+), 366 deletions(-) create mode 100644 polyapi/deployables.py create mode 100644 polyapi/parser.py create mode 100644 polyapi/prepare.py create mode 100644 polyapi/sync.py create mode 100644 tests/test_deployables.py delete mode 100644 tests/test_function_cli.py create mode 100644 tests/test_parser.py diff --git a/polyapi/__init__.py b/polyapi/__init__.py index f818b53..2a30c36 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -16,8 +16,8 @@ polyCustom: Dict[str, Any] = { - "executionId": None, - "executionApiKey": None, - "responseStatusCode": 200, - "responseContentType": None, + "executionId": None, + "executionApiKey": None, + "responseStatusCode": 200, + "responseContentType": None, } \ No newline at end of file diff --git a/polyapi/cli.py b/polyapi/cli.py index f450301..7f28a46 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -1,73 +1,186 @@ +import os import argparse -from polyapi.utils import print_green +from polyapi.utils import print_green, print_red from .config import initialize_config, set_api_key_and_url from .generate import generate, clear from .function_cli import function_add_or_update, function_execute from .rendered_spec import get_and_update_rendered_spec +from .prepare import prepare_deployables +from .sync import sync_deployables CLI_COMMANDS = ["setup", "generate", "function", "clear", "help", "update_rendered_spec"] -CLIENT_DESC = """Commands - python -m polyapi setup Setup your Poly connection - python -m polyapi generate Generates Poly library - python -m polyapi function Manages functions - python -m polyapi clear Clear current generated Poly library -""" - - -def execute_from_cli() -> None: +def execute_from_cli(): + # First we setup all our argument parsing logic + # Then we parse the arguments (waaay at the bottom) parser = argparse.ArgumentParser( - prog="python -m polyapi", description=CLIENT_DESC, formatter_class=argparse.RawTextHelpFormatter + prog="python -m polyapi", + description="Manage your Poly API configurations and functions", + formatter_class=argparse.RawTextHelpFormatter ) - parser.add_argument("--context", required=False, default="") - parser.add_argument("--description", required=False, default="") - parser.add_argument("--client", action="store_true", help="Pass --client when adding function to add a client function.") - parser.add_argument("--server", action="store_true", help="Pass --server when adding function to add a server function.") - parser.add_argument("--logs", choices=["enabled", "disabled"], default="disabled", help="Enable or disable logs for the function.") - parser.add_argument("--skip-generate", action="store_true", help="Pass --skip-generate to skip generating the library after adding a function.") - parser.add_argument("--execution-api-key", required=False, default="", help="API key for execution (for server functions only).") - parser.add_argument("command", choices=CLI_COMMANDS) - parser.add_argument("subcommands", nargs="*") - args = parser.parse_args() - command = args.command - - if command == "help": - parser.print_help() - elif command == "generate": + + subparsers = parser.add_subparsers(help="Available commands") + + ########################################################################### + # Setup command + setup_parser = subparsers.add_parser("setup", help="Setup your Poly connection") + setup_parser.add_argument("api_key", nargs="?", help="API key for Poly API") + setup_parser.add_argument("url", nargs="?", help="URL for the Poly API") + + def setup(args): + if args.api_key and args.url: + set_api_key_and_url(args.url, args.api_key) + else: + initialize_config(force=True) + generate() + + setup_parser.set_defaults(command=setup) + + + ########################################################################### + # Generate command + generate_parser = subparsers.add_parser("generate", help="Generates Poly library") + + def generate_command(args): + initialize_config() print("Generating Poly functions...", end="") generate() print_green("DONE") - elif command == "setup" and len(args.subcommands) == 2: - set_api_key_and_url(args.subcommands[1], args.subcommands[0]) - elif command == "setup": - initialize_config(force=True) - generate() - elif command == "update_rendered_spec": - assert len(args.subcommands) == 1 - updated = get_and_update_rendered_spec(args.subcommands[0]) + + generate_parser.set_defaults(command=generate_command) + + + ########################################################################### + # Function commands + fn_parser = subparsers.add_parser("function", help="Manage and execute functions") + fn_subparsers = fn_parser.add_subparsers(help="Available commands") + + # Function - Add command + fn_add_parser = fn_subparsers.add_parser("add", help="Add or update the function") + fn_add_parser.add_argument("name", help="Name of the function") + fn_add_parser.add_argument("file", help="Path to the function file") + fn_add_parser.add_argument("--context", required=False, default="", help="Context of the function") + fn_add_parser.add_argument("--description", required=False, default="", help="Description of the function") + fn_add_parser.add_argument("--server", action="store_true", help="Marks the function as a server function") + fn_add_parser.add_argument("--client", action="store_true", help="Marks the function as a client function") + fn_add_parser.add_argument("--logs", choices=["enabled", "disabled"], default="disabled", help="Enable or disable logs for the function.") + fn_add_parser.add_argument("--execution-api-key", required=False, default="", help="API key for execution (for server functions only).") + fn_add_parser.add_argument("--disable-ai", "--skip-generate", action="store_true", help="Pass --disable-ai skip AI generation of missing descriptions") + + def add_function(args): + initialize_config() + logs_enabled = args.logs == "enabled" + err = "" + if args.server and args.client: + err = "Specify either `--server` or `--client`. Found both." + elif not args.server and not args.client: + err = "You must specify `--server` or `--client`." + elif logs_enabled and not args.server: + err = "Option `logs` is only for server functions (--server)." + + if err: + print_red("ERROR") + print(err) + exit(1) + + function_add_or_update( + context=args.context, + description=args.description, + client=args.client, + server=args.server, + logs_enabled=logs_enabled, + generate=not args.disable_ai, + execution_api_key=args.execution_api_key + ) + + fn_add_parser.set_defaults(command=add_function) + + + # Function - Execute command + fn_exec_parser = fn_subparsers.add_parser("execute", help="Execute a function with the provided arguments") + fn_exec_parser.add_argument("name", help="Name of the function") + fn_exec_parser.add_argument("args", nargs="*", help="Arguments for the function") + fn_exec_parser.add_argument("--context", required=False, default="", help="Context of the function") + + def execute_function(args): + initialize_config() + print(function_execute(args.context, args.name, args.args)) + + fn_exec_parser.set_defaults(command=execute_function) + + + ########################################################################### + # Clear command + clear_parser = subparsers.add_parser("clear", help="Clear current generated Poly library") + + def clear_command(_): + print("Clearing the generated library...") + clear() + + clear_parser.set_defaults(command=clear_command) + + + ########################################################################### + # Update rendered spec command + update_spec_parser = subparsers.add_parser("update_rendered_spec", help="Update the rendered spec file") + update_spec_parser.add_argument("spec", help="Specification file to update") + + def update_rendered_spec(args): + updated = get_and_update_rendered_spec(args.spec) if updated: print("Updated rendered spec!") else: print("Failed to update rendered spec!") exit(1) - elif command == "clear": - print("Clearing the generated library...") - clear() - elif command == "function": - logs_enabled = args.logs == "enabled" - if args.subcommands[0] == "execute": - print(function_execute(args.context, args.subcommands)) - else: - function_add_or_update( - context=args.context, - description=args.description, - client=args.client, - server=args.server, - logs_enabled=logs_enabled, - subcommands=args.subcommands, - generate=not args.skip_generate, - execution_api_key=args.execution_api_key - ) + + update_spec_parser.set_defaults(command=update_rendered_spec) + + + ########################################################################### + # Prepare command + prepare_parser = subparsers.add_parser('prepare', help="Find and prepare all Poly deployables") + prepare_parser.add_argument("--lazy", action="store_true", help="Skip prepare work if the cache is up to date. (Relies on `git`)") + prepare_parser.add_argument("--disable-docs", action="store_true", help="Don't write any docstrings into the deployable files.") + prepare_parser.add_argument("--disable-ai", action="store_true", help="Don't use AI to fill in any missing descriptions.") + + def prepare(args): + initialize_config() + disable_ai = args.disable_ai or bool(os.getenv("DISABLE_AI")) + prepare_deployables(lazy=args.lazy, disable_docs=args.disable_docs, disable_ai=disable_ai) + + prepare_parser.set_defaults(command=prepare) + + + ########################################################################### + # Sync command + sync_parser = subparsers.add_parser("sync", help="Find and sync all Poly deployables") + sync_parser.add_argument("--dry-run", action="store_true", help="Run through sync steps with logging but don't make any changes.") + + def sync(args): + initialize_config() + prepare_deployables(lazy=True, disable_docs=True, disable_ai=True) + if args.dry_run: + print("Running dry-run of sync...") + sync_deployables(dry_run=args.dry_run) + print("Poly deployments synced.") + + sync_parser.set_defaults(command=sync) + + ########################################################################### + # _------. # + # / , \_ __ __ _ ________ # + # / / /{}\ |o\_ / / ___ / /( )_____ / ____/ /_ __ # + # / \ `--' /-' \ / / / _ \/ __/// ___/ / /_ / / / / / # + # | \ \ | / /___/ __/ /_ (__ ) / __/ / / /_/ / # + # | |`-, | /_____/\___/\__/ /____/ /_/ /_/\__, / # + # / /__/)/ /____/ # + # / | # + ########################################################################### + parsed_args = parser.parse_args() + if hasattr(parsed_args, "command"): + parsed_args.command(parsed_args) + else: + parser.print_help() diff --git a/polyapi/deployables.py b/polyapi/deployables.py new file mode 100644 index 0000000..4d7f4f1 --- /dev/null +++ b/polyapi/deployables.py @@ -0,0 +1,296 @@ +import os +import subprocess +import json +import hashlib +from pathlib import Path +from typing import TypedDict, List, Dict, Tuple, Optional, Any +from subprocess import check_output, CalledProcessError + + +# Constants +CACHE_VERSION_FILE = "./poly/deployments_revision" +CACHE_DIR = Path("./poly/deployables") + + +class DeployableTypes(str): + pass + +class DeployableTypeNames(str): + pass + +class Deployment(TypedDict): + context: str + name: str + type: DeployableTypes + instance: str + id: str + deployed: str + fileRevision: str + +class ParsedDeployableConfig(TypedDict): + context: str + name: str + type: DeployableTypes + disableAi: Optional[bool] + config: Dict[str, Any] + +class DeployableFunctionParamBase(TypedDict): + type: str + typeSchema: Optional[Dict[str, Any]] + description: str + +class DeployableFunctionParam(DeployableFunctionParamBase): + name: str + +class DeployableFunctionTypes(TypedDict): + description: str + params: List[DeployableFunctionParam] + returns: DeployableFunctionParamBase + +class DeployableRecord(ParsedDeployableConfig, total=False): + gitRevision: str + fileRevision: str + file: str + types: DeployableFunctionTypes + typeSchemas: Dict[str, Any] + dependencies: List[str] + deployments: List[Deployment] + deploymentCommentRanges: List[Tuple[int, int]] + docStartIndex: int + docEndIndex: int + dirty: Optional[bool] + +class SyncDeployment(TypedDict, total=False): + context: str + name: str + description: str + type: str # This should be an enumeration or a predefined set of strings if you have known types. + fileRevision: str + file: str + typeSchemas: Dict[str, any] + dependencies: List[str] + config: Dict[str, any] + instance: str + id: Optional[str] = None + deployed: Optional[str] = None + +DeployableTypeEntries: List[Tuple[DeployableTypeNames, DeployableTypes]] = [ + ("PolyServerFunction", "server-function"), + ("PolyClientFunction", "client-function"), +] + +DeployableTypeToName: Dict[DeployableTypeNames, DeployableTypes] = {name: type for name, type in DeployableTypeEntries} + +def prepare_deployable_directory() -> None: + Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) + +def load_deployable_records() -> List[DeployableRecord]: + return [read_json_file(CACHE_DIR / name) for name in os.listdir(CACHE_DIR) if name.endswith(".json")] + +def save_deployable_records(records: List[DeployableRecord]) -> None: + for record in records: + write_json_file(Path(CACHE_DIR) / f'{record["context"]}.{record["name"]}.json', record) + +def remove_deployable_records(records: List[DeployableRecord]) -> None: + for record in records: + os.remove(Path(CACHE_DIR) / f'{record["context"]}.{record["name"]}.json') + +def read_json_file(path: Path) -> Any: + with open(path, "r", encoding="utf-8") as file: + return json.load(file) + +def write_json_file(path: Path, contents: Any) -> None: + with open(path, "w", encoding="utf-8") as file: + json.dump(contents, file, indent=2) + +class PolyDeployConfig(TypedDict): + type_names: List[str] + include_dirs: List[str] + include_files_or_extensions: List[str] + exclude_dirs: List[str] + +def get_all_deployable_files_windows(config: PolyDeployConfig) -> List[str]: + # Constructing the Windows command using dir and findstr + include_pattern = " ".join(f"*.{f}" if "." in f else f"*.{f}" for f in config["include_files_or_extensions"]) or "*" + exclude_pattern = '|'.join(config["exclude_dirs"]) + pattern = '|'.join(f"polyConfig: {name}" for name in config["type_names"]) or 'polyConfig' + + exclude_command = f" | findstr /V /I \"{exclude_pattern}\"" if exclude_pattern else '' + search_command = f" | findstr /M /I /F:/ /C:\"{pattern}\"" + + result = [] + for dir_path in config["include_dirs"]: + dir_command = f"dir /S /P /B {include_pattern} {dir_path}" + full_command = f"{dir_command}{exclude_command}{search_command}" + try: + output = subprocess.check_output(full_command, shell=True, text=True) + result.extend(output.strip().split('\r\n')) + except subprocess.CalledProcessError: + pass + return result + +def get_all_deployable_files_linux(config: PolyDeployConfig) -> List[str]: + # Constructing the Linux grep command + include = " ".join(f'--include={f if "." in f else f"*.{f}"}' for f in config["include_files_or_extensions"]) + exclude_dir = " ".join(f"--exclude-dir={dir}" for dir in config["exclude_dirs"]) + + search_path = " ".join(config["include_dirs"]) or "." + patterns = " ".join(f"-e 'polyConfig: {name}'" for name in config["type_names"]) or "-e 'polyConfig'" + grep_command = f'grep {include} {exclude_dir} -Rl {patterns} {search_path}' + + try: + output = subprocess.check_output(grep_command, shell=True, text=True) + return output.strip().split('\n') + except subprocess.CalledProcessError: + return [] + +def get_all_deployable_files(config: PolyDeployConfig) -> List[str]: + # Setting default values if not provided + if not config.get("type_names"): + config["type_names"] = [entry[0] for entry in DeployableTypeEntries] # Assuming DeployableTypeEntries is defined elsewhere + if not config.get("include_dirs"): + config["include_dirs"] = ["."] + if not config.get("include_files_or_extensions"): + config["include_files_or_extensions"] = ["py"] + if not config.get("exclude_dirs"): + config["exclude_dirs"] = ["poly", "node_modules", "dist", "build", "output", ".vscode", ".poly", ".github", ".husky", ".yarn"] + + is_windows = os.name == "nt" + if is_windows: + return get_all_deployable_files_windows(config) + else: + return get_all_deployable_files_linux(config) + +def get_deployable_file_revision(file_contents: str) -> str: + # Remove leading single-line comments and hash the remaining contents + file_contents = "\n".join(line for line in file_contents.split("\n") if not line.strip().startswith("#")) + return hashlib.sha256(file_contents.encode('utf-8')).hexdigest()[:7] + +def get_git_revision(branch_or_tag: str = "HEAD") -> str: + try: + return check_output(["git", "rev-parse", "--short", branch_or_tag], text=True).strip() + except CalledProcessError: + # Return a random 7-character hash as a fallback + return "".join(format(ord(c), 'x') for c in os.urandom(4))[:7] + +def get_cache_deployments_revision() -> str: + """Retrieve the cache deployments revision from a file.""" + try: + with open(CACHE_VERSION_FILE, 'r', encoding='utf-8') as file: + return file.read().strip() + except FileNotFoundError: + return '' + +def write_cache_revision(git_revision: Optional[str] = None) -> None: + if git_revision is None: + git_revision = get_git_revision() + with open(CACHE_VERSION_FILE, 'w', encoding='utf-8') as file: + file.write(git_revision) + +def is_cache_up_to_date() -> bool: + if not Path(CACHE_VERSION_FILE).exists(): + return False + with open(CACHE_VERSION_FILE, 'r', encoding='utf-8') as file: + cached_revision = file.read().strip() + git_revision = get_git_revision() + return cached_revision == git_revision + +def is_cache_up_to_date() -> bool: + """Check if the cached revision matches the current Git revision.""" + cached_revision = get_cache_deployments_revision() + git_revision = get_git_revision() # This function needs to be defined or imported + return cached_revision == git_revision + +def write_deploy_comments(deployments: List[Dict]) -> str: + """Generate a string of deployment comments for each deployment.""" + canopy_path = 'polyui/collections' if 'localhost' in os.getenv('POLY_API_BASE_URL', '') else 'canopy/polyui/collections' + comments = [] + for d in deployments: + instance_url = d['instance'].replace(':8000', ':3000') if d['instance'].endswith(':8000') else d['instance'] + comment = f"# Poly deployed @ {d['deployed']} - {d['context']}.{d['name']} - {instance_url}/{canopy_path}/{d['type']}s/{d['id']} - {d['fileRevision']}" + comments.append(comment) + return '\n'.join(comments) + +def print_docstring_function_comment(description: str, args: list, returns: dict) -> str: + docstring = f'"""{description}\n\n' + if args: + docstring += ' Args:\n' + for arg in args: + name = arg.get('name') + arg_type = arg.get('type', '') + desc = arg.get('description', '') + if arg_type: + docstring += f' {name} ({arg_type}): {desc}\n' + else: + docstring += f' {name}: {desc}\n' + + return_type = returns.get('type', '') + return_description = returns.get('description', '') + if return_type: + docstring += f'\n Returns:\n {return_type}: {return_description}\n' + else: + docstring += f'\n Returns:\n {return_description}\n' + + docstring += ' """' + return docstring + + +def update_deployment_comments(file_content: str, deployable: dict) -> str: + """ + Remove old deployment comments based on the provided ranges and add new ones. + """ + for range in reversed(deployable['deploymentCommentRanges']): + file_content = file_content[:range[0]] + file_content[range[1]:] + if deployable['deployments']: + deployment_comments = write_deploy_comments(deployable['deployments']) + deployable['deploymentCommentRanges'] = [(0, len(deployment_comments) + 1)] + file_content = f"{deployment_comments}\n{file_content}" + return file_content + +def update_deployable_function_comments(file_content: str, deployable: dict, disable_docs: bool = False) -> str: + """ + Update the docstring in the file content based on the deployable's documentation data. + """ + if not disable_docs: + docstring = print_docstring_function_comment( + deployable['types']['description'], + deployable['types']['params'], + deployable['types']['returns'] + ) + if deployable["docStartIndex"] == deployable["docEndIndex"]: + # Function doesn't yet have any docstrings so we need to add additional whitespace + docstring = " " + docstring + "\n" + + return f"{file_content[:deployable['docStartIndex']]}{docstring}{file_content[deployable['docEndIndex']:]}" + return file_content + +def write_updated_deployable(deployable: dict, disable_docs: bool = False) -> dict: + """ + Read the deployable's file, update its comments and docstring, and write back to the file. + """ + with open(deployable['file'], 'r', encoding='utf-8') as file: + file_contents = file.read() + + if deployable['type'] in ['client-function', 'server-function']: + file_contents = update_deployable_function_comments(file_contents, deployable, disable_docs) + else: + raise ValueError(f"Unsupported deployable type: '{deployable['type']}'") + + file_contents = update_deployment_comments(file_contents, deployable) + + with open(deployable['file'], 'w', encoding='utf-8') as file: + file.write(file_contents) + + deployable['fileRevision'] = get_deployable_file_revision(file_contents) + return deployable + +def write_deploy_comments(deployments: list) -> str: + """ + Generate deployment comments for each deployment record. + """ + canopy_path = 'polyui/collections' if 'localhost' in os.getenv('POLY_API_BASE_URL', '') else 'canopy/polyui/collections' + comments = [] + for d in deployments: + instance_url = d['instance'].replace(':8000', ':3000') if d['instance'].endswith(':8000') else d['instance'] + comments.append(f"# Poly deployed @ {d['deployed']} - {d['context']}.{d['name']} - {instance_url}/{canopy_path}/{d['type']}s/{d['id']} - {d['fileRevision']}") + return "\n".join(comments) \ No newline at end of file diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 15e5f7b..22c0c6e 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -1,197 +1,13 @@ -import ast -import argparse -import json -import types import sys -from typing import Any, Dict, List, Mapping, Optional, Tuple -from typing import _TypedDictMeta as BaseTypedDict # type: ignore -from typing_extensions import _TypedDictMeta # type: ignore +from typing import Any, List import requests -from stdlib_list import stdlib_list -from pydantic import TypeAdapter -from importlib.metadata import packages_distributions from polyapi.generate import get_functions_and_parse, generate_functions from polyapi.config import get_api_key_and_url -from polyapi.constants import PYTHON_TO_JSONSCHEMA_TYPE_MAP from polyapi.utils import get_auth_headers, print_green, print_red, print_yellow +from polyapi.parser import parse_function_code, get_jsonschema_type import importlib -# these libraries are already installed in the base docker image -# and shouldnt be included in additional requirements -BASE_REQUIREMENTS = { - "polyapi", - "requests", - "typing_extensions", - "jsonschema-gentypes", - "pydantic", - "cloudevents", -} -all_stdlib_symbols = stdlib_list(".".join([str(v) for v in sys.version_info[0:2]])) -BASE_REQUIREMENTS.update( - all_stdlib_symbols -) # dont need to pip install stuff in the python standard library - - -def _get_schemas(code: str) -> List[Dict]: - schemas = [] - user_code = types.SimpleNamespace() - exec(code, user_code.__dict__) - for name, obj in user_code.__dict__.items(): - if isinstance(obj, BaseTypedDict): - print_red("ERROR") - print_red("\nERROR DETAILS: ") - print( - "It looks like you have used TypedDict in a custom function. Please use `from typing_extensions import TypedDict` instead. The `typing_extensions` version is more powerful and better allows us to provide rich types for your function." - ) - sys.exit(1) - elif ( - isinstance(obj, type) - and isinstance(obj, _TypedDictMeta) - and name != "TypedDict" - ): - schemas.append(TypeAdapter(obj).json_schema()) - return schemas - - -def _get_jsonschema_type(python_type: str): - if python_type == "Any": - return "Any" - - if python_type == "List": - return "array" - - if python_type.startswith("List["): - # the actual type will be returned as return_type_schema - subtype = python_type[5:-1] - if subtype == "Any": - return "any[]" - elif subtype in ["int", "float", "str", "bool"]: - jsonschema_type = PYTHON_TO_JSONSCHEMA_TYPE_MAP.get(subtype) - return f"{jsonschema_type}[]" - else: - # the schema will handle it! - return "object" - - if python_type.startswith("Dict"): - return "object" - - rv = PYTHON_TO_JSONSCHEMA_TYPE_MAP.get(python_type) - if rv: - return rv - - # should be custom type - return python_type - - -def get_python_type_from_ast(expr: ast.expr) -> str: - if isinstance(expr, ast.Name): - return str(expr.id) - elif isinstance(expr, ast.Subscript): - assert isinstance(expr, ast.Subscript) - name = getattr(expr.value, "id", "") - if name == "List": - slice = getattr(expr.slice, "id", "Any") - return f"List[{slice}]" - elif name == "Dict": - if expr.slice and isinstance(expr.slice, ast.Tuple): - key = get_python_type_from_ast(expr.slice.dims[0]) - value = get_python_type_from_ast(expr.slice.dims[1]) - return f"Dict[{key}, {value}]" - else: - return "Dict" - return "Any" - else: - return "Any" - - -def _get_type_schema(json_type: str, python_type: str, schemas: List[Dict]): - if python_type.startswith("List["): - subtype = python_type[5:-1] - for schema in schemas: - if schema["title"] == subtype: - return {"type": "array", "items": schema} - - # subtype somehow not in schema, just call it any - return None - else: - for schema in schemas: - if schema["title"] == json_type: - return schema - - -def _get_type(expr: ast.expr | None, schemas: List[Dict]) -> Tuple[str, Dict | None]: - if not expr: - return "Any", None - python_type = get_python_type_from_ast(expr) - json_type = _get_jsonschema_type(python_type) - return json_type, _get_type_schema(json_type, python_type, schemas) - - -def _get_req_name_if_not_in_base( - n: Optional[str], pip_name_lookup: Mapping[str, List[str]] -) -> Optional[str]: - if not n: - return None - - if "." in n: - n = n.split(".")[0] - - if n in BASE_REQUIREMENTS: - return None - else: - return pip_name_lookup[n][0] - - -def _parse_code(code: str, function_name: str): - parsed_args = [] - return_type = None - return_type_schema = None - requirements: List[str] = [] - - schemas = _get_schemas(code) - - parsed_code = ast.parse(code) - - # the pip name and the import name might be different - # e.g. kube_hunter is the import name, but the pip name is kube-hunter - # see https://stackoverflow.com/a/75144378 - pip_name_lookup = packages_distributions() - - for node in ast.iter_child_nodes(parsed_code): - if isinstance(node, ast.Import): - # TODO maybe handle `import foo.bar` case? - for name in node.names: - req = _get_req_name_if_not_in_base(name.name, pip_name_lookup) - if req: - requirements.append(req) - elif isinstance(node, ast.ImportFrom): - if node.module: - req = _get_req_name_if_not_in_base(node.module, pip_name_lookup) - if req: - requirements.append(req) - - elif isinstance(node, ast.FunctionDef) and node.name == function_name: - function_args = [arg for arg in node.args.args] - for arg in function_args: - json_type, type_schema = _get_type(arg.annotation, schemas) - json_arg = { - "key": arg.arg, - "name": arg.arg, - "type": json_type, - } - if type_schema: - json_arg["typeSchema"] = json.dumps(type_schema) - parsed_args.append(json_arg) - if node.returns: - return_type, return_type_schema = _get_type(node.returns, schemas) - else: - return_type = "Any" - break - - return parsed_args, return_type, return_type_schema, requirements - - def _func_already_exists(context: str, function_name: str) -> bool: try: module = importlib.import_module(f"polyapi.poly.{context}") @@ -201,57 +17,51 @@ def _func_already_exists(context: str, function_name: str) -> bool: def function_add_or_update( + name: str, + file: str, context: str, description: str, client: bool, server: bool, logs_enabled: bool, - subcommands: List, generate: bool = True, execution_api_key: str = "" ): - parser = argparse.ArgumentParser() - parser.add_argument("subcommand", choices=["add"]) - parser.add_argument("function_name") - parser.add_argument("filename") - args = parser.parse_args(subcommands) - - verb = "Updating" if _func_already_exists(context, args.function_name) else "Adding" + verb = "Updating" if _func_already_exists(context, name) else "Adding" ftype = "server" if server else "client" print(f"{verb} custom {ftype} function...", end="") - with open(args.filename, "r") as f: + with open(file, "r") as f: code = f.read() # OK! let's parse the code and generate the arguments - (arguments, return_type, return_type_schema, requirements) = _parse_code( - code, args.function_name - ) + parsed = parse_function_code(code, name, context) + return_type = parsed["types"]["returns"]["type"] if not return_type: print_red("ERROR") print( - f"Function {args.function_name} not found as top-level function in {args.filename}" + f"Function {name} not found as top-level function in {name}" ) sys.exit(1) data = { - "context": context, - "name": args.function_name, - "description": description, + "context": context or parsed["context"], + "name": name, + "description": description or parsed["types"]["description"], "code": code, "language": "python", "returnType": return_type, - "returnTypeSchema": return_type_schema, - "arguments": arguments, - "logsEnabled": logs_enabled, + "returnTypeSchema": parsed["types"]["returns"]["typeSchema"], + "arguments": [{**p, "type": get_jsonschema_type(p["type"]) } for p in parsed["types"]["params"]], + "logsEnabled": logs_enabled or parsed["config"].get("logs_enabled", False), } - if server and requirements: + if server and parsed["dependencies"]: print_yellow( "\nPlease note that deploying your functions will take a few minutes because it makes use of libraries other than polyapi." ) - data["requirements"] = requirements + data["requirements"] = parsed["dependencies"] api_key, api_url = get_api_key_and_url() assert api_key @@ -286,12 +96,11 @@ def function_add_or_update( sys.exit(1) -def function_execute(context: str, subcommands: List) -> Any: - assert subcommands[0] == "execute" +def function_execute(context: str, name: str, args: List) -> Any: context_code = importlib.import_module(f"polyapi.poly.{context}") - print(f"Executing poly.{context}.{subcommands[1]}... ") - fn = getattr(context_code, subcommands[1]) - return fn(*subcommands[2:]) + print(f"Executing poly.{context}.{name}... ") + fn = getattr(context_code, name) + return fn(*args) def spec_delete(function_type: str, function_id: str): diff --git a/polyapi/generate.py b/polyapi/generate.py index d004c6f..0869c8b 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -13,7 +13,7 @@ from .server import render_server_function from .utils import add_import_to_init, get_auth_headers, init_the_init, to_func_namespace from .variables import generate_variables -from .config import get_api_key_and_url, initialize_config +from .config import get_api_key_and_url SUPPORTED_FUNCTION_TYPES = { "apiFunction", @@ -122,7 +122,6 @@ def remove_old_library(): def generate() -> None: - initialize_config() remove_old_library() diff --git a/polyapi/parser.py b/polyapi/parser.py new file mode 100644 index 0000000..e12f105 --- /dev/null +++ b/polyapi/parser.py @@ -0,0 +1,519 @@ +import ast +import json +import types +import sys +import re +from typing import Dict, List, Mapping, Optional, Tuple, Any +from typing import _TypedDictMeta as BaseTypedDict # type: ignore +from typing_extensions import _TypedDictMeta # type: ignore +from stdlib_list import stdlib_list +from pydantic import TypeAdapter +from importlib.metadata import packages_distributions +from polyapi.constants import PYTHON_TO_JSONSCHEMA_TYPE_MAP +from polyapi.utils import print_red +from polyapi.deployables import Deployment, DeployableRecord, get_deployable_file_revision + + +# these libraries are already installed in the base docker image +# and shouldnt be included in additional requirements +BASE_REQUIREMENTS = { + "polyapi", + "requests", + "typing_extensions", + "jsonschema-gentypes", + "pydantic", + "cloudevents", +} +all_stdlib_symbols = stdlib_list(".".join([str(v) for v in sys.version_info[0:2]])) +BASE_REQUIREMENTS.update( + all_stdlib_symbols +) # dont need to pip install stuff in the python standard library + + +def _parse_sphinx_docstring(docstring: str) -> Dict[str, Any]: + """ + Parses a Sphinx-style docstring to extract parameters, return values, and descriptions. + + :param docstring: Docstring content in reST format. + :type docstring: str + :return: A dictionary with descriptions, parameters, and return values. + :rtype: Dict[str, Any] + """ + lines = docstring.split('\n') + description = [] + params = {} + returns = { + "description": "", + "type": "Any" + } + current_section = None + + for line in lines: + stripped_line = line.strip() + if stripped_line.startswith(":param "): + # Example line: :param x: This is x + param_name, _, param_desc = stripped_line[7:].partition(":") + param_name = param_name.strip() + if param_name in params: + params[param_name]["description"] = param_desc.strip() + else: + params[param_name] = { "name": param_name, "type": "", "description": param_desc.strip() } + current_section = param_name + + elif stripped_line.startswith(":type "): + # Example line: :type x: int + param_name, _, param_type = stripped_line[6:].partition(":") + param_name = param_name.strip() + if param_name in params: + params[param_name]["type"] = param_type.strip() + else: + params[param_name] = { "name": param_name, "type": param_type.strip(), "description": "" } + + elif stripped_line.startswith(":returns: "): + # Example line: :returns: This returns x + return_desc = stripped_line[10:].strip() + returns["description"] = return_desc + current_section = "returns" + + elif stripped_line.startswith(":rtype: "): + # Example line: :rtype: int + return_type = stripped_line[8:].strip() + returns["type"] = return_type + + elif current_section and not stripped_line.startswith(":"): + # Append continued description lines to the last param or return section + if current_section == "returns": + returns["description"] += ' ' + stripped_line + else: + params[current_section]["description"] += " " + stripped_line + + elif not stripped_line.startswith(":"): + # Normal description line + description.append(stripped_line) + + return { + "description": '\n'.join(description).strip(), + "params": list(params.values()), + "returns": returns + } + + +def _parse_google_docstring(docstring: str) -> Dict[str, Any]: + import re + lines = docstring.split('\n') + mode = None + params = {} + parsed = { + 'description': [], + 'params': [], + 'returns': {'description': []}, + 'raises': {} + } + current_key = None + + # Regex to capture the parts of the parameter and the start of type/exception sections + arg_pattern = re.compile(r'^\s*(\w+)\s*(\(.*?\))?:(.*)') + section_pattern = re.compile(r'^\s*(Args|Returns|Raises):') + + for line in lines: + line = line.rstrip() + section_match = section_pattern.match(line) + + if section_match: + mode = section_match.group(1).lower() + continue + + if mode == 'args': + arg_match = arg_pattern.match(line) + if arg_match: + current_key = arg_match.group(1) + type_desc = arg_match.group(2) if arg_match.group(2) else '' + description = arg_match.group(3).strip() + params[current_key] = {'name': current_key, 'type': type_desc.strip('() '), 'description': [description]} + elif current_key: + params[current_key]['description'].append(line.strip()) + + elif mode == 'returns': + if not parsed['returns']['description']: + ret_type, _, desc = line.partition(':') + parsed['returns']['type'] = ret_type.strip() + parsed['returns']['description'].append(desc.strip()) + else: + parsed['returns']['description'].append(line.strip()) + + elif mode == 'raises': + if ':' in line: + exc_type, desc = line.split(':', 1) + parsed['raises'][exc_type.strip()] = desc.strip() + elif current_key: + parsed['raises'][current_key] += ' ' + line.strip() + + elif mode is None: + parsed['description'].append(line.strip()) + + # Consolidate descriptions + parsed['description'] = ' '.join(parsed['description']).strip() + parsed['returns']['description'] = ' '.join(parsed['returns']['description']).strip() + parsed['params'] = [{ **v, 'description': ' '.join(v['description']).strip() } for v in params.values()] + + return parsed + +def _get_schemas(code: str) -> List[Dict]: + schemas = [] + user_code = types.SimpleNamespace() + exec(code, user_code.__dict__) + for name, obj in user_code.__dict__.items(): + if isinstance(obj, BaseTypedDict): + print_red("ERROR") + print_red("\nERROR DETAILS: ") + print( + "It looks like you have used TypedDict in a custom function. Please use `from typing_extensions import TypedDict` instead. The `typing_extensions` version is more powerful and better allows us to provide rich types for your function." + ) + sys.exit(1) + elif ( + isinstance(obj, type) + and isinstance(obj, _TypedDictMeta) + and name != "TypedDict" + ): + schemas.append(TypeAdapter(obj).json_schema()) + return schemas + + +def get_jsonschema_type(python_type: str): + if python_type == "Any": + return "Any" + + if python_type == "List": + return "array" + + if python_type.startswith("List["): + # the actual type will be returned as return_type_schema + subtype = python_type[5:-1] + if subtype == "Any": + return "any[]" + elif subtype in ["int", "float", "str", "bool"]: + jsonschema_type = PYTHON_TO_JSONSCHEMA_TYPE_MAP.get(subtype) + return f"{jsonschema_type}[]" + else: + # the schema will handle it! + return "object" + + if python_type.startswith("Dict"): + return "object" + + rv = PYTHON_TO_JSONSCHEMA_TYPE_MAP.get(python_type) + if rv: + return rv + + # should be custom type + return python_type + + +def get_python_type_from_ast(expr: ast.expr) -> str: + if isinstance(expr, ast.Name): + return str(expr.id) + elif isinstance(expr, ast.Subscript): + assert isinstance(expr, ast.Subscript) + name = getattr(expr.value, "id", "") + if name == "List": + slice = getattr(expr.slice, "id", "Any") + return f"List[{slice}]" + elif name == "Dict": + if expr.slice and isinstance(expr.slice, ast.Tuple): + key = get_python_type_from_ast(expr.slice.dims[0]) + value = get_python_type_from_ast(expr.slice.dims[1]) + return f"Dict[{key}, {value}]" + else: + return "Dict" + return "Any" + else: + return "Any" + + +def _get_type_schema(json_type: str, python_type: str, schemas: List[Dict]): + if python_type.startswith("List["): + subtype = python_type[5:-1] + for schema in schemas: + if schema["title"] == subtype: + return {"type": "array", "items": schema} + + # subtype somehow not in schema, just call it any + return None + else: + for schema in schemas: + if schema["title"] == json_type: + return schema + + +def _get_type(expr: ast.expr | None, schemas: List[Dict]) -> Tuple[str, Dict | None]: + if not expr: + return "any", "Any", None + python_type = get_python_type_from_ast(expr) + json_type = get_jsonschema_type(python_type) + return json_type, python_type, _get_type_schema(json_type, python_type, schemas) + + + +def _get_req_name_if_not_in_base( + n: Optional[str], pip_name_lookup: Mapping[str, List[str]] +) -> Optional[str]: + if not n: + return None + + if "." in n: + n = n.split(".")[0] + + if n in BASE_REQUIREMENTS: + return None + else: + return pip_name_lookup[n][0] + + +def _parse_deploy_comment(comment: str) -> Optional[Deployment]: + # Poly deployed @ 2024-08-29T22:46:46.791Z - test.weeklyReport - https://develop-k8s.polyapi.io/canopy/polyui/collections/server-functions/f0630f95-eac8-4c7d-9d23-639d39034bb6 - e3b0c44 + pattern = r"^\s*(?:#\s*)*Poly deployed @ (\S+) - (\S+)\.([^.]+) - (https?:\/\/[^\/]+)\/\S+\/(\S+)s\/(\S+) - (\S+)$" + match = re.match(pattern, comment) + if not match: + return None + + deployed, context, name, instance, deploy_type, id, file_revision = match.groups() + + # Local development puts canopy on a different port than the poly-server + if instance.endswith("localhost:3000"): + instance = instance.replace(":3000', ':8000") + + return { + "name": name, + "context": context, + "type": deploy_type, + "id": id, + "deployed": deployed, + "fileRevision": file_revision, + "instance": instance + } + + +def _parse_dict(node): + """Recursively parse an ast.Dict node into a Python dictionary.""" + result = {} + for key, value in zip(node.keys, node.values): + parsed_key = _parse_value(key) # Keys can be other expressions too + parsed_value = _parse_value(value) + result[parsed_key] = parsed_value + return result + + +def _parse_value(value): + """Parse a value from different possible AST nodes to Python data.""" + if isinstance(value, ast.Constant): + return value.value # Handles str, int, float, NoneType, etc. + elif isinstance(value, ast.Dict): + return _parse_dict(value) + elif isinstance(value, ast.List): + return [_parse_value(item) for item in value.elts] + elif isinstance(value, ast.Name): + return value.id # Could be a variable reference + else: + return None + + +def parse_function_code(code: str, name: Optional[str] = "", context: Optional[str] = ""): + schemas = _get_schemas(code) + + # the pip name and the import name might be different + # e.g. kube_hunter is the import name, but the pip name is kube-hunter + # see https://stackoverflow.com/a/75144378 + pip_name_lookup = packages_distributions() + + deployable: DeployableRecord = { + "context": context, + "name": name, + "description": "", + "config": {}, + "gitRevision": "", + "fileRevision": "", + "file": "", + "types": { + "description": "", + "params": [], + "returns": { + "type": "", + "description": "", + } + }, + "typeSchemas": {}, + "dependencies": [], + "deployments" : [], + "deploymentCommentRanges": [], + "docStartIndex": -1, + "docEndIndex": -1, + "dirty": False, + } + + class FunctionParserVisitor(ast.NodeVisitor): + """ + Custom visitor so that we can keep track of the global offsets of text so we can easily generate replacements later + """ + + def __init__(self): + self._name = name + self._lines = code.splitlines(keepends=True) # Keep line endings to maintain accurate indexing + self._current_offset = 0 + self._line_offsets = [0] + for i in range(1, len(self._lines)): + self._line_offsets.append( + self._line_offsets[i-1] + len(self._lines[i-1]) + ) + + self._extract_deploy_comments() + + def visit_AnnAssign(self, node): + """Visit an assignment and check if it's defining a polyConfig.""" + self.generic_visit(node) # Continue to visit children first + + if ( + isinstance(node.target, ast.Name) and + node.target.id == "polyConfig" and + isinstance(node.annotation, ast.Name) + ): + # We've found a polyConfig dictionary assignment + if node.annotation.id == "PolyServerFunction": + deployable["type"] = "server-function" + elif node.annotation.id == "PolyClientFunction": + deployable["type"] = "server-function" + else: + print_red("ERROR") + print(f"Unsupported polyConfig type '${node.annotation.id}'") + sys.exit(1) + deployable["config"] = _parse_dict(node.value) + self._name = deployable["config"]["name"] + + def _extract_docstring_from_function(self, node: ast.FunctionDef): + start_lineno = (node.body[0].lineno if node.body else node.lineno) - 1 + start_offset = self._line_offsets[start_lineno] + end_offset = start_offset + deployable["docStartIndex"] = start_offset + deployable["docEndIndex"] = end_offset + + try: + docstring = ast.get_docstring(node) + finally: + # Handle case where there is no doc string + # Also handle case where docstring exists but is empty + if type(docstring) is None or (not docstring and '"""' not in self._lines[start_lineno] and "'''" not in self._lines[start_lineno]): + return None + + + # Support both types of triple quotation marks + pattern = '"""' + str_offset = self._lines[start_lineno].find(pattern) + if str_offset == -1: + pattern = "'''" + str_offset = self._lines[start_lineno].find(pattern) + start_offset += str_offset + # Determine end_offset for multiline or single line doc strings by searching until we hit the end of the opening pattern + # We have to do this manually because the docstring we get from the ast excludes the quotation marks and whitespace + if self._lines[start_lineno].find(pattern, str_offset + 3) == -1: + end_offset = start_offset + for i in range(start_lineno + 1, len(self._lines)): + end_offset = self._line_offsets[i] + str_offset = self._lines[i].find(pattern) + if str_offset >= 0: + end_offset += str_offset + 3 + break + else: + end_offset += len(self._lines[start_lineno]) - 1 + + deployable["docStartIndex"] = start_offset + deployable["docEndIndex"] = end_offset + + # Check if the docstring is likely to be Google Docstring format https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html + if 'Args:' in docstring or 'Returns:' in docstring: + deployable["types"] = _parse_google_docstring(docstring) + else: + deployable["types"]["description"] = docstring.strip() + + def _extract_deploy_comments(self): + for i in range(len(self._lines)): + line = self._lines[i].strip() + if line and not line.startswith("#"): + return + deployment = _parse_deploy_comment(line) + if deployment: + deployable["deployments"].append(deployment) + deployable["deploymentCommentRanges"].append([self._line_offsets[i], len(line)]) + + def visit_Import(self, node: ast.Import): + # TODO maybe handle `import foo.bar` case? + for name in node.names: + req = _get_req_name_if_not_in_base(name.name, pip_name_lookup) + if req: + deployable["dependencies"].append(req) + + def visit_ImportFrom(self, node: ast.ImportFrom): + if node.module: + req = _get_req_name_if_not_in_base(node.module, pip_name_lookup) + if req: + deployable["dependencies"].append(req) + + def visit_FunctionDef(self, node: ast.FunctionDef): + if node.name == self._name: + # Parse docstring which may contain param types and descriptions + self._extract_docstring_from_function(node) + function_args = [arg for arg in node.args.args] + docstring_params = deployable["types"]["params"] + parsed_params = [] + # parse params from actual function and merge in any data from the docstring + for arg in function_args: + _, python_type, type_schema = _get_type(arg.annotation, schemas) + json_arg = { + "name": arg.arg, + "type": python_type, + "description": "", + } + if type_schema: + json_arg["typeSchema"] = json.dumps(type_schema) + + if docstring_params: + type_index = next(i for i, d in enumerate(docstring_params) if d["name"] == arg.arg) + if type_index >= 0: + json_arg["description"] = docstring_params[type_index]["description"] + if docstring_params[type_index]["type"] != python_type: + deployable["dirty"] = True + else: + deployable["dirty"] = True + + parsed_params.append(json_arg) + deployable["types"]["params"] = parsed_params + if node.returns: + _, python_type, return_type_schema = _get_type(node.returns, schemas) + if deployable["types"]["returns"]["type"] != python_type: + deployable["dirty"] = True + deployable["types"]["returns"]["type"] = python_type + deployable["types"]["returns"]["typeSchema"] = return_type_schema + else: + deployable["types"]["returns"]["type"] = "Any" + + def generic_visit(self, node): + if hasattr(node, 'lineno') and hasattr(node, 'col_offset'): + self._current_offset = self._line_offsets[node.lineno - 1] + node.col_offset + super().generic_visit(node) + + tree = ast.parse(code) + visitor = FunctionParserVisitor() + visitor.visit(tree) + + # Setting some top-level config values for convenience + deployable["context"] = context or deployable["config"].get("context", "") + deployable["name"] = name or deployable["config"].get("name", "") + deployable["disableAi"] = deployable["config"].get("disableAi", False) + deployable["description"] = deployable["types"].get("description", "") + if not deployable["name"]: + print_red("ERROR") + print("Function config is missing a name.") + sys.exit(1) + + deployable["fileRevision"] = get_deployable_file_revision(code) + + return deployable + diff --git a/polyapi/prepare.py b/polyapi/prepare.py new file mode 100644 index 0000000..0ef276c --- /dev/null +++ b/polyapi/prepare.py @@ -0,0 +1,135 @@ +import os +import sys +from typing import List, Tuple, Literal +import requests + +from polyapi.parser import parse_function_code +from polyapi.deployables import ( + prepare_deployable_directory, write_cache_revision, + save_deployable_records, get_all_deployable_files, + is_cache_up_to_date, get_git_revision, + write_updated_deployable, DeployableRecord +) + +class FunctionArgumentDto: + def __init__(self, name, type, description=None): + self.name = name + self.type = type + self.description = description + +def get_function_description(deploy_type: Literal["server-function", "client-function"], description: str, arguments, code: str) -> str: + if deploy_type == "server-function": + return get_server_function_description(description, arguments, code) + elif deploy_type == "client-function": + return get_client_function_description(description, arguments, code) + else: + raise ValueError("Unsupported deployable type") + +def get_server_function_description(description: str, arguments, code: str) -> str: + # Simulated API call to generate server function descriptions + data = {"description": description, "arguments": arguments, "code": code} + response = requests.post("http://your-api-url/server-function-description", json=data) + return response.json() + +def get_client_function_description(description: str, arguments, code: str) -> str: + # Simulated API call to generate client function descriptions + data = {"description": description, "arguments": arguments, "code": code} + response = requests.post("http://your-api-url/client-function-description", json=data) + return response.json() + +def fill_in_missing_function_details(deployable: DeployableRecord, code: str) -> DeployableRecord: + is_missing_descriptions = ( + not deployable["types"]["description"] or + not deployable["types"]["returns"]["description"] or + any(not param["description"] for param in deployable["types"]["params"]) + ) + if is_missing_descriptions: + try: + ai_generated = get_function_description( + deployable["type"], + deployable["types"]["description"], + [{"name": p["name"], "type": p["type"], "description": p.get("description")} for p in deployable["types"]["params"]], + code + ) + if not deployable["types"]["description"] and ai_generated.get("description"): + deployable["types"]["description"] = ai_generated["description"] + deployable["dirty"] = True + + deployable["types"]["params"] = [ + {**p, "description": ai_arg["description"]} if ai_arg and ai_arg.get("description") else p + for p, ai_arg in zip(deployable["types"]["params"], ai_generated.get("arguments", [])) + ] + except Exception as e: + print(f"Failed to generate descriptions due to: {str(e)}") + return deployable + +def fill_in_missing_details(deployable: DeployableRecord, code: str) -> DeployableRecord: + if deployable["type"] in ["server-function", "client-function"]: + return fill_in_missing_function_details(deployable, code) + else: + raise ValueError(f'Unsupported deployable type: "{deployable["type"]}"') + + +def get_base_url() -> str: + # Placeholder for getting base URL + return "." + +def get_all_deployables(disable_docs: bool, disable_ai: bool, git_revision: str) -> List[DeployableRecord]: + print("Searching for poly deployables.") + base_url = get_base_url() or "." + possible_deployables = get_all_deployable_files({"includeDirs": [base_url]}) + print(f'Found {len(possible_deployables)} possible deployable file{"s" if len(possible_deployables) != 1 else ""}.') + + found = {} + for possible in possible_deployables: + deployable, code = parse_deployable(possible, base_url, git_revision) + full_name = f'{deployable["context"]}.{deployable["name"]}' + if full_name in found: + print(f'ERROR: Prepared {deployable["type"].replace("-", " ")} {full_name}: DUPLICATE') + else: + if not disable_ai and not deployable.get("disableAi", False): + deployable = fill_in_missing_details(deployable, code) + found[full_name] = deployable + status = "UPDATED" if deployable.get("dirty", False) and not disable_docs else "OK" + print(f'Prepared {deployable["type"].replace("-", " ")} {full_name}: {status}') + + return list(found.values()) + +def parse_deployable(file_path: str, base_url: str, git_revision: str) -> Tuple[DeployableRecord, str]: + # Simulate parsing deployable; adapt with actual logic to parse deployables + # This function should return a tuple of (deployable_dict, code_string) + code = "" + with open(file_path, "r", encoding="utf-8") as file: + code = file.read() + + deployable = parse_function_code(code) + deployable["gitRevision"] = git_revision + deployable["file"] = file_path + return deployable, code + +def prepare_deployables(lazy: bool = False, disable_docs: bool = False, disable_ai: bool = False) -> None: + if lazy and is_cache_up_to_date(): + print("Poly deployments are prepared.") + return + + print("Preparing Poly deployments...") + + prepare_deployable_directory() + git_revision = get_git_revision() + # Parse deployable files + parsed_deployables = get_all_deployables(disable_docs, disable_ai, git_revision) + if not parsed_deployables: + print("No deployable files found. Did you define a `polyConfig` within your deployment?") + return sys.exit(1) + dirty_deployables = [d for d in parsed_deployables if d["dirty"]] + if dirty_deployables: + # Write back deployables files with updated comments + print(f'Fixing {len(dirty_deployables)} deployable file{"" if len(dirty_deployables) == 1 else "s"}.') + # NOTE: write_updated_deployable has side effects that update deployable.fileRevision which is in both this list and parsed_deployables + for deployable in dirty_deployables: + write_updated_deployable(deployable, disable_docs) + + print("Poly deployments are prepared.") + save_deployable_records(parsed_deployables) + write_cache_revision(git_revision) + print("Cached deployables and generated typedefs into mode_modules/.poly/deployables directory.") diff --git a/polyapi/sync.py b/polyapi/sync.py new file mode 100644 index 0000000..2ce252c --- /dev/null +++ b/polyapi/sync.py @@ -0,0 +1,122 @@ +import os +from datetime import datetime +from typing import List, Dict +import requests + +from polyapi.deployables import ( + prepare_deployable_directory, load_deployable_records, + save_deployable_records, remove_deployable_records, + get_cache_deployments_revision, write_updated_deployable, + DeployableRecord, SyncDeployment, Deployment +) + +DEPLOY_ORDER = [ + 'server-function', + 'client-function', +] + +def read_file(file_path: str) -> str: + with open(file_path, 'r', encoding='utf-8') as file: + return file.read() + +def group_by(items: List[Dict], key: str) -> Dict[str, List[Dict]]: + grouped = {} + for item in items: + grouped.setdefault(item[key], []).append(item) + return grouped + +def remove_deployable(deployable: SyncDeployment) -> bool: + # Example function call, adjust as needed + url = f"{deployable['instance']}/{deployable['type']}/{deployable['name']}" + response = requests.get(url) + if response.status_code != 200: + return False + requests.delete(url) + return True + +def sync_deployable_and_get_id(deployable: SyncDeployment, code: str) -> str: + # Example function call, adjust as needed + url = f"{deployable['instance']}/{deployable['type']}" + print(deployable) + payload = { + "context": deployable["context"], + "name": deployable["name"], + "description": deployable["description"], + "code": code, + "typeSchemas": deployable["typeSchemas"], + "config": deployable["config"] + } + response = requests.post(url, json=payload) + response.raise_for_status() + return response.json()['id'] + +def sync_deployable(deployable: SyncDeployment) -> Deployment: + code = read_file(deployable['file']) + id = sync_deployable_and_get_id(deployable, code) + return { + "name": deployable["name"], + "context": deployable["context"], + "instance": deployable["instance"], + "type": deployable["type"], + "id": id, + "deployed": datetime.now().isoformat(), + "fileRevision": deployable["fileRevision"], + } + +def sync_deployables(dry_run: bool, instance: str = os.getenv('POLY_API_BASE_URL')): + prepare_deployable_directory() + git_revision = get_cache_deployments_revision() + all_deployables = load_deployable_records() + to_remove: List[DeployableRecord] = [] + + if not all_deployables: + print('No deployables found. Skipping sync.') + return + + # TODO: Improve our deploy ordering. + # Right now we're doing rudimentary ordering by type + # But this does not safely handle cases where one server function may reference another + # We should parse the functions bodies for references to other Poly deployables and work them into a DAG + grouped_deployables = group_by(all_deployables, 'type') + for type_name in DEPLOY_ORDER: + deployables = grouped_deployables.get(type_name, []) + for deployable in deployables: + previous_deployment = next((d for d in deployable.get('deployments', []) if d['instance'] == instance), None) + git_revision_changed = git_revision != deployable['gitRevision'] + file_revision_changed = not previous_deployment or previous_deployment['fileRevision'] != deployable['fileRevision'] + + action = 'REMOVED' if git_revision_changed else \ + 'ADDED' if not previous_deployment else \ + 'UPDATED' if file_revision_changed else 'OK' + + if not dry_run and (git_revision_changed or file_revision_changed): + # Any deployable may be deployed to multiple instances/environments at the same time + # So we reduce the deployable record down to a single instance we want to deploy to + if previous_deployment: + sync_deployment = { **deployable, **previous_deployment, "instance": instance } + else: + sync_deployment = { **deployable, "instance": instance } + if git_revision == deployable['gitRevision']: + deployment = sync_deployable(sync_deployment) + print(deployment) + if previous_deployment: + previous_deployment.update(deployment) + else: + deployable['deployments'].insert(0, deployment) + else: + found = remove_deployable(sync_deployment) + action = 'NOT FOUND' if not found else action + remove_index = all_deployables.index(deployable) + to_remove.append(all_deployables.pop(remove_index)) + + print(f"{'Would sync' if dry_run else 'Synced'} {deployable['type'].replace('-', ' ')} {deployable['context']}.{deployable['name']}: {'TO BE ' if dry_run else ''}{action}") + + if dry_run: + return + + for deployable in all_deployables: + write_updated_deployable(deployable, True) + + save_deployable_records(all_deployables) + if to_remove: + remove_deployable_records(to_remove) diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index e23113a..929464a 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -1,5 +1,5 @@ -from typing import Any, List, Literal, Dict, TypedDict -from typing_extensions import NotRequired +from typing import Any, List, Literal, Dict, Union, Optional +from typing_extensions import NotRequired, TypedDict class PropertySpecification(TypedDict): @@ -53,4 +53,22 @@ class VariableSpecDto(TypedDict): name: str description: str variable: VariableSpecification - type: Literal['serverVariable'] \ No newline at end of file + type: Literal['serverVariable'] + +Visibility = Union[Literal['PUBLIC'], Literal['TENANT'], Literal['ENVIRONMENT']] + + +class PolyDeployable(TypedDict, total=False): + context: str + name: str + disable_ai: Optional[bool] # Optional field to disable AI + + +class PolyServerFunction(PolyDeployable): + logs_enabled: Optional[bool] + always_on: Optional[bool] + visibility: Optional[Visibility] + +class PolyClientFunction(PolyDeployable): + logs_enabled: Optional[bool] + visibility: Optional[Visibility] diff --git a/pyproject.toml b/pyproject.toml index 6dfc79c..017111b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev1" +version = "0.3.1.dev2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ diff --git a/tests/test_deployables.py b/tests/test_deployables.py new file mode 100644 index 0000000..c5dd9e2 --- /dev/null +++ b/tests/test_deployables.py @@ -0,0 +1,114 @@ +import unittest + +from polyapi.parser import parse_function_code +from polyapi.deployables import update_deployable_function_comments, update_deployment_comments + + +INITIAL_SERVER_FN_DEPLOYMENTS = """ +# Poly deployed @ 2024-11-11T14:43:22.631113 - testing.foobar - https://dev.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar() -> int: + print("Okay then!") + return 7 +""" + +EXPECTED_SERVER_FN_DEPLOYMENTS = '''# Poly deployed @ 2024-11-12T14:43:22.631113 - testing.foobar - https://na1.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd +# Poly deployed @ 2024-11-11T14:43:22.631113 - testing.foobar - https://dev.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar() -> int: + print("Okay then!") + return 7 +''' + +INITIAL_SERVER_FN_DOCSTRINGS = ''' +from typing import Dict +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar(foo: str, bar: Dict[str, str]) -> int: + """A function that does something really import. + """ + print("Okay then!") + return 7 +''' + +EXPECTED_SERVER_FN_DOCSTRINGS = ''' +from typing import Dict +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar(foo: str, bar: Dict[str, str]) -> int: + """A function that does something really import. + + Args: + foo (str): + bar (Dict[str, str]): + + Returns: + int: + """ + print("Okay then!") + return 7 +''' + +class T(unittest.TestCase): + def test_write_deployment_comment(self): + test_deployable = { + "deployments": [ + { + 'context': 'testing', + 'deployed': '2024-11-12T14:43:22.631113', + 'fileRevision': '086aedd', + 'id': 'jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h', + 'instance': 'https://na1.polyapi.io', + 'name': 'foobar', + 'type': 'server-function' + }, + { + 'context': 'testing', + 'deployed': '2024-11-11T14:43:22.631113', + 'fileRevision': '086aedd', + 'id': 'jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h', + 'instance': 'https://dev.polyapi.io', + 'name': 'foobar', + 'type': 'server-function' + } + ], + "deploymentCommentRanges": [[0, 178]] + } + updated_file_contents = update_deployment_comments(INITIAL_SERVER_FN_DEPLOYMENTS, test_deployable) + self.assertEqual(updated_file_contents, EXPECTED_SERVER_FN_DEPLOYMENTS) + + def test_parse_and_write_deployable_docstring(self): + parsed_deployable = parse_function_code(INITIAL_SERVER_FN_DOCSTRINGS) + updated_file_contents = update_deployable_function_comments(INITIAL_SERVER_FN_DOCSTRINGS, parsed_deployable) + self.assertEqual(updated_file_contents, EXPECTED_SERVER_FN_DOCSTRINGS) + + def test_parse_and_overwrite_docstring(self): + parsed_deployable = parse_function_code(EXPECTED_SERVER_FN_DOCSTRINGS) + updated_file_contents = update_deployable_function_comments(EXPECTED_SERVER_FN_DOCSTRINGS, parsed_deployable) + self.assertEqual(EXPECTED_SERVER_FN_DOCSTRINGS, updated_file_contents) \ No newline at end of file diff --git a/tests/test_function_cli.py b/tests/test_function_cli.py deleted file mode 100644 index ac3ccda..0000000 --- a/tests/test_function_cli.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest - -from polyapi.function_cli import _parse_code - - -SIMPLE_CODE = """ -def foobar(n: int) -> int: - return 9 -""" - -COMPLEX_RETURN_TYPE = """ -from typing_extensions import TypedDict - - -class Barbar(TypedDict): - count: int - - -def foobar(n: int) -> Barbar: - return Barbar(count=n) -""" - -LIST_COMPLEX_RETURN_TYPE = """ -from typing import List -from typing_extensions import TypedDict - - -class Barbar(TypedDict): - count: int - - -def foobar(n: int) -> List[Barbar]: - return [Barbar(count=n)] -""" - -COMPLEX_ARG_TYPE = """ -from typing_extensions import TypedDict - - -class Barbar(TypedDict): - count: int - - -def foobar(n: Barbar) -> int: - return 7 -""" - - -class T(unittest.TestCase): - def test_simple_types(self): - args, return_type, return_type_schema, additional_requirements = _parse_code(SIMPLE_CODE, "foobar") - self.assertEqual(len(args), 1) - self.assertEqual(args[0], {"key": "n", "name": "n", "type": "integer"}) - self.assertEqual(return_type, "integer") - self.assertIsNone(return_type_schema) - self.assertEqual(additional_requirements, []) - - def test_complex_return_type(self): - args, return_type, return_type_schema, _ = _parse_code(COMPLEX_RETURN_TYPE, "foobar") - self.assertEqual(len(args), 1) - self.assertEqual(args[0], {"key": "n", "name": "n", "type": "integer"}) - self.assertEqual(return_type, "Barbar") - self.assertEqual(return_type_schema['title'], "Barbar") - - def test_complex_arg_type(self): - args, return_type, return_type_schema, _ = _parse_code(COMPLEX_ARG_TYPE, "foobar") - self.assertEqual(len(args), 1) - self.assertEqual(args[0]["type"], "Barbar") - self.assertEqual(return_type, "integer") - self.assertIsNone(return_type_schema) - - def test_list_complex_return_type(self): - args, return_type, return_type_schema, _ = _parse_code(LIST_COMPLEX_RETURN_TYPE, "foobar") - self.assertEqual(len(args), 1) - self.assertEqual(args[0], {"key": "n", "name": "n", "type": "integer"}) - self.assertEqual(return_type, "object") - self.assertEqual(return_type_schema["items"]['title'], "Barbar") - - def test_parse_import_basic(self): - code = "import flask\n\n\ndef foobar(n: int) -> int:\n return 9\n" - _, _, _, additional_requirements = _parse_code(code, "foobar") - self.assertEqual(additional_requirements, ["Flask"]) - - def test_parse_import_from(self): - code = "from flask import Request, Response\n\n\ndef foobar(n: int) -> int:\n return 9\n" - _, _, _, additional_requirements = _parse_code(code, "foobar") - self.assertEqual(additional_requirements, ["Flask"]) - - def test_parse_import_base(self): - code = "import requests\n\n\ndef foobar(n: int) -> int:\n return 9\n" - _, _, _, additional_requirements = _parse_code(code, "foobar") - self.assertEqual(additional_requirements, []) \ No newline at end of file diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..5be84dd --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,243 @@ +import unittest + +from polyapi.parser import parse_function_code + + +SIMPLE_CODE = """ +def foobar(n: int) -> int: + return 9 +""" + +COMPLEX_RETURN_TYPE = """ +from typing_extensions import TypedDict + + +class Barbar(TypedDict): + count: int + + +def foobar(n: int) -> Barbar: + return Barbar(count=n) +""" + +LIST_COMPLEX_RETURN_TYPE = """ +from typing import List +from typing_extensions import TypedDict + + +class Barbar(TypedDict): + count: int + + +def foobar(n: int) -> List[Barbar]: + return [Barbar(count=n)] +""" + +COMPLEX_ARG_TYPE = """ +from typing_extensions import TypedDict + + +class Barbar(TypedDict): + count: int + + +def foobar(n: Barbar) -> int: + return 7 +""" + +GLIDE_SIMPLE_SERVER_FN = """ +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar() -> int: + print("Okay then!") + return 7 +""" + +GLIDE_DOCSTRING_BAD_SERVER_FN = ''' +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar(foo, bar): + """A function that does something really import. + + Args: + foo (str): The foo in question + bar (Dict[str, str]): Configuration of bars + + Returns: + int: import number please keep handy + """ + print("Okay then!") + return 7 +''' + +GLIDE_DOCSTRING_OK_SERVER_FN = ''' +from typing import Dict +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar(foo: str, bar: Dict[str, str]) -> int: + """A function that does something really import. + + Args: + foo (str): The foo in question + bar (Dict[str, str]): Configuration of bars + + Returns: + int: import number please keep handy + """ + print("Okay then!") + return 7 +''' + +GLIDE_DEPLOYMENTS_SERVER_FN = ''' +# Poly deployed @ 2024-11-12T14:43:22.631113 - testing.foobar - https://na1.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd +# Poly deployed @ 2024-11-11T14:43:22.631113 - testing.foobar - https://dev.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd +from typing import Dict +from polyapi.typedefs import PolyServerFunction + +polyConfig: PolyServerFunction = { + "name": "foobar", + "context": "testing", + "logsEnabled": True, +} + +def foobar(foo: str, bar: Dict[str, str]) -> int: + print("Okay then!") + return 7 +''' + +class T(unittest.TestCase): + def test_simple_types(self): + deployable = parse_function_code(SIMPLE_CODE, "foobar") + types = deployable["types"] + self.assertEqual(len(types["params"]), 1) + self.assertEqual(types["params"][0], {"name": "n", "type": "int", "description": ""}) + self.assertEqual(types["returns"]["type"], "int") + self.assertIsNone(types["returns"]["typeSchema"]) + self.assertEqual(deployable["dependencies"], []) + + def test_complex_return_type(self): + deployable = parse_function_code(COMPLEX_RETURN_TYPE, "foobar") + types = deployable["types"] + self.assertEqual(len(types["params"]), 1) + self.assertEqual(types["params"][0], {"name": "n", "type": "int", "description": ""}) + self.assertEqual(types["returns"]["type"], "Barbar") + self.assertEqual(types["returns"]["typeSchema"]['title'], "Barbar") + + def test_complex_arg_type(self): + deployable = parse_function_code(COMPLEX_ARG_TYPE, "foobar") + types = deployable["types"] + self.assertEqual(len(types["params"]), 1) + self.assertEqual(types["params"][0]["type"], "Barbar") + self.assertEqual(types["returns"]["type"], "int") + self.assertIsNone(types["returns"]["typeSchema"]) + + def test_list_complex_return_type(self): + deployable = parse_function_code(LIST_COMPLEX_RETURN_TYPE, "foobar") + types = deployable["types"] + self.assertEqual(len(types["params"]), 1) + self.assertEqual(types["params"][0], {"name": "n", "type": "int", "description": ""}) + self.assertEqual(types["returns"]["type"], "List[Barbar]") + self.assertEqual(types["returns"]["typeSchema"]["items"]['title'], "Barbar") + + def test_parse_import_basic(self): + code = "import flask\n\n\ndef foobar(n: int) -> int:\n return 9\n" + deployable = parse_function_code(code, "foobar") + self.assertEqual(deployable["dependencies"], ["Flask"]) + + def test_parse_import_from(self): + code = "from flask import Request, Response\n\n\ndef foobar(n: int) -> int:\n return 9\n" + deployable = parse_function_code(code, "foobar") + self.assertEqual(deployable["dependencies"], ["Flask"]) + + def test_parse_import_base(self): + code = "import requests\n\n\ndef foobar(n: int) -> int:\n return 9\n" + deployable = parse_function_code(code, "foobar") + self.assertEqual(deployable["dependencies"], []) + + def test_parse_glide_server_function_no_docstring(self): + code = GLIDE_SIMPLE_SERVER_FN + deployable = parse_function_code(code, "foobar") + self.assertEqual(deployable["name"], "foobar") + self.assertEqual(deployable["context"], "testing") + self.assertEqual(deployable["config"]["logsEnabled"], True) + + def test_parse_glide_server_function_bad_docstring(self): + code = GLIDE_DOCSTRING_BAD_SERVER_FN + deployable = parse_function_code(code, "foobar") + self.assertEqual(deployable["types"]["description"], "A function that does something really import.") + self.assertEqual(deployable["types"]["params"][0], { + "name": "foo", + "type": "Any", + "description": "The foo in question" + }) + self.assertEqual(deployable["types"]["params"][1], { + "name": "bar", + "type": "Any", + "description": "Configuration of bars" + }) + self.assertEqual(deployable["types"]["returns"], { + "type": "Any", + "description": "import number please keep handy" + }) + + def test_parse_glide_server_function_ok_docstring(self): + code = GLIDE_DOCSTRING_OK_SERVER_FN + deployable = parse_function_code(code, "foobar") + self.assertEqual(deployable["types"]["description"], "A function that does something really import.") + self.assertEqual(deployable["types"]["params"][0], { + "name": "foo", + "type": "str", + "description": "The foo in question" + }) + self.assertEqual(deployable["types"]["params"][1], { + "name": "bar", + "type": "Dict[str, str]", + "description": "Configuration of bars" + }) + self.assertEqual(deployable["types"]["returns"], { + "type": "int", + "typeSchema": None, + "description": "import number please keep handy" + }) + + def test_parse_glide_server_function_deploy_receipt(self): + code = GLIDE_DEPLOYMENTS_SERVER_FN + deployable = parse_function_code(code, "foobar") + + self.assertEqual(len(deployable["deployments"]), 2) + self.assertEqual(deployable["deployments"][0], { + 'context': 'testing', + 'deployed': '2024-11-12T14:43:22.631113', + 'fileRevision': '086aedd', + 'id': 'jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h', + 'instance': 'https://na1.polyapi.io', + 'name': 'foobar', + 'type': 'server-function' + }) + self.assertEqual(deployable["deployments"][1], { + 'context': 'testing', + 'deployed': '2024-11-11T14:43:22.631113', + 'fileRevision': '086aedd', + 'id': 'jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h', + 'instance': 'https://dev.polyapi.io', + 'name': 'foobar', + 'type': 'server-function' + }) \ No newline at end of file From cd89d8a104e636a07194ea1cbb083cdf23a73a5f Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:19:54 -0800 Subject: [PATCH 05/72] fix for poly cache directory path construction --- polyapi/deployables.py | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/polyapi/deployables.py b/polyapi/deployables.py index 4d7f4f1..6d51c4f 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -3,13 +3,13 @@ import json import hashlib from pathlib import Path -from typing import TypedDict, List, Dict, Tuple, Optional, Any +from typing import TypedDict, List, Dict, Tuple, Optional, Any, Union from subprocess import check_output, CalledProcessError # Constants -CACHE_VERSION_FILE = "./poly/deployments_revision" -CACHE_DIR = Path("./poly/deployables") +CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployments_revision") +CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployables") class DeployableTypes(str): @@ -85,21 +85,21 @@ def prepare_deployable_directory() -> None: Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) def load_deployable_records() -> List[DeployableRecord]: - return [read_json_file(CACHE_DIR / name) for name in os.listdir(CACHE_DIR) if name.endswith(".json")] + return [read_json_file(os.path.join(CACHE_DIR, name)) for name in os.listdir(CACHE_DIR) if name.endswith(".json")] def save_deployable_records(records: List[DeployableRecord]) -> None: for record in records: - write_json_file(Path(CACHE_DIR) / f'{record["context"]}.{record["name"]}.json', record) + write_json_file(os.path.join(CACHE_DIR, f'{record["context"]}.{record["name"]}.json'), record) def remove_deployable_records(records: List[DeployableRecord]) -> None: for record in records: - os.remove(Path(CACHE_DIR) / f'{record["context"]}.{record["name"]}.json') + os.remove(os.path.join(CACHE_DIR, f'{record["context"]}.{record["name"]}.json')) -def read_json_file(path: Path) -> Any: +def read_json_file(path: Union[str, Path]) -> Any: with open(path, "r", encoding="utf-8") as file: return json.load(file) -def write_json_file(path: Path, contents: Any) -> None: +def write_json_file(path: Union[str, Path], contents: Any) -> None: with open(path, "w", encoding="utf-8") as file: json.dump(contents, file, indent=2) diff --git a/pyproject.toml b/pyproject.toml index 017111b..ad4564a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev2" +version = "0.3.1.dev3" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From a8c2f5294325eb285ee04ecb935c2190f2a5cb47 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:31:35 -0800 Subject: [PATCH 06/72] one more adjustment to the deployables cache directory so there can't be any conflict with any custom namespace --- polyapi/deployables.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/polyapi/deployables.py b/polyapi/deployables.py index 6d51c4f..61441c2 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -8,8 +8,8 @@ # Constants -CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployments_revision") -CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployables") +CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "deployments_revision") +CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "deployables") class DeployableTypes(str): diff --git a/pyproject.toml b/pyproject.toml index ad4564a..a9562b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev3" +version = "0.3.1.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 4a0277a02788a17f023e31be5bad351e1693634f Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:39:18 -0800 Subject: [PATCH 07/72] this better? --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9562b3..69385c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev4" +version = "0.3.1.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ @@ -28,4 +28,4 @@ packages = ["polyapi"] [tools.setuptools.packages.find] include = ["polyapi"] -exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env"] # exclude the generated libraries from builds +exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/deployables*", "polyapi/deployments_revision"] # exclude the generated libraries from builds From 571e85afe368df914bd703686f97ddd81c558428 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:46:10 -0800 Subject: [PATCH 08/72] verbose logging on upload code to see what's failing in CI/CD --- .github/workflows/polyapi-update-python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 9edae77..08e61b6 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -107,5 +107,6 @@ jobs: with: name: python-package-distributions path: dist/ + verbose: true - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From dbbbe61a9da10fa8fde11a0cff7b6803304f39a4 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:48:25 -0800 Subject: [PATCH 09/72] bumpity --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69385c2..a96b1ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev5" +version = "0.3.1.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From c03268ea131ee6ad831f7f7d7991ebb32416bc56 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:54:20 -0800 Subject: [PATCH 10/72] whoops --- .github/workflows/polyapi-update-python-package.yml | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 08e61b6..7bf29e0 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -107,6 +107,7 @@ jobs: with: name: python-package-distributions path: dist/ - verbose: true - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true diff --git a/pyproject.toml b/pyproject.toml index a96b1ab..69385c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev4" +version = "0.3.1.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 64c2e37c201dcc350ea61572145df9a3caafe82f Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 14:58:35 -0800 Subject: [PATCH 11/72] so close --- .github/workflows/polyapi-update-python-package.yml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 7bf29e0..8915d42 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -57,6 +57,8 @@ jobs: path: dist/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true main-build: @@ -109,5 +111,3 @@ jobs: path: dist/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true diff --git a/pyproject.toml b/pyproject.toml index 69385c2..a96b1ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev5" +version = "0.3.1.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 2f71748fa076f63c7d99d9fca051460570effd1e Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 15:05:35 -0800 Subject: [PATCH 12/72] better? --- .github/workflows/polyapi-update-python-package.yml | 2 -- polyapi/deployables.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 8915d42..9edae77 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -57,8 +57,6 @@ jobs: path: dist/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true main-build: diff --git a/polyapi/deployables.py b/polyapi/deployables.py index 61441c2..6d51c4f 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -8,8 +8,8 @@ # Constants -CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "deployments_revision") -CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "deployables") +CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployments_revision") +CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployables") class DeployableTypes(str): From 57a2d40c3b8d727934fdb7da0bbe55210bc879f9 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 15:09:34 -0800 Subject: [PATCH 13/72] okay this should be the fix --- polyapi/deployables.py | 4 ++-- polyapi/prepare.py | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/polyapi/deployables.py b/polyapi/deployables.py index 6d51c4f..64f9947 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -8,8 +8,8 @@ # Constants -CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployments_revision") -CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "poly/deployables") +CACHE_VERSION_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "deployments_revision") +CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cached_deployables") class DeployableTypes(str): diff --git a/polyapi/prepare.py b/polyapi/prepare.py index 0ef276c..880f5f2 100644 --- a/polyapi/prepare.py +++ b/polyapi/prepare.py @@ -132,4 +132,4 @@ def prepare_deployables(lazy: bool = False, disable_docs: bool = False, disable_ print("Poly deployments are prepared.") save_deployable_records(parsed_deployables) write_cache_revision(git_revision) - print("Cached deployables and generated typedefs into mode_modules/.poly/deployables directory.") + print("Cached deployables and generated typedefs into polyapi/cached_deployables directory.") diff --git a/pyproject.toml b/pyproject.toml index a96b1ab..256b813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,4 +28,4 @@ packages = ["polyapi"] [tools.setuptools.packages.find] include = ["polyapi"] -exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/deployables*", "polyapi/deployments_revision"] # exclude the generated libraries from builds +exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/cached_deployables*", "polyapi/deployments_revision"] # exclude the generated libraries from builds From 0e78bd3766472af9c553bd018f8649bbb399487d Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 15:18:46 -0800 Subject: [PATCH 14/72] is it this? --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 256b813..a9562b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,4 +28,4 @@ packages = ["polyapi"] [tools.setuptools.packages.find] include = ["polyapi"] -exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/cached_deployables*", "polyapi/deployments_revision"] # exclude the generated libraries from builds +exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env"] # exclude the generated libraries from builds From 30b2a287d29a79c0d638004650f12b3d5c0dc858 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 15:21:33 -0800 Subject: [PATCH 15/72] maybe --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a9562b3..970a2bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev4" +version = "0.3.1.dev6" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 156d3df55d3c2f9af9b228aabd59711c520b8292 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 15:23:27 -0800 Subject: [PATCH 16/72] oh for the love of pete --- .github/workflows/polyapi-update-python-package.yml | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 9edae77..8915d42 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -57,6 +57,8 @@ jobs: path: dist/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true main-build: diff --git a/pyproject.toml b/pyproject.toml index 970a2bf..a9562b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev6" +version = "0.3.1.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 20d8982fbbcf1045b37b0fd6e9e361e5c8dae394 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 16:15:32 -0800 Subject: [PATCH 17/72] whatever. might be a pypi issue --- .gitignore | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d20cb99..2ab87b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .env .env* +.venv/ +.venv/* .DS_Store # Pip diff --git a/pyproject.toml b/pyproject.toml index a9562b3..256b813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,4 +28,4 @@ packages = ["polyapi"] [tools.setuptools.packages.find] include = ["polyapi"] -exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env"] # exclude the generated libraries from builds +exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/cached_deployables*", "polyapi/deployments_revision"] # exclude the generated libraries from builds From 37b421bc65c354cff60750b6290857ffc3f3e3bf Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Mon, 18 Nov 2024 16:37:35 -0800 Subject: [PATCH 18/72] removing verbose logging --- .github/workflows/polyapi-update-python-package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 8915d42..9edae77 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -57,8 +57,6 @@ jobs: path: dist/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true main-build: From 4cc8a1656261303baa5dac696b614d8d2985c530 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 14:23:43 -0800 Subject: [PATCH 19/72] fixing bugs in sync command to use correct api urls --- polyapi/deployables.py | 3 ++- polyapi/sync.py | 37 +++++++++++++++++++++++++++---------- pyproject.toml | 2 +- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/polyapi/deployables.py b/polyapi/deployables.py index 64f9947..af321e8 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -64,9 +64,10 @@ class SyncDeployment(TypedDict, total=False): context: str name: str description: str - type: str # This should be an enumeration or a predefined set of strings if you have known types. + type: str fileRevision: str file: str + types: DeployableFunctionTypes typeSchemas: Dict[str, any] dependencies: List[str] config: Dict[str, any] diff --git a/polyapi/sync.py b/polyapi/sync.py index 2ce252c..abe27dd 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -3,6 +3,7 @@ from typing import List, Dict import requests +from polyapi.parser import get_jsonschema_type from polyapi.deployables import ( prepare_deployable_directory, load_deployable_records, save_deployable_records, remove_deployable_records, @@ -25,31 +26,43 @@ def group_by(items: List[Dict], key: str) -> Dict[str, List[Dict]]: grouped.setdefault(item[key], []).append(item) return grouped -def remove_deployable(deployable: SyncDeployment) -> bool: - # Example function call, adjust as needed - url = f"{deployable['instance']}/{deployable['type']}/{deployable['name']}" +def remove_deployable_function(deployable: SyncDeployment) -> bool: + url = f"{deployable['instance']}/functions/{deployable["type"].replace("-function", "")}/{deployable['id']}" response = requests.get(url) if response.status_code != 200: return False requests.delete(url) return True -def sync_deployable_and_get_id(deployable: SyncDeployment, code: str) -> str: - # Example function call, adjust as needed - url = f"{deployable['instance']}/{deployable['type']}" - print(deployable) +def remove_deployable(deployable: SyncDeployment) -> bool: + + if deployable["type"] == 'client-function' or deployable["type"] == 'server-function': + return remove_deployable_function(deployable) + + raise Exception(f"Unsupported deployable type '{deployable["type"]}'") + +def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: + url = f"{deployable['instance']}/functions/{deployable["type"].replace("-function", "")}" payload = { "context": deployable["context"], "name": deployable["name"], "description": deployable["description"], "code": code, "typeSchemas": deployable["typeSchemas"], - "config": deployable["config"] + **deployable["config"], + "arguments": [{**p, "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]], } response = requests.post(url, json=payload) response.raise_for_status() return response.json()['id'] +def sync_deployable_and_get_id(deployable: SyncDeployment, code: str) -> str: + + if deployable["type"] == 'client-function' or deployable["type"] == 'server-function': + return sync_function_and_get_id(deployable, code) + + raise Exception(f"Unsupported deployable type '{deployable["type"]}'") + def sync_deployable(deployable: SyncDeployment) -> Deployment: code = read_file(deployable['file']) id = sync_deployable_and_get_id(deployable, code) @@ -93,12 +106,16 @@ def sync_deployables(dry_run: bool, instance: str = os.getenv('POLY_API_BASE_URL # Any deployable may be deployed to multiple instances/environments at the same time # So we reduce the deployable record down to a single instance we want to deploy to if previous_deployment: - sync_deployment = { **deployable, **previous_deployment, "instance": instance } + sync_deployment = { + **deployable, + **previous_deployment, + "description": deployable["types"]["description"], + "instance": instance + } else: sync_deployment = { **deployable, "instance": instance } if git_revision == deployable['gitRevision']: deployment = sync_deployable(sync_deployment) - print(deployment) if previous_deployment: previous_deployment.update(deployment) else: diff --git a/pyproject.toml b/pyproject.toml index 256b813..ed1eea7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev4" +version = "0.3.1.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From aa0be64c6a110f548a25edccf3c8ba8c5e316a24 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 14:30:09 -0800 Subject: [PATCH 20/72] update logging --- polyapi/sync.py | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/polyapi/sync.py b/polyapi/sync.py index abe27dd..9663cee 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -27,7 +27,7 @@ def group_by(items: List[Dict], key: str) -> Dict[str, List[Dict]]: return grouped def remove_deployable_function(deployable: SyncDeployment) -> bool: - url = f"{deployable['instance']}/functions/{deployable["type"].replace("-function", "")}/{deployable['id']}" + url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}/{deployable['id']}' response = requests.get(url) if response.status_code != 200: return False @@ -39,10 +39,10 @@ def remove_deployable(deployable: SyncDeployment) -> bool: if deployable["type"] == 'client-function' or deployable["type"] == 'server-function': return remove_deployable_function(deployable) - raise Exception(f"Unsupported deployable type '{deployable["type"]}'") + raise Exception(f"Unsupported deployable type '{deployable['type']}'") def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: - url = f"{deployable['instance']}/functions/{deployable["type"].replace("-function", "")}" + url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}' payload = { "context": deployable["context"], "name": deployable["name"], @@ -61,7 +61,7 @@ def sync_deployable_and_get_id(deployable: SyncDeployment, code: str) -> str: if deployable["type"] == 'client-function' or deployable["type"] == 'server-function': return sync_function_and_get_id(deployable, code) - raise Exception(f"Unsupported deployable type '{deployable["type"]}'") + raise Exception(f"Unsupported deployable type '{deployable['type']}'") def sync_deployable(deployable: SyncDeployment) -> Deployment: code = read_file(deployable['file']) diff --git a/pyproject.toml b/pyproject.toml index ed1eea7..0ca34fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev5" +version = "0.3.1.dev6" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From db8aec6017c440345b629f4f80ec1f3a5fb301af Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 14:34:20 -0800 Subject: [PATCH 21/72] lint --- polyapi/sync.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/sync.py b/polyapi/sync.py index 9663cee..ae49066 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -27,7 +27,7 @@ def group_by(items: List[Dict], key: str) -> Dict[str, List[Dict]]: return grouped def remove_deployable_function(deployable: SyncDeployment) -> bool: - url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}/{deployable['id']}' + url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}/{deployable["id"]}' response = requests.get(url) if response.status_code != 200: return False diff --git a/pyproject.toml b/pyproject.toml index 0ca34fe..d28c0b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev6" +version = "0.3.1.dev7" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 32a65d1607c64c6916cf80d695b5c8ad0e373b4c Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 14:45:48 -0800 Subject: [PATCH 22/72] improved auth --- polyapi/prepare.py | 11 ++++++++--- polyapi/sync.py | 12 +++++++++--- pyproject.toml | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/polyapi/prepare.py b/polyapi/prepare.py index 880f5f2..0a10972 100644 --- a/polyapi/prepare.py +++ b/polyapi/prepare.py @@ -3,6 +3,8 @@ from typing import List, Tuple, Literal import requests +from polyapi.utils import get_auth_headers +from polyapi.config import get_api_key_and_url from polyapi.parser import parse_function_code from polyapi.deployables import ( prepare_deployable_directory, write_cache_revision, @@ -26,15 +28,18 @@ def get_function_description(deploy_type: Literal["server-function", "client-fun raise ValueError("Unsupported deployable type") def get_server_function_description(description: str, arguments, code: str) -> str: - # Simulated API call to generate server function descriptions + api_key, api_url = get_api_key_and_url() + headers = get_auth_headers(api_key) data = {"description": description, "arguments": arguments, "code": code} - response = requests.post("http://your-api-url/server-function-description", json=data) + response = requests.post(f"{api_url}/server-function-description", headers=headers, json=data) return response.json() def get_client_function_description(description: str, arguments, code: str) -> str: + api_key, api_url = get_api_key_and_url() + headers = get_auth_headers(api_key) # Simulated API call to generate client function descriptions data = {"description": description, "arguments": arguments, "code": code} - response = requests.post("http://your-api-url/client-function-description", json=data) + response = requests.post(f"{api_url}/client-function-description", headers=headers, json=data) return response.json() def fill_in_missing_function_details(deployable: DeployableRecord, code: str) -> DeployableRecord: diff --git a/polyapi/sync.py b/polyapi/sync.py index ae49066..0aa4246 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -3,6 +3,8 @@ from typing import List, Dict import requests +from polyapi.utils import get_auth_headers +from polyapi.config import get_api_key_and_url from polyapi.parser import get_jsonschema_type from polyapi.deployables import ( prepare_deployable_directory, load_deployable_records, @@ -27,11 +29,13 @@ def group_by(items: List[Dict], key: str) -> Dict[str, List[Dict]]: return grouped def remove_deployable_function(deployable: SyncDeployment) -> bool: + api_key, _ = get_api_key_and_url() + headers = get_auth_headers(api_key) url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}/{deployable["id"]}' - response = requests.get(url) + response = requests.get(url, headers=headers) if response.status_code != 200: return False - requests.delete(url) + requests.delete(url, headers) return True def remove_deployable(deployable: SyncDeployment) -> bool: @@ -42,6 +46,8 @@ def remove_deployable(deployable: SyncDeployment) -> bool: raise Exception(f"Unsupported deployable type '{deployable['type']}'") def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: + api_key, _ = get_api_key_and_url() + headers = get_auth_headers(api_key) url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}' payload = { "context": deployable["context"], @@ -52,7 +58,7 @@ def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: **deployable["config"], "arguments": [{**p, "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]], } - response = requests.post(url, json=payload) + response = requests.post(url, headers=headers, json=payload) response.raise_for_status() return response.json()['id'] diff --git a/pyproject.toml b/pyproject.toml index d28c0b6..c7811a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev7" +version = "0.3.1.dev8" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 14cf1ce2e4121f57eaa56c102d7a1fe309638952 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 14:58:42 -0800 Subject: [PATCH 23/72] last fix for function sync --- polyapi/sync.py | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/polyapi/sync.py b/polyapi/sync.py index 0aa4246..011d053 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -54,7 +54,9 @@ def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: "name": deployable["name"], "description": deployable["description"], "code": code, - "typeSchemas": deployable["typeSchemas"], + "language": "python", + "returnType": deployable["types"]["returns"]["type"], + "returnTypeSchema": deployable["types"]["returns"]["typeSchema"], **deployable["config"], "arguments": [{**p, "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]], } diff --git a/pyproject.toml b/pyproject.toml index c7811a2..f7fe3ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev8" +version = "0.3.1.dev9" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 9d4824ab73191ed417003808aa653b371f3b5818 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 15:24:17 -0800 Subject: [PATCH 24/72] fix bug when comment arguments don't align with the function --- polyapi/deployables.py | 2 +- polyapi/parser.py | 13 ++++++++----- polyapi/prepare.py | 10 ++++++---- polyapi/sync.py | 6 +++++- pyproject.toml | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/polyapi/deployables.py b/polyapi/deployables.py index af321e8..9095101 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -245,7 +245,7 @@ def update_deployment_comments(file_content: str, deployable: dict) -> str: if deployable['deployments']: deployment_comments = write_deploy_comments(deployable['deployments']) deployable['deploymentCommentRanges'] = [(0, len(deployment_comments) + 1)] - file_content = f"{deployment_comments}\n{file_content}" + file_content = f"{deployment_comments}{file_content}" return file_content def update_deployable_function_comments(file_content: str, deployable: dict, disable_docs: bool = False) -> str: diff --git a/polyapi/parser.py b/polyapi/parser.py index e12f105..ca73cbc 100644 --- a/polyapi/parser.py +++ b/polyapi/parser.py @@ -475,11 +475,14 @@ def visit_FunctionDef(self, node: ast.FunctionDef): json_arg["typeSchema"] = json.dumps(type_schema) if docstring_params: - type_index = next(i for i, d in enumerate(docstring_params) if d["name"] == arg.arg) - if type_index >= 0: - json_arg["description"] = docstring_params[type_index]["description"] - if docstring_params[type_index]["type"] != python_type: - deployable["dirty"] = True + try: + type_index = next(i for i, d in enumerate(docstring_params) if d["name"] == arg.arg) + if type_index >= 0: + json_arg["description"] = docstring_params[type_index]["description"] + if docstring_params[type_index]["type"] != python_type: + deployable["dirty"] = True + except: + pass else: deployable["dirty"] = True diff --git a/polyapi/prepare.py b/polyapi/prepare.py index 0a10972..af953ec 100644 --- a/polyapi/prepare.py +++ b/polyapi/prepare.py @@ -60,10 +60,12 @@ def fill_in_missing_function_details(deployable: DeployableRecord, code: str) -> deployable["types"]["description"] = ai_generated["description"] deployable["dirty"] = True - deployable["types"]["params"] = [ - {**p, "description": ai_arg["description"]} if ai_arg and ai_arg.get("description") else p - for p, ai_arg in zip(deployable["types"]["params"], ai_generated.get("arguments", [])) - ] + for i, p in enumerate(deployable["types"]["params"]): + ai_params = ai_generated.get("arguments", []) + ai_param = ai_params[i] if ai_params else None + if ai_param and not p.get("description"): + deployable["types"]["params"][i]["description"] = ai_param["description"] + except Exception as e: print(f"Failed to generate descriptions due to: {str(e)}") return deployable diff --git a/polyapi/sync.py b/polyapi/sync.py index 011d053..c4de1c2 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -102,7 +102,11 @@ def sync_deployables(dry_run: bool, instance: str = os.getenv('POLY_API_BASE_URL for type_name in DEPLOY_ORDER: deployables = grouped_deployables.get(type_name, []) for deployable in deployables: - previous_deployment = next((d for d in deployable.get('deployments', []) if d['instance'] == instance), None) + previous_deployment = None + try: + previous_deployment = next((d for d in deployable.get('deployments', []) if d['instance'] == instance), None) + except: + pass git_revision_changed = git_revision != deployable['gitRevision'] file_revision_changed = not previous_deployment or previous_deployment['fileRevision'] != deployable['fileRevision'] diff --git a/pyproject.toml b/pyproject.toml index f7fe3ee..39cd9bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev9" +version = "0.3.1.dev10" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From ffe19f00fb3c1ec518b3713485b77a739c6ab7ba Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 16:28:19 -0800 Subject: [PATCH 25/72] try forcing the poly directory to exist --- polyapi/poly/__init__.py | 0 pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 polyapi/poly/__init__.py diff --git a/polyapi/poly/__init__.py b/polyapi/poly/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index 39cd9bf..fb4b4d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev10" +version = "0.3.1.dev11" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 0d32c4a3fcc2c9d5a6f6e2b48aef0227ae243923 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 16:33:04 -0800 Subject: [PATCH 26/72] test logging --- polyapi/__init__.py | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/polyapi/__init__.py b/polyapi/__init__.py index 2a30c36..8a84ca0 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -11,7 +11,8 @@ if len(sys.argv) > 1 and sys.argv[1] not in CLI_COMMANDS: currdir = os.path.dirname(os.path.abspath(__file__)) if not os.path.isdir(os.path.join(currdir, "poly")): - print("No 'poly' found. Please run 'python3 -m polyapi generate' to generate the 'poly' library for your tenant.") + print(sys.argv) + # print("No 'poly' found. Please run 'python3 -m polyapi generate' to generate the 'poly' library for your tenant.") sys.exit(1) diff --git a/pyproject.toml b/pyproject.toml index fb4b4d2..c3749b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev11" +version = "0.3.1.dev12" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 8bbb2c709cee6453e05629e55970cac7d046f2a8 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Tue, 19 Nov 2024 16:42:10 -0800 Subject: [PATCH 27/72] remove debug logging --- polyapi/__init__.py | 3 +-- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/polyapi/__init__.py b/polyapi/__init__.py index 8a84ca0..2a30c36 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -11,8 +11,7 @@ if len(sys.argv) > 1 and sys.argv[1] not in CLI_COMMANDS: currdir = os.path.dirname(os.path.abspath(__file__)) if not os.path.isdir(os.path.join(currdir, "poly")): - print(sys.argv) - # print("No 'poly' found. Please run 'python3 -m polyapi generate' to generate the 'poly' library for your tenant.") + print("No 'poly' found. Please run 'python3 -m polyapi generate' to generate the 'poly' library for your tenant.") sys.exit(1) diff --git a/pyproject.toml b/pyproject.toml index c3749b5..3f8278d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev12" +version = "0.3.1.dev13" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 2808c974c640ed61b1643e21270d807b2ccbffcf Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Thu, 21 Nov 2024 16:46:33 -0800 Subject: [PATCH 28/72] fixing project glide deployable types and bumping the version --- polyapi/typedefs.py | 14 +++++++------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index 929464a..c8d77cf 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -1,4 +1,4 @@ -from typing import Any, List, Literal, Dict, Union, Optional +from typing import Any, List, Literal, Dict, Union from typing_extensions import NotRequired, TypedDict @@ -61,14 +61,14 @@ class VariableSpecDto(TypedDict): class PolyDeployable(TypedDict, total=False): context: str name: str - disable_ai: Optional[bool] # Optional field to disable AI + disable_ai: NotRequired[bool] # Optional field to disable AI class PolyServerFunction(PolyDeployable): - logs_enabled: Optional[bool] - always_on: Optional[bool] - visibility: Optional[Visibility] + logs_enabled: NotRequired[bool] + always_on: NotRequired[bool] + visibility: NotRequired[Visibility] class PolyClientFunction(PolyDeployable): - logs_enabled: Optional[bool] - visibility: Optional[Visibility] + logs_enabled: NotRequired[bool] + visibility: NotRequired[Visibility] diff --git a/pyproject.toml b/pyproject.toml index 3f8278d..b3e41db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev13" +version = "0.3.1.dev14" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 123f0b7a5eef5d1b9f29d4c0d5a9ea77d7f248fe Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Fri, 22 Nov 2024 13:21:20 -0800 Subject: [PATCH 29/72] fixing missing arguments in python client function upload --- polyapi/cli.py | 2 ++ polyapi/function_cli.py | 2 +- polyapi/sync.py | 2 +- pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 7f28a46..8659475 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -87,6 +87,8 @@ def add_function(args): exit(1) function_add_or_update( + name=args.name, + file=args.file, context=args.context, description=args.description, client=args.client, diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 22c0c6e..1f70fd1 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -53,7 +53,7 @@ def function_add_or_update( "language": "python", "returnType": return_type, "returnTypeSchema": parsed["types"]["returns"]["typeSchema"], - "arguments": [{**p, "type": get_jsonschema_type(p["type"]) } for p in parsed["types"]["params"]], + "arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in parsed["types"]["params"]], "logsEnabled": logs_enabled or parsed["config"].get("logs_enabled", False), } diff --git a/polyapi/sync.py b/polyapi/sync.py index c4de1c2..dcdfb7c 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -58,7 +58,7 @@ def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: "returnType": deployable["types"]["returns"]["type"], "returnTypeSchema": deployable["types"]["returns"]["typeSchema"], **deployable["config"], - "arguments": [{**p, "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]], + "arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]], } response = requests.post(url, headers=headers, json=payload) response.raise_for_status() diff --git a/pyproject.toml b/pyproject.toml index b3e41db..f5a51c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev14" +version = "0.3.1.dev15" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 7dc7afe37560ac980c726a4f0b5123971239ddee Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Fri, 22 Nov 2024 14:56:06 -0800 Subject: [PATCH 30/72] fixing return type for trained functions --- polyapi/function_cli.py | 2 +- polyapi/sync.py | 4 ++-- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 1f70fd1..ccfb1b7 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -51,7 +51,7 @@ def function_add_or_update( "description": description or parsed["types"]["description"], "code": code, "language": "python", - "returnType": return_type, + "returnType": get_jsonschema_type(return_type), "returnTypeSchema": parsed["types"]["returns"]["typeSchema"], "arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in parsed["types"]["params"]], "logsEnabled": logs_enabled or parsed["config"].get("logs_enabled", False), diff --git a/polyapi/sync.py b/polyapi/sync.py index dcdfb7c..a6a8b93 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -50,14 +50,14 @@ def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str: headers = get_auth_headers(api_key) url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}' payload = { + **deployable["config"], "context": deployable["context"], "name": deployable["name"], "description": deployable["description"], "code": code, "language": "python", - "returnType": deployable["types"]["returns"]["type"], + "returnType": get_jsonschema_type(deployable["types"]["returns"]["type"]), "returnTypeSchema": deployable["types"]["returns"]["typeSchema"], - **deployable["config"], "arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]], } response = requests.post(url, headers=headers, json=payload) diff --git a/pyproject.toml b/pyproject.toml index f5a51c9..a77452e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev15" +version = "0.3.1.dev16" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From e117f1ee2838524d1211a8e3fe3d4228b277bdf3 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Fri, 22 Nov 2024 15:19:17 -0800 Subject: [PATCH 31/72] fix bug preventing use of poly sync command locally --- polyapi/sync.py | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/polyapi/sync.py b/polyapi/sync.py index a6a8b93..9dcfac6 100644 --- a/polyapi/sync.py +++ b/polyapi/sync.py @@ -84,7 +84,9 @@ def sync_deployable(deployable: SyncDeployment) -> Deployment: "fileRevision": deployable["fileRevision"], } -def sync_deployables(dry_run: bool, instance: str = os.getenv('POLY_API_BASE_URL')): +def sync_deployables(dry_run: bool, instance: str | None = None): + if not instance: + _, instance = get_api_key_and_url() prepare_deployable_directory() git_revision = get_cache_deployments_revision() all_deployables = load_deployable_records() diff --git a/pyproject.toml b/pyproject.toml index a77452e..9aeae5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev16" +version = "0.3.1.dev17" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From d4a656f93e259d02d9f9154270ff31a67a842ecf Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 26 Nov 2024 11:49:19 -0800 Subject: [PATCH 32/72] next version of client! --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9aeae5b..490b3f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1.dev17" +version = "0.3.1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 7441019f0ae6e163cc40c30f2dc5ec800f5fe6af Mon Sep 17 00:00:00 2001 From: Eric Neumann Date: Thu, 5 Dec 2024 10:48:19 -0800 Subject: [PATCH 33/72] EN #3183 allow null logs flag for python client (#28) --- polyapi/cli.py | 4 ++-- polyapi/function_cli.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 8659475..725d979 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -66,13 +66,13 @@ def generate_command(args): fn_add_parser.add_argument("--description", required=False, default="", help="Description of the function") fn_add_parser.add_argument("--server", action="store_true", help="Marks the function as a server function") fn_add_parser.add_argument("--client", action="store_true", help="Marks the function as a client function") - fn_add_parser.add_argument("--logs", choices=["enabled", "disabled"], default="disabled", help="Enable or disable logs for the function.") + fn_add_parser.add_argument("--logs", choices=["enabled", "disabled"], default=None, help="Enable or disable logs for the function.") fn_add_parser.add_argument("--execution-api-key", required=False, default="", help="API key for execution (for server functions only).") fn_add_parser.add_argument("--disable-ai", "--skip-generate", action="store_true", help="Pass --disable-ai skip AI generation of missing descriptions") def add_function(args): initialize_config() - logs_enabled = args.logs == "enabled" + logs_enabled = args.logs == "enabled" if args.logs else None err = "" if args.server and args.client: err = "Specify either `--server` or `--client`. Found both." diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index ccfb1b7..2348abf 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -1,5 +1,5 @@ import sys -from typing import Any, List +from typing import Any, List, Optional import requests from polyapi.generate import get_functions_and_parse, generate_functions from polyapi.config import get_api_key_and_url @@ -23,7 +23,7 @@ def function_add_or_update( description: str, client: bool, server: bool, - logs_enabled: bool, + logs_enabled: Optional[bool], generate: bool = True, execution_api_key: str = "" ): @@ -45,6 +45,9 @@ def function_add_or_update( ) sys.exit(1) + if logs_enabled is None: + logs_enabled = parsed["config"].get("logs_enabled", None) + data = { "context": context or parsed["context"], "name": name, @@ -54,7 +57,7 @@ def function_add_or_update( "returnType": get_jsonschema_type(return_type), "returnTypeSchema": parsed["types"]["returns"]["typeSchema"], "arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in parsed["types"]["params"]], - "logsEnabled": logs_enabled or parsed["config"].get("logs_enabled", False), + "logsEnabled": logs_enabled, } if server and parsed["dependencies"]: From 57e7369c30717f327cd0637849cefb1958d89cc0 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 6 Dec 2024 11:36:08 -0800 Subject: [PATCH 34/72] let the typing_extensions versions increase to support latest openai pypi package version --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 490b3f8..647b00a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.1" +version = "0.3.2.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ diff --git a/requirements.txt b/requirements.txt index 3e890b0..ee8dbde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests==2.31.0 -typing_extensions==4.10.0 +typing_extensions>=4.10.0 jsonschema-gentypes==2.6.0 pydantic==2.6.4 stdlib_list==0.10.0 From 03fd34cb41ba79edd1a81d48d256325b30664539 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 6 Dec 2024 11:40:09 -0800 Subject: [PATCH 35/72] update dependency in one more place --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 647b00a..ee92e6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,12 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.2.dev0" +version = "0.3.2.dev1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ "requests==2.31.0", - "typing_extensions==4.10.0", + "typing_extensions>=4.10.0", "jsonschema-gentypes==2.6.0", "pydantic==2.6.4", "stdlib_list==0.10.0", From 28a1c39d97a595ebf3d4a825ff06425a85d5a2af Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Fri, 13 Dec 2024 17:02:16 -0800 Subject: [PATCH 36/72] Some bug fixes for python client (#29) * fixed bug with parsing python functions without any types, and bug where functions with multiple deployment receipts were getting mangled * whoops. uncommenting tests * last test fix --- polyapi/deployables.py | 2 +- polyapi/function_cli.py | 1 - polyapi/parser.py | 13 +++++++------ pyproject.toml | 2 +- tests/test_deployables.py | 12 +++++++++++- tests/test_parser.py | 25 ++++++++++++++++++++++--- 6 files changed, 42 insertions(+), 13 deletions(-) diff --git a/polyapi/deployables.py b/polyapi/deployables.py index 9095101..af321e8 100644 --- a/polyapi/deployables.py +++ b/polyapi/deployables.py @@ -245,7 +245,7 @@ def update_deployment_comments(file_content: str, deployable: dict) -> str: if deployable['deployments']: deployment_comments = write_deploy_comments(deployable['deployments']) deployable['deploymentCommentRanges'] = [(0, len(deployment_comments) + 1)] - file_content = f"{deployment_comments}{file_content}" + file_content = f"{deployment_comments}\n{file_content}" return file_content def update_deployable_function_comments(file_content: str, deployable: dict, disable_docs: bool = False) -> str: diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 2348abf..39612dd 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -55,7 +55,6 @@ def function_add_or_update( "code": code, "language": "python", "returnType": get_jsonschema_type(return_type), - "returnTypeSchema": parsed["types"]["returns"]["typeSchema"], "arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in parsed["types"]["params"]], "logsEnabled": logs_enabled, } diff --git a/polyapi/parser.py b/polyapi/parser.py index ca73cbc..4a5b391 100644 --- a/polyapi/parser.py +++ b/polyapi/parser.py @@ -181,7 +181,7 @@ def _get_schemas(code: str) -> List[Dict]: def get_jsonschema_type(python_type: str): if python_type == "Any": - return "Any" + return "any" if python_type == "List": return "array" @@ -338,6 +338,7 @@ def parse_function_code(code: str, name: Optional[str] = "", context: Optional[s "params": [], "returns": { "type": "", + "typeSchema": None, "description": "", } }, @@ -435,13 +436,14 @@ def _extract_docstring_from_function(self, node: ast.FunctionDef): def _extract_deploy_comments(self): for i in range(len(self._lines)): - line = self._lines[i].strip() + line = self._lines[i] if line and not line.startswith("#"): return - deployment = _parse_deploy_comment(line) + deployment = _parse_deploy_comment(line.strip()) if deployment: + start = self._line_offsets[i] deployable["deployments"].append(deployment) - deployable["deploymentCommentRanges"].append([self._line_offsets[i], len(line)]) + deployable["deploymentCommentRanges"].append([start, start + len(line)]) def visit_Import(self, node: ast.Import): # TODO maybe handle `import foo.bar` case? @@ -471,8 +473,7 @@ def visit_FunctionDef(self, node: ast.FunctionDef): "type": python_type, "description": "", } - if type_schema: - json_arg["typeSchema"] = json.dumps(type_schema) + json_arg["typeSchema"] = json.dumps(type_schema) if type_schema else None if docstring_params: try: diff --git a/pyproject.toml b/pyproject.toml index ee92e6f..0a4bc51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.2.dev1" +version = "0.3.2.dev2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ diff --git a/tests/test_deployables.py b/tests/test_deployables.py index c5dd9e2..80ec742 100644 --- a/tests/test_deployables.py +++ b/tests/test_deployables.py @@ -21,6 +21,7 @@ def foobar() -> int: EXPECTED_SERVER_FN_DEPLOYMENTS = '''# Poly deployed @ 2024-11-12T14:43:22.631113 - testing.foobar - https://na1.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd # Poly deployed @ 2024-11-11T14:43:22.631113 - testing.foobar - https://dev.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd + from polyapi.typedefs import PolyServerFunction polyConfig: PolyServerFunction = { @@ -76,6 +77,15 @@ def foobar(foo: str, bar: Dict[str, str]) -> int: ''' class T(unittest.TestCase): + def test_parse_and_write_deployment_comment(self): + test_deployable = parse_function_code(EXPECTED_SERVER_FN_DEPLOYMENTS, "foobar") + deployable_comment_ranges = test_deployable["deploymentCommentRanges"] + updated_file_contents = update_deployment_comments(EXPECTED_SERVER_FN_DEPLOYMENTS, test_deployable) + self.assertEqual(updated_file_contents, EXPECTED_SERVER_FN_DEPLOYMENTS) + # Deployment comment ranges collapsed into one of equal size + self.assertEqual(test_deployable["deploymentCommentRanges"][0][0], deployable_comment_ranges[0][0]) + self.assertEqual(test_deployable["deploymentCommentRanges"][0][1], deployable_comment_ranges[1][1]) + def test_write_deployment_comment(self): test_deployable = { "deployments": [ @@ -98,7 +108,7 @@ def test_write_deployment_comment(self): 'type': 'server-function' } ], - "deploymentCommentRanges": [[0, 178]] + "deploymentCommentRanges": [[0, 177]] } updated_file_contents = update_deployment_comments(INITIAL_SERVER_FN_DEPLOYMENTS, test_deployable) self.assertEqual(updated_file_contents, EXPECTED_SERVER_FN_DEPLOYMENTS) diff --git a/tests/test_parser.py b/tests/test_parser.py index 5be84dd..0119b69 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -3,6 +3,11 @@ from polyapi.parser import parse_function_code +CODE_NO_TYPES = """ +def foobar(a, b): + return a + b +""" + SIMPLE_CODE = """ def foobar(n: int) -> int: return 9 @@ -124,11 +129,21 @@ def foobar(foo: str, bar: Dict[str, str]) -> int: ''' class T(unittest.TestCase): + def test_no_types(self): + deployable = parse_function_code(CODE_NO_TYPES, "foobar") + types = deployable["types"] + self.assertEqual(len(types["params"]), 2) + self.assertEqual(types["params"][0], {"name": "a", "type": "Any", "typeSchema": None, "description": ""}) + self.assertEqual(types["params"][1], {"name": "b", "type": "Any", "typeSchema": None, "description": ""}) + self.assertEqual(types["returns"]["type"], "Any") + self.assertIsNone(types["returns"]["typeSchema"]) + self.assertEqual(deployable["dependencies"], []) + def test_simple_types(self): deployable = parse_function_code(SIMPLE_CODE, "foobar") types = deployable["types"] self.assertEqual(len(types["params"]), 1) - self.assertEqual(types["params"][0], {"name": "n", "type": "int", "description": ""}) + self.assertEqual(types["params"][0], {"name": "n", "type": "int", "typeSchema": None, "description": ""}) self.assertEqual(types["returns"]["type"], "int") self.assertIsNone(types["returns"]["typeSchema"]) self.assertEqual(deployable["dependencies"], []) @@ -137,7 +152,7 @@ def test_complex_return_type(self): deployable = parse_function_code(COMPLEX_RETURN_TYPE, "foobar") types = deployable["types"] self.assertEqual(len(types["params"]), 1) - self.assertEqual(types["params"][0], {"name": "n", "type": "int", "description": ""}) + self.assertEqual(types["params"][0], {"name": "n", "type": "int", "typeSchema": None, "description": ""}) self.assertEqual(types["returns"]["type"], "Barbar") self.assertEqual(types["returns"]["typeSchema"]['title'], "Barbar") @@ -153,7 +168,7 @@ def test_list_complex_return_type(self): deployable = parse_function_code(LIST_COMPLEX_RETURN_TYPE, "foobar") types = deployable["types"] self.assertEqual(len(types["params"]), 1) - self.assertEqual(types["params"][0], {"name": "n", "type": "int", "description": ""}) + self.assertEqual(types["params"][0], {"name": "n", "type": "int", "typeSchema": None, "description": ""}) self.assertEqual(types["returns"]["type"], "List[Barbar]") self.assertEqual(types["returns"]["typeSchema"]["items"]['title'], "Barbar") @@ -186,11 +201,13 @@ def test_parse_glide_server_function_bad_docstring(self): self.assertEqual(deployable["types"]["params"][0], { "name": "foo", "type": "Any", + "typeSchema": None, "description": "The foo in question" }) self.assertEqual(deployable["types"]["params"][1], { "name": "bar", "type": "Any", + "typeSchema": None, "description": "Configuration of bars" }) self.assertEqual(deployable["types"]["returns"], { @@ -205,11 +222,13 @@ def test_parse_glide_server_function_ok_docstring(self): self.assertEqual(deployable["types"]["params"][0], { "name": "foo", "type": "str", + "typeSchema": None, "description": "The foo in question" }) self.assertEqual(deployable["types"]["params"][1], { "name": "bar", "type": "Dict[str, str]", + "typeSchema": None, "description": "Configuration of bars" }) self.assertEqual(deployable["types"]["returns"], { From 4ff9ba93aaaa08c53413dc045c206a25f6643d60 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 23 Dec 2024 08:34:58 -0800 Subject: [PATCH 37/72] 0.3.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0a4bc51..0a5384d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.2.dev2" +version = "0.3.2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From e895ae95884ff55850420b2d0e6182f576d34d5b Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:32:18 -0700 Subject: [PATCH 38/72] add poly schemas support (#31) * onward * adding schemas for Pythonland! * onward * next * next * next * next * test * next * next * next * little tweak for A-Aron * fix * next --- .flake8 | 4 +- .gitignore | 5 +- README.md | 10 ++++ polyapi/api.py | 1 + polyapi/cli.py | 3 +- polyapi/function_cli.py | 7 ++- polyapi/generate.py | 114 +++++++++++++++++++++++++++++++-------- polyapi/poly/__init__.py | 0 polyapi/poly_schemas.py | 61 +++++++++++++++++++++ polyapi/schema.py | 26 ++++++++- polyapi/typedefs.py | 14 +++++ polyapi/utils.py | 29 ++++++---- pyproject.toml | 4 +- requirements.txt | 4 +- tests/test_generate.py | 14 +++++ 15 files changed, 252 insertions(+), 44 deletions(-) delete mode 100644 polyapi/poly/__init__.py create mode 100644 polyapi/poly_schemas.py create mode 100644 tests/test_generate.py diff --git a/.flake8 b/.flake8 index 5d7316b..da87de0 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] -extend-ignore = E203,E303,E402,E501,E722,W391,F401,W292 +ignore = E203,E303,E402,E501,E722,W391,F401,W292,F811 max-line-length = 150 -max-complexity = 20 +max-complexity = 22 diff --git a/.gitignore b/.gitignore index 2ab87b6..49ae8f4 100644 --- a/.gitignore +++ b/.gitignore @@ -34,5 +34,6 @@ __pycache__ .polyapi-python function_add_test.py lib_test*.py -polyapi/poly/ -polyapi/vari/ +polyapi/poly +polyapi/vari +polyapi/schemas diff --git a/README.md b/README.md index 9eea17b..c929f3f 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,16 @@ To run this library's unit tests, please clone the repo then run: python -m unittest discover ``` +## Linting + +The flake8 config is at the root of this repo at `.flake8`. + +When hacking on this library, please enable flake8 and add this line to your flake8 args (e.g., in your VSCode Workspace Settings): + +``` +--config=.flake8 +``` + ## Support If you run into any issues or want help getting started with this project, please contact support@polyapi.io \ No newline at end of file diff --git a/polyapi/api.py b/polyapi/api.py index 8dc4c85..7ddebc5 100644 --- a/polyapi/api.py +++ b/polyapi/api.py @@ -8,6 +8,7 @@ from typing import List, Dict, Any, TypedDict {args_def} {return_type_def} + class {api_response_type}(TypedDict): status: int headers: Dict diff --git a/polyapi/cli.py b/polyapi/cli.py index 836d701..9b6e0cf 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -13,6 +13,7 @@ CLI_COMMANDS = ["setup", "generate", "function", "clear", "help", "update_rendered_spec"] + def execute_from_cli(): # First we setup all our argument parsing logic # Then we parse the arguments (waaay at the bottom) @@ -46,7 +47,7 @@ def setup(args): def generate_command(args): initialize_config() - print("Generating Poly functions...", end="") + print("Generating Poly Python SDK...", end="") generate() print_green("DONE") diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index f7fd677..1256075 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -1,7 +1,7 @@ import sys from typing import Any, List, Optional import requests -from polyapi.generate import get_functions_and_parse, generate_functions +from polyapi.generate import cache_specs, generate_functions, get_specs, parse_function_specs from polyapi.config import get_api_key_and_url from polyapi.utils import get_auth_headers, print_green, print_red, print_yellow from polyapi.parser import parse_function_code, get_jsonschema_type @@ -88,7 +88,10 @@ def function_add_or_update( print(f"Function ID: {function_id}") if generate: print("Generating new custom function...", end="") - functions = get_functions_and_parse(limit_ids=[function_id]) + # TODO do something more efficient here rather than regetting ALL the specs again + specs = get_specs() + cache_specs(specs) + functions = parse_function_specs(specs) generate_functions(functions) print_green("DONE") else: diff --git a/polyapi/generate.py b/polyapi/generate.py index 0869c8b..78d0e07 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -2,13 +2,15 @@ import requests import os import shutil -from typing import List +from typing import List, cast +from polyapi import schema from polyapi.auth import render_auth_function from polyapi.client import render_client_function +from polyapi.poly_schemas import generate_schemas from polyapi.webhook import render_webhook_handle -from .typedefs import PropertySpecification, SpecificationDto, VariableSpecDto +from .typedefs import PropertySpecification, SchemaSpecDto, SpecificationDto, VariableSpecDto from .api import render_api_function from .server import render_server_function from .utils import add_import_to_init, get_auth_headers, init_the_init, to_func_namespace @@ -18,12 +20,21 @@ SUPPORTED_FUNCTION_TYPES = { "apiFunction", "authFunction", - "customFunction", + "customFunction", # client function - this is badly named in /specs atm "serverFunction", "webhookHandle", } -SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable"} +SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable", "schema", "snippet"} + + +X_POLY_REF_WARNING = '''""" +x-poly-ref: + path:''' + +X_POLY_REF_BETTER_WARNING = '''""" +Unresolved schema, please add the following schema to complete it: + path:''' def get_specs() -> List: @@ -38,9 +49,56 @@ def get_specs() -> List: raise NotImplementedError(resp.content) +def build_schema_index(items): + index = {} + for item in items: + if item.get("type") == "schema" and "contextName" in item: + index[item["contextName"]] = {**item.get("definition", {}), "name": item.get("name")} + return index + + +def resolve_poly_refs(obj, schema_index): + if isinstance(obj, dict): + if "x-poly-ref" in obj: + ref = obj["x-poly-ref"] + if isinstance(ref, dict) and "path" in ref: + path = ref["path"] + if path in schema_index: + return resolve_poly_refs(schema_index[path], schema_index) + else: + return obj + return {k: resolve_poly_refs(v, schema_index) for k, v in obj.items()} + elif isinstance(obj, list): + return [resolve_poly_refs(item, schema_index) for item in obj] + else: + return obj + + +def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): + spec_idxs_to_remove = [] + for idx, spec in enumerate(specs): + if spec.get("type") in ("apiFunction", "customFunction", "serverFunction"): + func = spec.get("function") + if func: + try: + spec["function"] = resolve_poly_refs(func, schema_index) + except Exception: + print() + print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") + spec_idxs_to_remove.append(idx) + + # reverse the list so we pop off later indexes first + spec_idxs_to_remove.reverse() + + for idx in spec_idxs_to_remove: + specs.pop(idx) + + return specs + + def parse_function_specs( specs: List[SpecificationDto], - limit_ids: List[str] | None, # optional list of ids to limit to + limit_ids: List[str] | None = None, # optional list of ids to limit to ) -> List[SpecificationDto]: functions = [] for spec in specs: @@ -91,23 +149,14 @@ def read_cached_specs() -> List[SpecificationDto]: return json.loads(f.read()) -def get_functions_and_parse(limit_ids: List[str] | None = None) -> List[SpecificationDto]: - specs = get_specs() - cache_specs(specs) - return parse_function_specs(specs, limit_ids=limit_ids) +def get_variables() -> List[VariableSpecDto]: + specs = read_cached_specs() + return [cast(VariableSpecDto, spec) for spec in specs if spec["type"] == "serverVariable"] -def get_variables() -> List[VariableSpecDto]: - api_key, api_url = get_api_key_and_url() - headers = {"Authorization": f"Bearer {api_key}"} - # TODO do some caching so this and get_functions just do 1 function call - url = f"{api_url}/specs" - resp = requests.get(url, headers=headers) - if resp.status_code == 200: - specs = resp.json() - return [spec for spec in specs if spec["type"] == "serverVariable"] - else: - raise NotImplementedError(resp.content) +def get_schemas() -> List[SchemaSpecDto]: + specs = read_cached_specs() + return [cast(SchemaSpecDto, spec) for spec in specs if spec["type"] == "schema"] def remove_old_library(): @@ -120,12 +169,28 @@ def remove_old_library(): if os.path.exists(path): shutil.rmtree(path) + path = os.path.join(currdir, "schemas") + if os.path.exists(path): + shutil.rmtree(path) + def generate() -> None: remove_old_library() - functions = get_functions_and_parse() + limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug + + specs = get_specs() + cache_specs(specs) + functions = parse_function_specs(specs, limit_ids=limit_ids) + + schemas = get_schemas() + if schemas: + generate_schemas(schemas) + + schema_index = build_schema_index(schemas) + functions = replace_poly_refs_in_functions(functions, schema_index) + if functions: generate_functions(functions) else: @@ -138,6 +203,7 @@ def generate() -> None: if variables: generate_variables(variables) + # indicator to vscode extension that this is a polyapi-python project file_path = os.path.join(os.getcwd(), ".polyapi-python") open(file_path, "w").close() @@ -214,6 +280,12 @@ def render_spec(spec: SpecificationDto): arguments, return_type, ) + + if X_POLY_REF_WARNING in func_type_defs: + # this indicates that jsonschema_gentypes has detected an x-poly-ref + # let's add a more user friendly error explaining what is going on + func_type_defs = func_type_defs.replace(X_POLY_REF_WARNING, X_POLY_REF_BETTER_WARNING) + return func_str, func_type_defs diff --git a/polyapi/poly/__init__.py b/polyapi/poly/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py new file mode 100644 index 0000000..0d0d224 --- /dev/null +++ b/polyapi/poly_schemas.py @@ -0,0 +1,61 @@ +import os +from typing import Any, Dict, List, Tuple + +from polyapi.schema import wrapped_generate_schema_types +from polyapi.utils import add_import_to_init, init_the_init +from tests.test_schema import SCHEMA + +from .typedefs import SchemaSpecDto + +SCHEMA_CODE_IMPORTS = """from typing_extensions import TypedDict, NotRequired + + +""" + + +FALLBACK_SPEC_TEMPLATE = """class {name}(TypedDict, total=False): + ''' unable to generate schema for {name}, defaulting to permissive type ''' + pass +""" + + +def generate_schemas(specs: List[SchemaSpecDto]): + for spec in specs: + create_schema(spec) + + +def create_schema(spec: SchemaSpecDto) -> None: + folders = ["schemas"] + if spec["context"]: + folders += [s for s in spec["context"].split(".")] + + # build up the full_path by adding all the folders + full_path = os.path.join(os.path.dirname(os.path.abspath(__file__))) + + for idx, folder in enumerate(folders): + full_path = os.path.join(full_path, folder) + if not os.path.exists(full_path): + os.makedirs(full_path) + next = folders[idx + 1] if idx + 1 < len(folders) else None + if next: + add_import_to_init(full_path, next, code_imports=SCHEMA_CODE_IMPORTS) + + add_schema_to_init(full_path, spec) + + +def add_schema_to_init(full_path: str, spec: SchemaSpecDto): + init_the_init(full_path, code_imports="") + init_path = os.path.join(full_path, "__init__.py") + with open(init_path, "a") as f: + f.write(render_poly_schema(spec) + "\n\n") + + +def render_poly_schema(spec: SchemaSpecDto) -> str: + definition = spec["definition"] + if not definition.get("type"): + definition["type"] = "object" + root, schema_types = wrapped_generate_schema_types( + definition, root=spec["name"], fallback_type=Dict + ) + return schema_types + # return FALLBACK_SPEC_TEMPLATE.format(name=spec["name"]) diff --git a/polyapi/schema.py b/polyapi/schema.py index 860bbaa..4477745 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -1,11 +1,18 @@ +""" NOTE: this file represents the schema parsing logic for jsonschema_gentypes +""" +import random +import string import logging import contextlib from typing import Dict from jsonschema_gentypes.cli import process_config from jsonschema_gentypes import configuration +import referencing import tempfile import json +import referencing.exceptions + from polyapi.constants import JSONSCHEMA_TO_PYTHON_TYPE_MAP @@ -33,8 +40,18 @@ def _temp_store_input_data(input_data: Dict) -> str: def wrapped_generate_schema_types(type_spec: dict, root, fallback_type): + from polyapi.utils import pascalCase if not root: - root = "MyList" if fallback_type == "List" else "MyDict" + root = "List" if fallback_type == "List" else "Dict" + if type_spec.get("x-poly-ref") and type_spec["x-poly-ref"].get("path"): + # x-poly-ref occurs when we have an unresolved reference + # lets name the root after the reference for some level of visibility + root += pascalCase(type_spec["x-poly-ref"]["path"].replace(".", " ")) + else: + # add three random letters for uniqueness + root += random.choice(string.ascii_letters).upper() + root += random.choice(string.ascii_letters).upper() + root += random.choice(string.ascii_letters).upper() root = clean_title(root) @@ -44,8 +61,13 @@ def wrapped_generate_schema_types(type_spec: dict, root, fallback_type): # some schemas are so huge, our library cant handle it # TODO identify critical recursion penalty and maybe switch underlying logic to iterative? return fallback_type, "" + except referencing.exceptions.CannotDetermineSpecification: + # just go with fallback_type here + # we couldn't match the right $ref earlier in resolve_poly_refs + # {'$ref': '#/definitions/FinanceAccountListModel'} + return fallback_type, "" except: - logging.exception(f"Error when generating schema type: {type_spec}") + logging.error(f"Error when generating schema type: {type_spec}\nusing fallback type '{fallback_type}'") return fallback_type, "" diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index c8d77cf..3a6d84a 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -55,6 +55,19 @@ class VariableSpecDto(TypedDict): variable: VariableSpecification type: Literal['serverVariable'] + +class SchemaSpecDto(TypedDict): + id: str + context: str + name: str + contextName: str + type: Literal['schema'] + definition: Dict[Any, Any] + visibilityMetadata: object + unresolvedPolySchemaRefs: List + # TODO add more + + Visibility = Union[Literal['PUBLIC'], Literal['TENANT'], Literal['ENVIRONMENT']] @@ -69,6 +82,7 @@ class PolyServerFunction(PolyDeployable): always_on: NotRequired[bool] visibility: NotRequired[Visibility] + class PolyClientFunction(PolyDeployable): logs_enabled: NotRequired[bool] visibility: NotRequired[Visibility] diff --git a/polyapi/utils.py b/polyapi/utils.py index a5141a6..06dc0fc 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -11,19 +11,19 @@ # this string should be in every __init__ file. # it contains all the imports needed for the function or variable code to run -CODE_IMPORTS = "from typing import List, Dict, Any, TypedDict, Optional, Callable\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" -FALLBACK_TYPES = {"Dict", "List"} +CODE_IMPORTS = "from typing import List, Dict, Any, Optional, Callable\nfrom typing_extensions import TypedDict, NotRequired\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" -def init_the_init(full_path: str) -> None: +def init_the_init(full_path: str, code_imports="") -> None: init_path = os.path.join(full_path, "__init__.py") if not os.path.exists(init_path): + code_imports = code_imports or CODE_IMPORTS with open(init_path, "w") as f: - f.write(CODE_IMPORTS) + f.write(code_imports) -def add_import_to_init(full_path: str, next: str) -> None: - init_the_init(full_path) +def add_import_to_init(full_path: str, next: str, code_imports="") -> None: + init_the_init(full_path, code_imports=code_imports) init_path = os.path.join(full_path, "__init__.py") with open(init_path, "a+") as f: @@ -38,7 +38,7 @@ def get_auth_headers(api_key: str): return {"Authorization": f"Bearer {api_key}"} -def camelCase(s): +def camelCase(s: str) -> str: s = s.strip() if " " in s or "-" in s: s = re.sub(r"(_|-)+", " ", s).title().replace(" ", "") @@ -48,6 +48,10 @@ def camelCase(s): return s +def pascalCase(s) -> str: + return re.sub(r"(^|_)([a-z])", lambda match: match.group(2).upper(), s) + + def print_green(s: str): print(Fore.GREEN + s + Style.RESET_ALL) @@ -111,15 +115,20 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: elif type_spec["kind"] == "object": if type_spec.get("schema"): schema = type_spec["schema"] - title = schema.get("title", "") + title = schema.get("title", schema.get("name", "")) if title: assert isinstance(title, str) return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore - + elif schema.get("allOf") and len(schema['allOf']): + # we are in a case of a single allOf, lets strip off the allOf and move on! + # our library doesn't handle allOf well yet + allOf = schema['allOf'][0] + title = allOf.get("title", allOf.get("name", "")) + return wrapped_generate_schema_types(allOf, title, "Dict") elif schema.get("items"): # fallback to schema $ref name if no explicit title items = schema.get("items") # type: ignore - title = items.get("title", "") # type: ignore + title = items.get("title") # type: ignore if not title: # title is actually a reference to another schema title = items.get("$ref", "") # type: ignore diff --git a/pyproject.toml b/pyproject.toml index 0a5384d..3e3549d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,11 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.2" +version = "0.3.3.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ - "requests==2.31.0", + "requests>=2.32.3", "typing_extensions>=4.10.0", "jsonschema-gentypes==2.6.0", "pydantic==2.6.4", diff --git a/requirements.txt b/requirements.txt index ee8dbde..d967c75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -requests==2.31.0 +requests>=2.32.3 typing_extensions>=4.10.0 -jsonschema-gentypes==2.6.0 +jsonschema-gentypes==2.10.0 pydantic==2.6.4 stdlib_list==0.10.0 colorama==0.4.4 diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..ecc7d05 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,14 @@ +import unittest +from polyapi.utils import get_type_and_def, rewrite_reserved + +OPENAPI_FUNCTION = {'kind': 'function', 'spec': {'arguments': [{'name': 'event', 'required': False, 'type': {'kind': 'object', 'schema': {'$schema': 'http://json-schema.org/draft-06/schema#', 'type': 'array', 'items': {'$ref': '#/definitions/WebhookEventTypeElement'}, 'definitions': {'WebhookEventTypeElement': {'type': 'object', 'additionalProperties': False, 'properties': {'title': {'type': 'string'}, 'manufacturerName': {'type': 'string'}, 'carType': {'type': 'string'}, 'id': {'type': 'integer'}}, 'required': ['carType', 'id', 'manufacturerName', 'title'], 'title': 'WebhookEventTypeElement'}}}}}, {'name': 'headers', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'params', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'polyCustom', 'required': False, 'type': {'kind': 'object', 'properties': [{'name': 'responseStatusCode', 'type': {'type': 'number', 'kind': 'primitive'}, 'required': True}, {'name': 'responseContentType', 'type': {'type': 'string', 'kind': 'primitive'}, 'required': True, 'nullable': True}]}}], 'returnType': {'kind': 'void'}, 'synchronous': True}} + + +class T(unittest.TestCase): + def test_get_type_and_def(self): + arg_type, arg_def = get_type_and_def(OPENAPI_FUNCTION) + self.assertEqual(arg_type, "Callable[[List[WebhookEventTypeElement], Dict, Dict, Dict], None]") + + def test_rewrite_reserved(self): + rv = rewrite_reserved("from") + self.assertEqual(rv, "_from") \ No newline at end of file From d189377826bc17223e3d582147ee9696629e5976 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:44:32 -0700 Subject: [PATCH 39/72] update to v4 --- .../polyapi-update-python-package.yml | 20 +++++++++---------- pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index 9edae77..ba0bb97 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -1,4 +1,4 @@ -name: Update python pip package +name: Update python pip package on: push: paths: @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.ref == 'refs/heads/develop' }} environment: dev - + steps: - uses: actions/checkout@v4 - name: Set up Python @@ -31,7 +31,7 @@ jobs: - name: Build a binary wheel and a source tarball run: python3 -m build - name: Store the distribution packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-package-distributions path: dist/ @@ -44,10 +44,10 @@ jobs: needs: develop-build environment: name: dev - url: https://pypi.org/p/polyapi-python + url: https://pypi.org/p/polyapi-python permissions: - id-token: write + id-token: write steps: - name: Download all the dists @@ -58,13 +58,13 @@ jobs: - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - + main-build: name: Build distribution 📦 runs-on: ubuntu-latest if: ${{ github.ref == 'refs/heads/main' }} environment: main - + steps: - uses: actions/checkout@v4 - name: Set up Python @@ -90,15 +90,15 @@ jobs: main-publish-to-pypi: name: >- Publish Python 🐍 distribution 📦 to PyPI - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} needs: - main-build runs-on: ubuntu-latest environment: name: main - url: https://pypi.org/p/polyapi-python + url: https://pypi.org/p/polyapi-python permissions: - id-token: write + id-token: write steps: diff --git a/pyproject.toml b/pyproject.toml index 3e3549d..17e007e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ description = "The Python Client for PolyAPI, the IPaaS by Developers for Develo authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ "requests>=2.32.3", - "typing_extensions>=4.10.0", + "typing_extensions>=4.12.2", "jsonschema-gentypes==2.6.0", "pydantic==2.6.4", "stdlib_list==0.10.0", From a140802b5a5335aca3eaf3e677457d0c87f38e94 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:45:39 -0700 Subject: [PATCH 40/72] v4 everywhere --- .github/workflows/polyapi-update-python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/polyapi-update-python-package.yml b/.github/workflows/polyapi-update-python-package.yml index ba0bb97..1612bb8 100644 --- a/.github/workflows/polyapi-update-python-package.yml +++ b/.github/workflows/polyapi-update-python-package.yml @@ -51,7 +51,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-package-distributions path: dist/ @@ -81,7 +81,7 @@ jobs: - name: Build a binary wheel and a source tarball run: python3 -m build - name: Store the distribution packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-package-distributions path: dist/ @@ -103,7 +103,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-package-distributions path: dist/ From 96f63c37f34e2a60cb2cf06158db41a1be675f70 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:47:40 -0700 Subject: [PATCH 41/72] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 17e007e..8d1fb3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev0" +version = "0.3.3.dev1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 15eab87054615d7d89a2632db7fc03a1b4e893ad Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 11:00:22 -0700 Subject: [PATCH 42/72] add new version --- polyapi/poly_schemas.py | 1 - pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index 0d0d224..3c69c5b 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -3,7 +3,6 @@ from polyapi.schema import wrapped_generate_schema_types from polyapi.utils import add_import_to_init, init_the_init -from tests.test_schema import SCHEMA from .typedefs import SchemaSpecDto diff --git a/pyproject.toml b/pyproject.toml index 8d1fb3a..5236ad5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev1" +version = "0.3.3.dev2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 3ad26074dcf7368592c95f67fb2babc3461a81c3 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 26 Mar 2025 10:14:32 -0700 Subject: [PATCH 43/72] remove warning, just go with any type for now --- polyapi/generate.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 78d0e07..55a924b 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -83,8 +83,8 @@ def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): try: spec["function"] = resolve_poly_refs(func, schema_index) except Exception: - print() - print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") + # print() + # print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") spec_idxs_to_remove.append(idx) # reverse the list so we pop off later indexes first diff --git a/pyproject.toml b/pyproject.toml index 5236ad5..71c3b30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev2" +version = "0.3.3.dev3" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 1479d39af2c17a285d04905ba41a6774dbc7f44f Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 26 Mar 2025 12:14:13 -0700 Subject: [PATCH 44/72] better generate printed messages, fix generate bug after function add --- polyapi/cli.py | 2 -- polyapi/function_cli.py | 10 ++-------- polyapi/generate.py | 15 ++++++++------- pyproject.toml | 2 +- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 9b6e0cf..67866d7 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -47,9 +47,7 @@ def setup(args): def generate_command(args): initialize_config() - print("Generating Poly Python SDK...", end="") generate() - print_green("DONE") generate_parser.set_defaults(command=generate_command) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 1256075..bc836fa 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -1,7 +1,7 @@ import sys from typing import Any, List, Optional import requests -from polyapi.generate import cache_specs, generate_functions, get_specs, parse_function_specs +from polyapi.generate import generate as generate_library from polyapi.config import get_api_key_and_url from polyapi.utils import get_auth_headers, print_green, print_red, print_yellow from polyapi.parser import parse_function_code, get_jsonschema_type @@ -87,13 +87,7 @@ def function_add_or_update( function_id = resp.json()["id"] print(f"Function ID: {function_id}") if generate: - print("Generating new custom function...", end="") - # TODO do something more efficient here rather than regetting ALL the specs again - specs = get_specs() - cache_specs(specs) - functions = parse_function_specs(specs) - generate_functions(functions) - print_green("DONE") + generate_library() else: print("Error adding function.") print(resp.status_code) diff --git a/polyapi/generate.py b/polyapi/generate.py index 55a924b..278dba2 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -4,16 +4,15 @@ import shutil from typing import List, cast -from polyapi import schema -from polyapi.auth import render_auth_function -from polyapi.client import render_client_function -from polyapi.poly_schemas import generate_schemas -from polyapi.webhook import render_webhook_handle +from .auth import render_auth_function +from .client import render_client_function +from .poly_schemas import generate_schemas +from .webhook import render_webhook_handle from .typedefs import PropertySpecification, SchemaSpecDto, SpecificationDto, VariableSpecDto from .api import render_api_function from .server import render_server_function -from .utils import add_import_to_init, get_auth_headers, init_the_init, to_func_namespace +from .utils import add_import_to_init, get_auth_headers, init_the_init, print_green, to_func_namespace from .variables import generate_variables from .config import get_api_key_and_url @@ -175,7 +174,7 @@ def remove_old_library(): def generate() -> None: - + print("Generating Poly Python SDK...", end="", flush=True) remove_old_library() limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug @@ -208,6 +207,8 @@ def generate() -> None: file_path = os.path.join(os.getcwd(), ".polyapi-python") open(file_path, "w").close() + print_green("DONE") + def clear() -> None: base = os.path.dirname(os.path.abspath(__file__)) diff --git a/pyproject.toml b/pyproject.toml index 71c3b30..69f179a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev3" +version = "0.3.3.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 7db1cbe3070727117ac4867ef5853035141eb50c Mon Sep 17 00:00:00 2001 From: Don Chiniquy Date: Thu, 27 Mar 2025 09:01:45 -0700 Subject: [PATCH 45/72] Update python version (#32) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69f179a..b356b69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev4" +version = "0.3.3" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From a3f09cdcae3cabf47c43b8718c4a9b7add5686fd Mon Sep 17 00:00:00 2001 From: Don Chiniquy Date: Thu, 27 Mar 2025 09:11:18 -0700 Subject: [PATCH 46/72] Update python image (#33) * Update python version * Updated version From 19fafdeff54c567acf8af88189da8417150cf409 Mon Sep 17 00:00:00 2001 From: nahuel-polyapi Date: Thu, 27 Mar 2025 13:12:40 -0300 Subject: [PATCH 47/72] Rollback version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b356b69..69f179a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3" +version = "0.3.3.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From fccbf1e85aef5057dc28cf789794dd8af2cbb908 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 8 Apr 2025 09:13:15 -0700 Subject: [PATCH 48/72] onward (#34) * woot! we have some better return types * toward working tests - except parser/deployables * more * getting there * next * onawrd * next * next * next * next --- README.md | 10 +++++ check_mypy.sh | 5 +++ polyapi/api.py | 1 + polyapi/generate.py | 8 ++-- polyapi/parser.py | 1 + polyapi/poly_schemas.py | 64 ++++++++++++++++++++++------- polyapi/schema.py | 23 ++++++++--- polyapi/server.py | 14 ++++--- polyapi/utils.py | 85 +++++++++++++++++++++++++++++++-------- pyproject.toml | 6 ++- tests/test_deployables.py | 7 ++-- tests/test_generate.py | 73 ++++++++++++++++++++++++++++++++- tests/test_parser.py | 4 ++ tests/test_schema.py | 8 +++- tests/test_server.py | 57 +++++++++++++++++++++++++- tests/test_utils.py | 78 +++++++++++++++++++++++++++++++++-- 16 files changed, 386 insertions(+), 58 deletions(-) create mode 100755 check_mypy.sh diff --git a/README.md b/README.md index c929f3f..56e3ba1 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,16 @@ When hacking on this library, please enable flake8 and add this line to your fla --config=.flake8 ``` +## Mypy Type Improvements + +This script is handy for checking for any mypy types: + +```bash +./check_mypy.sh +``` + +Please ignore \[name-defined\] errors for now. This is a known bug we are working to fix! + ## Support If you run into any issues or want help getting started with this project, please contact support@polyapi.io \ No newline at end of file diff --git a/check_mypy.sh b/check_mypy.sh new file mode 100755 index 0000000..33ec2d3 --- /dev/null +++ b/check_mypy.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +mypy polyapi/poly +mypy polyapi/vari +mypy polyapi/schemas \ No newline at end of file diff --git a/polyapi/api.py b/polyapi/api.py index 7ddebc5..9533b21 100644 --- a/polyapi/api.py +++ b/polyapi/api.py @@ -42,6 +42,7 @@ def render_api_function( arg_names = [a["name"] for a in arguments] args, args_def = parse_arguments(function_name, arguments) return_type_name, return_type_def = get_type_and_def(return_type) # type: ignore + data = "{" + ", ".join([f"'{arg}': {rewrite_arg_name(arg)}" for arg in arg_names]) + "}" api_response_type = f"{function_name}Response" diff --git a/polyapi/generate.py b/polyapi/generate.py index 278dba2..fc605df 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -2,7 +2,7 @@ import requests import os import shutil -from typing import List, cast +from typing import List, Tuple, cast from .auth import render_auth_function from .client import render_client_function @@ -177,10 +177,10 @@ def generate() -> None: print("Generating Poly Python SDK...", end="", flush=True) remove_old_library() - limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug - specs = get_specs() cache_specs(specs) + + limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug functions = parse_function_specs(specs, limit_ids=limit_ids) schemas = get_schemas() @@ -222,7 +222,7 @@ def clear() -> None: print("Cleared!") -def render_spec(spec: SpecificationDto): +def render_spec(spec: SpecificationDto) -> Tuple[str, str]: function_type = spec["type"] function_description = spec["description"] function_name = spec["name"] diff --git a/polyapi/parser.py b/polyapi/parser.py index 736e454..c89d274 100644 --- a/polyapi/parser.py +++ b/polyapi/parser.py @@ -158,6 +158,7 @@ def _parse_google_docstring(docstring: str) -> Dict[str, Any]: return parsed + def _get_schemas(code: str) -> List[Dict]: schemas = [] user_code = types.SimpleNamespace() diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index 3c69c5b..6528341 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Tuple from polyapi.schema import wrapped_generate_schema_types -from polyapi.utils import add_import_to_init, init_the_init +from polyapi.utils import add_import_to_init, init_the_init, to_func_namespace from .typedefs import SchemaSpecDto @@ -23,23 +23,57 @@ def generate_schemas(specs: List[SchemaSpecDto]): create_schema(spec) -def create_schema(spec: SchemaSpecDto) -> None: - folders = ["schemas"] - if spec["context"]: - folders += [s for s in spec["context"].split(".")] +def add_schema_file( + full_path: str, + schema_name: str, + spec: SchemaSpecDto, +): + # first lets add the import to the __init__ + init_the_init(full_path, SCHEMA_CODE_IMPORTS) - # build up the full_path by adding all the folders - full_path = os.path.join(os.path.dirname(os.path.abspath(__file__))) + if not spec["definition"].get("title"): + # very empty schemas like mews.Unit are possible + # add a title here to be sure they render + spec["definition"]["title"] = schema_name + schema_defs = render_poly_schema(spec) + + if schema_defs: + # add function to init + init_path = os.path.join(full_path, "__init__.py") + with open(init_path, "a") as f: + f.write(f"\n\nfrom ._{to_func_namespace(schema_name)} import {schema_name}") + + # add type_defs to underscore file + file_path = os.path.join(full_path, f"_{to_func_namespace(schema_name)}.py") + with open(file_path, "w") as f: + f.write(schema_defs) + + +def create_schema( + spec: SchemaSpecDto +) -> None: + full_path = os.path.dirname(os.path.abspath(__file__)) + folders = f"schemas.{spec['context']}.{spec['name']}".split(".") for idx, folder in enumerate(folders): - full_path = os.path.join(full_path, folder) - if not os.path.exists(full_path): - os.makedirs(full_path) - next = folders[idx + 1] if idx + 1 < len(folders) else None - if next: - add_import_to_init(full_path, next, code_imports=SCHEMA_CODE_IMPORTS) - - add_schema_to_init(full_path, spec) + if idx + 1 == len(folders): + # special handling for final level + add_schema_file( + full_path, + folder, + spec, + ) + else: + full_path = os.path.join(full_path, folder) + if not os.path.exists(full_path): + os.makedirs(full_path) + + # append to __init__.py file if nested folders + next = folders[idx + 1] if idx + 2 < len(folders) else "" + if next: + init_the_init(full_path, SCHEMA_CODE_IMPORTS) + add_import_to_init(full_path, next) + def add_schema_to_init(full_path: str, spec: SchemaSpecDto): diff --git a/polyapi/schema.py b/polyapi/schema.py index 4477745..1dccd22 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -1,9 +1,8 @@ """ NOTE: this file represents the schema parsing logic for jsonschema_gentypes """ -import random -import string import logging import contextlib +import re from typing import Dict from jsonschema_gentypes.cli import process_config from jsonschema_gentypes import configuration @@ -48,10 +47,8 @@ def wrapped_generate_schema_types(type_spec: dict, root, fallback_type): # lets name the root after the reference for some level of visibility root += pascalCase(type_spec["x-poly-ref"]["path"].replace(".", " ")) else: - # add three random letters for uniqueness - root += random.choice(string.ascii_letters).upper() - root += random.choice(string.ascii_letters).upper() - root += random.choice(string.ascii_letters).upper() + # if we have no root, just add "My" + root = "My" + root root = clean_title(root) @@ -99,9 +96,23 @@ def generate_schema_types(input_data: Dict, root=None): with open(tmp_output) as f: output = f.read() + output = clean_malformed_examples(output) + return output +# Regex to match everything between "# example: {\n" and "^}$" +MALFORMED_EXAMPLES_PATTERN = re.compile(r"# example: \{\n.*?^\}$", flags=re.DOTALL | re.MULTILINE) + + +def clean_malformed_examples(example: str) -> str: + """ there is a bug in the `jsonschmea_gentypes` library where if an example from a jsonchema is an object, + it will break the code because the object won't be properly commented out + """ + cleaned_example = MALFORMED_EXAMPLES_PATTERN.sub("", example) + return cleaned_example + + def clean_title(title: str) -> str: """ used by library generation, sometimes functions can be added with spaces in the title or other nonsense. fix them! diff --git a/polyapi/server.py b/polyapi/server.py index 61e4658..53b173e 100644 --- a/polyapi/server.py +++ b/polyapi/server.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, cast -from polyapi.typedefs import PropertySpecification -from polyapi.utils import add_type_import_path, parse_arguments, get_type_and_def, rewrite_arg_name +from polyapi.typedefs import PropertySpecification, PropertyType +from polyapi.utils import add_type_import_path, parse_arguments, get_type_and_def, return_type_already_defined_in_args, rewrite_arg_name SERVER_DEFS_TEMPLATE = """ from typing import List, Dict, Any, TypedDict, Callable @@ -21,7 +21,7 @@ def {function_name}( try: return {return_action} except: - return resp.text + return resp.text # type: ignore # fallback for debugging """ @@ -37,7 +37,11 @@ def render_server_function( ) -> Tuple[str, str]: arg_names = [a["name"] for a in arguments] args, args_def = parse_arguments(function_name, arguments) - return_type_name, return_type_def = get_type_and_def(return_type) # type: ignore + return_type_name, return_type_def = get_type_and_def(cast(PropertyType, return_type), "ReturnType") + + if return_type_def and return_type_already_defined_in_args(return_type_name, args_def): + return_type_def = "" + data = "{" + ", ".join([f"'{arg}': {rewrite_arg_name(arg)}" for arg in arg_names]) + "}" func_type_defs = SERVER_DEFS_TEMPLATE.format( args_def=args_def, diff --git a/polyapi/utils.py b/polyapi/utils.py index 06dc0fc..a2fc42c 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -6,7 +6,11 @@ from colorama import Fore, Style from polyapi.constants import BASIC_PYTHON_TYPES from polyapi.typedefs import PropertySpecification, PropertyType -from polyapi.schema import wrapped_generate_schema_types, clean_title, map_primitive_types +from polyapi.schema import ( + wrapped_generate_schema_types, + clean_title, + map_primitive_types, +) # this string should be in every __init__ file. @@ -42,7 +46,7 @@ def camelCase(s: str) -> str: s = s.strip() if " " in s or "-" in s: s = re.sub(r"(_|-)+", " ", s).title().replace(" ", "") - return ''.join([s[0].lower(), s[1:]]) + return "".join([s[0].lower(), s[1:]]) else: # s is already in camelcase as best as we can tell, just move on! return s @@ -65,8 +69,7 @@ def print_red(s: str): def add_type_import_path(function_name: str, arg: str) -> str: - """ if not basic type, coerce to camelCase and add the import path - """ + """if not basic type, coerce to camelCase and add the import path""" # for now, just treat Callables as basic types if arg.startswith("Callable"): return arg @@ -83,12 +86,16 @@ def add_type_import_path(function_name: str, arg: str) -> str: sub = sub.replace('"', "") return f'List["{to_func_namespace(function_name)}.{camelCase(sub)}"]' else: - return f'List[{to_func_namespace(function_name)}.{camelCase(sub)}]' + return f"List[{to_func_namespace(function_name)}.{camelCase(sub)}]" - return f'{to_func_namespace(function_name)}.{camelCase(arg)}' + return f"{to_func_namespace(function_name)}.{camelCase(arg)}" -def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: +def get_type_and_def( + type_spec: PropertyType, title_fallback: str = "" +) -> Tuple[str, str]: + """ returns type and type definition for a given PropertyType + """ if type_spec["kind"] == "plain": value = type_spec["value"] if value.endswith("[]"): @@ -115,15 +122,19 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: elif type_spec["kind"] == "object": if type_spec.get("schema"): schema = type_spec["schema"] - title = schema.get("title", schema.get("name", "")) - if title: + title = schema.get("title", schema.get("name", title_fallback)) + if title and schema.get("type") == "array": + # TODO fix me + # we don't use ReturnType as name for the list type here, we use _ReturnTypeItem + return "List", "" + elif title: assert isinstance(title, str) return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore - elif schema.get("allOf") and len(schema['allOf']): + elif schema.get("allOf") and len(schema["allOf"]): # we are in a case of a single allOf, lets strip off the allOf and move on! # our library doesn't handle allOf well yet - allOf = schema['allOf'][0] - title = allOf.get("title", allOf.get("name", "")) + allOf = schema["allOf"][0] + title = allOf.get("title", allOf.get("name", title_fallback)) return wrapped_generate_schema_types(allOf, title, "Dict") elif schema.get("items"): # fallback to schema $ref name if no explicit title @@ -131,7 +142,7 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: title = items.get("title") # type: ignore if not title: # title is actually a reference to another schema - title = items.get("$ref", "") # type: ignore + title = items.get("$ref", title_fallback) # type: ignore title = title.rsplit("/", 1)[-1] if not title: @@ -153,12 +164,18 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: return_type = "Any" for argument in type_spec["spec"]["arguments"]: + # do NOT add this fallback here + # callable arguments don't understand the imports yet + # if it's not a basic type here, we'll just do Any + # _maybe_add_fallback_schema_name(argument) arg_type, arg_def = get_type_and_def(argument["type"]) arg_types.append(arg_type) if arg_def: arg_defs.append(arg_def) - final_arg_type = "Callable[[{}], {}]".format(", ".join(arg_types), return_type) + final_arg_type = "Callable[[{}], {}]".format( + ", ".join(arg_types), return_type + ) return final_arg_type, "\n".join(arg_defs) else: return "Callable", "" @@ -168,15 +185,27 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: return "Any", "" -def parse_arguments(function_name: str, arguments: List[PropertySpecification]) -> Tuple[str, str]: +def _maybe_add_fallback_schema_name(a: PropertySpecification): + if a["type"]["kind"] == "object" and a["type"].get("schema"): + schema = a["type"].get("schema", {}) + if not schema.get("title") and not schema.get("name") and a["name"]: + schema["title"] = a["name"].title() + + +def parse_arguments( + function_name: str, arguments: List[PropertySpecification] +) -> Tuple[str, str]: args_def = [] arg_string = "" for idx, a in enumerate(arguments): + _maybe_add_fallback_schema_name(a) arg_type, arg_def = get_type_and_def(a["type"]) if arg_def: args_def.append(arg_def) a["name"] = rewrite_arg_name(a["name"]) - arg_string += f" {a['name']}: {add_type_import_path(function_name, arg_type)}" + arg_string += ( + f" {a['name']}: {add_type_import_path(function_name, arg_type)}" + ) description = a.get("description", "") description = description.replace("\n", " ") if description: @@ -202,7 +231,7 @@ def poly_full_path(context, name) -> str: def to_func_namespace(s: str) -> str: - """ convert a function name to some function namespace + """convert a function name to some function namespace by default it is """ rv = s[0].upper() + s[1:] @@ -221,6 +250,10 @@ def rewrite_arg_name(s: str): return rewrite_reserved(camelCase(s)) +# def get_return_type_name(function_name: str) -> str: +# return function_name[0].upper() + function_name[1:] + "ReturnType" + + valid_subdomains = ["na[1-2]", "eu[1-2]", "dev"] @@ -238,3 +271,21 @@ def is_valid_uuid(uuid_string, version=4): return False return str(uuid_obj) == uuid_string + + +def return_type_already_defined_in_args(return_type_name: str, args_def: str) -> bool: + """ + Checks if the return_type_name preceded optionally by 'class ' and followed by ' =' exists in args_def. + + Args: + return_type_name (str): The name of the return type to check. + args_def (str): The string containing argument definitions. + + Returns: + bool: True if the pattern exists, False otherwise. + """ + basic_pattern = rf"^{re.escape(return_type_name)}\s=" + basic_match = bool(re.search(basic_pattern, args_def, re.MULTILINE)) + class_pattern = rf"^class {re.escape(return_type_name)}\(TypedDict" + class_match = bool(re.search(class_pattern, args_def, re.MULTILINE)) + return basic_match or class_match \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 69f179a..8f0ac72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev4" +version = "0.3.3.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ @@ -29,3 +29,7 @@ packages = ["polyapi"] [tools.setuptools.packages.find] include = ["polyapi"] exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/cached_deployables*", "polyapi/deployments_revision"] # exclude the generated libraries from builds + +[tool.mypy] +# for now redef errors happen sometimes, we will clean this up in the future! +disable_error_code = "no-redef,name-defined" \ No newline at end of file diff --git a/tests/test_deployables.py b/tests/test_deployables.py index eb49769..80ec742 100644 --- a/tests/test_deployables.py +++ b/tests/test_deployables.py @@ -21,6 +21,7 @@ def foobar() -> int: EXPECTED_SERVER_FN_DEPLOYMENTS = '''# Poly deployed @ 2024-11-12T14:43:22.631113 - testing.foobar - https://na1.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd # Poly deployed @ 2024-11-11T14:43:22.631113 - testing.foobar - https://dev.polyapi.io/canopy/polyui/collections/server-functions/jh23h5g3h5b24jh5b2j3h45v2jhg43v52j3h - 086aedd + from polyapi.typedefs import PolyServerFunction polyConfig: PolyServerFunction = { @@ -65,11 +66,11 @@ def foobar(foo: str, bar: Dict[str, str]) -> int: """A function that does something really import. Args: - foo (str): - bar (Dict[str, str]): + foo (str): + bar (Dict[str, str]): Returns: - int: + int: """ print("Okay then!") return 7 diff --git a/tests/test_generate.py b/tests/test_generate.py index ecc7d05..2e6bcb2 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,7 +1,76 @@ import unittest from polyapi.utils import get_type_and_def, rewrite_reserved -OPENAPI_FUNCTION = {'kind': 'function', 'spec': {'arguments': [{'name': 'event', 'required': False, 'type': {'kind': 'object', 'schema': {'$schema': 'http://json-schema.org/draft-06/schema#', 'type': 'array', 'items': {'$ref': '#/definitions/WebhookEventTypeElement'}, 'definitions': {'WebhookEventTypeElement': {'type': 'object', 'additionalProperties': False, 'properties': {'title': {'type': 'string'}, 'manufacturerName': {'type': 'string'}, 'carType': {'type': 'string'}, 'id': {'type': 'integer'}}, 'required': ['carType', 'id', 'manufacturerName', 'title'], 'title': 'WebhookEventTypeElement'}}}}}, {'name': 'headers', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'params', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'polyCustom', 'required': False, 'type': {'kind': 'object', 'properties': [{'name': 'responseStatusCode', 'type': {'type': 'number', 'kind': 'primitive'}, 'required': True}, {'name': 'responseContentType', 'type': {'type': 'string', 'kind': 'primitive'}, 'required': True, 'nullable': True}]}}], 'returnType': {'kind': 'void'}, 'synchronous': True}} +OPENAPI_FUNCTION = { + "kind": "function", + "spec": { + "arguments": [ + { + "name": "event", + "required": False, + "type": { + "kind": "object", + "schema": { + "$schema": "http://json-schema.org/draft-06/schema#", + "type": "array", + "items": {"$ref": "#/definitions/WebhookEventTypeElement"}, + "definitions": { + "WebhookEventTypeElement": { + "type": "object", + "additionalProperties": False, + "properties": { + "title": {"type": "string"}, + "manufacturerName": {"type": "string"}, + "carType": {"type": "string"}, + "id": {"type": "integer"}, + }, + "required": [ + "carType", + "id", + "manufacturerName", + "title", + ], + "title": "WebhookEventTypeElement", + } + }, + }, + }, + }, + { + "name": "headers", + "required": False, + "type": {"kind": "object", "typeName": "Record"}, + }, + { + "name": "params", + "required": False, + "type": {"kind": "object", "typeName": "Record"}, + }, + { + "name": "polyCustom", + "required": False, + "type": { + "kind": "object", + "properties": [ + { + "name": "responseStatusCode", + "type": {"type": "number", "kind": "primitive"}, + "required": True, + }, + { + "name": "responseContentType", + "type": {"type": "string", "kind": "primitive"}, + "required": True, + "nullable": True, + }, + ], + }, + }, + ], + "returnType": {"kind": "void"}, + "synchronous": True, + }, +} class T(unittest.TestCase): @@ -11,4 +80,4 @@ def test_get_type_and_def(self): def test_rewrite_reserved(self): rv = rewrite_reserved("from") - self.assertEqual(rv, "_from") \ No newline at end of file + self.assertEqual(rv, "_from") diff --git a/tests/test_parser.py b/tests/test_parser.py index b81d29b..22631f5 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -128,7 +128,10 @@ def foobar(foo: str, bar: Dict[str, str]) -> int: return 7 ''' + class T(unittest.TestCase): + maxDiff = 640 + def test_no_types(self): deployable = parse_function_code(CODE_NO_TYPES, "foobar") types = deployable["types"] @@ -237,6 +240,7 @@ def test_parse_glide_server_function_ok_docstring(self): "description": "import number please keep handy" }) + @unittest.skip("TODO fix test") def test_parse_glide_server_function_deploy_receipt(self): code = GLIDE_DEPLOYMENTS_SERVER_FN deployable = parse_function_code(code, "foobar") diff --git a/tests/test_schema.py b/tests/test_schema.py index 8602fd6..223ec39 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,5 +1,5 @@ import unittest -from polyapi.schema import wrapped_generate_schema_types +from polyapi.schema import clean_malformed_examples, wrapped_generate_schema_types SCHEMA = { "$schema": "http://json-schema.org/draft-06/schema#", @@ -10,6 +10,8 @@ "definitions": {}, } +APALEO_MALFORMED_EXAMPLE = 'from typing import List, TypedDict, Union\nfrom typing_extensions import Required\n\n\n# Body.\n# \n# example: {\n "from": "2024-04-21",\n "to": "2024-04-24",\n "grossDailyRate": {\n "amount": 160.0,\n "currency": "EUR"\n },\n "timeSlices": [\n {\n "blockedUnits": 3\n },\n {\n "blockedUnits": 0\n },\n {\n "blockedUnits": 7\n }\n ]\n}\n# x-readme-ref-name: ReplaceBlockModel\nBody = TypedDict(\'Body\', {\n # Start date and time from which the inventory will be blockedSpecify either a pure date or a date and time (without fractional second part) in UTC or with UTC offset as defined in ISO8601:2004\n # \n # Required property\n \'from\': Required[str],\n # End date and time until which the inventory will be blocked. Cannot be more than 5 years after the start date.Specify either a pure date or a date and time (without fractional second part) in UTC or with UTC offset as defined in ISO8601:2004\n # \n # Required property\n \'to\': Required[str],\n # x-readme-ref-name: MonetaryValueModel\n # \n # Required property\n \'grossDailyRate\': Required["_BodygrossDailyRate"],\n # The list of time slices\n # \n # Required property\n \'timeSlices\': Required[List["_BodytimeSlicesitem"]],\n}, total=False)\n\n\nclass _BodygrossDailyRate(TypedDict, total=False):\n """ x-readme-ref-name: MonetaryValueModel """\n\n amount: Required[Union[int, float]]\n """\n format: double\n\n Required property\n """\n\n currency: Required[str]\n """ Required property """\n\n\n\nclass _BodytimeSlicesitem(TypedDict, total=False):\n """ x-readme-ref-name: CreateBlockTimeSliceModel """\n\n blockedUnits: Required[Union[int, float]]\n """\n Number of units blocked for the time slice\n\n format: int32\n\n Required property\n """\n\n' + class T(unittest.TestCase): def test_fix_titles(self): @@ -18,3 +20,7 @@ def test_fix_titles(self): self.assertIn("class MyDict", output[1]) # should not throw with unknown dialect error + + def test_clean_malformed_examples(self): + output = clean_malformed_examples(APALEO_MALFORMED_EXAMPLE) + self.assertNotIn("# example: {", output) \ No newline at end of file diff --git a/tests/test_server.py b/tests/test_server.py index c767d2d..c8126be 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -31,6 +31,40 @@ "visibilityMetadata": {"visibility": "ENVIRONMENT"}, } +LIST_RECOMMENDATIONS = { + "id": "1234-5678-90ab-cdef", + "type": "serverFunction", + "context": "foo", + "name": "listRecommendations", + "contextName": "foo.listRecommendations", + "description": "", + "requirements": [], + "serverSideAsync": False, + "function": { + "arguments": [], + "returnType": { + "kind": "object", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "stay_date": {"type": "string"}, + }, + "additionalProperties": False, + "required": ["id", "stay_date"], + }, + }, + }, + "synchronous": False, + }, + "sourceCode": '', + "language": "javascript", + "state": "ALPHA", + "visibilityMetadata": {"visibility": "ENVIRONMENT"}, +} + class T(unittest.TestCase): def test_render_function_twilio_server(self): @@ -61,4 +95,25 @@ def test_render_function_get_products_count(self): ) self.assertIn(GET_PRODUCTS_COUNT["id"], func_str) self.assertIn("products: List[str]", func_str) - self.assertIn("-> float", func_str) \ No newline at end of file + self.assertIn("-> float", func_str) + + def test_render_function_list_recommendations(self): + return_type = LIST_RECOMMENDATIONS["function"]["returnType"] + func_str, func_type_defs = render_server_function( + LIST_RECOMMENDATIONS["type"], + LIST_RECOMMENDATIONS["name"], + LIST_RECOMMENDATIONS["id"], + LIST_RECOMMENDATIONS["description"], + LIST_RECOMMENDATIONS["function"]["arguments"], + return_type, + ) + self.assertIn(LIST_RECOMMENDATIONS["id"], func_str) + self.assertIn("-> List", func_str) + +# expected_return_type = '''class ReturnType(TypedDict, total=False): +# id: Required[str] +# """ Required property """ + +# stay_date: Required[str] +# """ Required property """''' +# self.assertIn(expected_return_type, func_str) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index ecc7d05..55c446b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,86 @@ import unittest from polyapi.utils import get_type_and_def, rewrite_reserved -OPENAPI_FUNCTION = {'kind': 'function', 'spec': {'arguments': [{'name': 'event', 'required': False, 'type': {'kind': 'object', 'schema': {'$schema': 'http://json-schema.org/draft-06/schema#', 'type': 'array', 'items': {'$ref': '#/definitions/WebhookEventTypeElement'}, 'definitions': {'WebhookEventTypeElement': {'type': 'object', 'additionalProperties': False, 'properties': {'title': {'type': 'string'}, 'manufacturerName': {'type': 'string'}, 'carType': {'type': 'string'}, 'id': {'type': 'integer'}}, 'required': ['carType', 'id', 'manufacturerName', 'title'], 'title': 'WebhookEventTypeElement'}}}}}, {'name': 'headers', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'params', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'polyCustom', 'required': False, 'type': {'kind': 'object', 'properties': [{'name': 'responseStatusCode', 'type': {'type': 'number', 'kind': 'primitive'}, 'required': True}, {'name': 'responseContentType', 'type': {'type': 'string', 'kind': 'primitive'}, 'required': True, 'nullable': True}]}}], 'returnType': {'kind': 'void'}, 'synchronous': True}} +OPENAPI_FUNCTION = { + "kind": "function", + "spec": { + "arguments": [ + { + "name": "event", + "required": False, + "type": { + "kind": "object", + "schema": { + "$schema": "http://json-schema.org/draft-06/schema#", + "type": "array", + "items": {"$ref": "#/definitions/WebhookEventTypeElement"}, + "definitions": { + "WebhookEventTypeElement": { + "type": "object", + "additionalProperties": False, + "properties": { + "title": {"type": "string"}, + "manufacturerName": {"type": "string"}, + "carType": {"type": "string"}, + "id": {"type": "integer"}, + }, + "required": [ + "carType", + "id", + "manufacturerName", + "title", + ], + "title": "WebhookEventTypeElement", + } + }, + }, + }, + }, + { + "name": "headers", + "required": False, + "type": {"kind": "object", "typeName": "Record"}, + }, + { + "name": "params", + "required": False, + "type": {"kind": "object", "typeName": "Record"}, + }, + { + "name": "polyCustom", + "required": False, + "type": { + "kind": "object", + "properties": [ + { + "name": "responseStatusCode", + "type": {"type": "number", "kind": "primitive"}, + "required": True, + }, + { + "name": "responseContentType", + "type": {"type": "string", "kind": "primitive"}, + "required": True, + "nullable": True, + }, + ], + }, + }, + ], + "returnType": {"kind": "void"}, + "synchronous": True, + }, +} class T(unittest.TestCase): def test_get_type_and_def(self): arg_type, arg_def = get_type_and_def(OPENAPI_FUNCTION) - self.assertEqual(arg_type, "Callable[[List[WebhookEventTypeElement], Dict, Dict, Dict], None]") + self.assertEqual( + arg_type, + "Callable[[List[WebhookEventTypeElement], Dict, Dict, Dict], None]", + ) def test_rewrite_reserved(self): rv = rewrite_reserved("from") - self.assertEqual(rv, "_from") \ No newline at end of file + self.assertEqual(rv, "_from") From 1e947d698c2fc4959a63cb8732609a890c4cc4d0 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 8 Apr 2025 11:35:21 -0700 Subject: [PATCH 49/72] next (#35) --- polyapi/parser.py | 13 +++++++------ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/polyapi/parser.py b/polyapi/parser.py index c89d274..24dd7bf 100644 --- a/polyapi/parser.py +++ b/polyapi/parser.py @@ -5,7 +5,7 @@ import re from typing import Dict, List, Mapping, Optional, Tuple, Any from typing import _TypedDictMeta as BaseTypedDict # type: ignore -from typing_extensions import _TypedDictMeta # type: ignore +from typing_extensions import _TypedDictMeta, cast # type: ignore from stdlib_list import stdlib_list from pydantic import TypeAdapter from importlib.metadata import packages_distributions @@ -318,7 +318,7 @@ def _parse_value(value): return None -def parse_function_code(code: str, name: Optional[str] = "", context: Optional[str] = ""): +def parse_function_code(code: str, name: Optional[str] = "", context: Optional[str] = ""): # noqa: C901 schemas = _get_schemas(code) # the pip name and the import name might be different @@ -326,9 +326,9 @@ def parse_function_code(code: str, name: Optional[str] = "", context: Optional[s # see https://stackoverflow.com/a/75144378 pip_name_lookup = packages_distributions() - deployable: DeployableRecord = { - "context": context, - "name": name, + deployable: DeployableRecord = { # type: ignore + "context": context, # type: ignore + "name": name, # type: ignore "description": "", "config": {}, "gitRevision": "", @@ -382,7 +382,7 @@ def visit_AnnAssign(self, node): if node.annotation.id == "PolyServerFunction": deployable["type"] = "server-function" elif node.annotation.id == "PolyClientFunction": - deployable["type"] = "server-function" + deployable["type"] = "client-function" else: print_red("ERROR") print(f"Unsupported polyConfig type '${node.annotation.id}'") @@ -405,6 +405,7 @@ def _extract_docstring_from_function(self, node: ast.FunctionDef): if type(docstring) is None or (not docstring and '"""' not in self._lines[start_lineno] and "'''" not in self._lines[start_lineno]): return None + docstring = cast(str, docstring) # Support both types of triple quotation marks pattern = '"""' diff --git a/pyproject.toml b/pyproject.toml index 8f0ac72..f0156d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev5" +version = "0.3.3.dev6" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 11108e03db7f6c5ff321ae4fec454bfbd8c996cd Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 11 Apr 2025 07:34:00 -0700 Subject: [PATCH 50/72] improve intellisense detection of schemas --- polyapi/poly_schemas.py | 5 +++-- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index 6528341..01aa0a4 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -8,6 +8,8 @@ SCHEMA_CODE_IMPORTS = """from typing_extensions import TypedDict, NotRequired +__all__ = [] + """ @@ -42,7 +44,7 @@ def add_schema_file( # add function to init init_path = os.path.join(full_path, "__init__.py") with open(init_path, "a") as f: - f.write(f"\n\nfrom ._{to_func_namespace(schema_name)} import {schema_name}") + f.write(f"\n\nfrom ._{to_func_namespace(schema_name)} import {schema_name}\n__all__.append('{schema_name}')\n") # add type_defs to underscore file file_path = os.path.join(full_path, f"_{to_func_namespace(schema_name)}.py") @@ -75,7 +77,6 @@ def create_schema( add_import_to_init(full_path, next) - def add_schema_to_init(full_path: str, spec: SchemaSpecDto): init_the_init(full_path, code_imports="") init_path = os.path.join(full_path, "__init__.py") diff --git a/pyproject.toml b/pyproject.toml index f0156d4..ac992d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev6" +version = "0.3.3.dev7" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From ef7e5e9c84a45b6796e99ac034d5371511c637fa Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 14 Apr 2025 11:26:15 -0700 Subject: [PATCH 51/72] release 0.3.3.dev8, fix misleading generate after setup --- polyapi/config.py | 6 +++++- pyproject.toml | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/polyapi/config.py b/polyapi/config.py index 19016f5..2d8dedf 100644 --- a/polyapi/config.py +++ b/polyapi/config.py @@ -55,6 +55,10 @@ def set_api_key_and_url(key: str, url: str): config.set("polyapi", "poly_api_base_url", url) with open(get_config_file_path(), "w") as f: config.write(f) + global API_KEY + global API_URL + API_KEY = key + API_URL = url def initialize_config(force=False): @@ -81,7 +85,7 @@ def initialize_config(force=False): sys.exit(1) set_api_key_and_url(key, url) - print_green(f"Poly setup complete.") + print_green("Poly setup complete.") if not key or not url: print_yellow("Poly API Key and Poly API Base URL are required.") diff --git a/pyproject.toml b/pyproject.toml index ac992d9..cb5e394 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev7" +version = "0.3.3.dev8" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ @@ -32,4 +32,4 @@ exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/cac [tool.mypy] # for now redef errors happen sometimes, we will clean this up in the future! -disable_error_code = "no-redef,name-defined" \ No newline at end of file +disable_error_code = "no-redef,name-defined" From 71e2332b429996542268850aeded6cbf6f7510fd Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 14 Apr 2025 13:56:00 -0700 Subject: [PATCH 52/72] 0.3.3.dev9 - add support for optional arguments (#36) --- LICENSE | 2 +- polyapi/execute.py | 7 ++++++- polyapi/parser.py | 2 +- polyapi/utils.py | 3 +++ pyproject.toml | 3 ++- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index 946285e..6b95772 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 PolyAPI Inc. +Copyright (c) 2025 PolyAPI Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/polyapi/execute.py b/polyapi/execute.py index abd1cee..26b5948 100644 --- a/polyapi/execute.py +++ b/polyapi/execute.py @@ -1,3 +1,4 @@ +from typing import Dict import requests from requests import Response from polyapi.config import get_api_key_and_url @@ -7,10 +8,14 @@ def execute(function_type, function_id, data) -> Response: """ execute a specific function id/type """ + data_without_None = data + if isinstance(data, Dict): + data_without_None = {k: v for k, v in data.items() if v is not None} + api_key, api_url = get_api_key_and_url() headers = {"Authorization": f"Bearer {api_key}"} url = f"{api_url}/functions/{function_type}/{function_id}/execute" - resp = requests.post(url, json=data, headers=headers) + resp = requests.post(url, json=data_without_None, headers=headers) # print(resp.status_code) # print(resp.headers["content-type"]) if resp.status_code < 200 or resp.status_code >= 300: diff --git a/polyapi/parser.py b/polyapi/parser.py index 24dd7bf..8ae3397 100644 --- a/polyapi/parser.py +++ b/polyapi/parser.py @@ -246,7 +246,7 @@ def _get_type_schema(json_type: str, python_type: str, schemas: List[Dict]): return schema -def _get_type(expr: ast.expr | None, schemas: List[Dict]) -> Tuple[str, Dict | None]: +def _get_type(expr: ast.expr | None, schemas: List[Dict]) -> Tuple[Any, Any, Any]: if not expr: return "any", "Any", None python_type = get_python_type_from_ast(expr) diff --git a/polyapi/utils.py b/polyapi/utils.py index a2fc42c..e09383d 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -206,6 +206,9 @@ def parse_arguments( arg_string += ( f" {a['name']}: {add_type_import_path(function_name, arg_type)}" ) + if not a["required"]: + arg_string += " = None" + description = a.get("description", "") description = description.replace("\n", " ") if description: diff --git a/pyproject.toml b/pyproject.toml index cb5e394..6240ded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev8" +version = "0.3.3.dev9" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ @@ -33,3 +33,4 @@ exclude = ["polyapi/poly*", "polyapi/vari*", "polyapi/.config.env", "polyapi/cac [tool.mypy] # for now redef errors happen sometimes, we will clean this up in the future! disable_error_code = "no-redef,name-defined" +implicit_optional = true From 2b4a2d03f500f8555c666d740817a1c98c32da20 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 15 Apr 2025 07:15:04 -0700 Subject: [PATCH 53/72] next --- polyapi/generate.py | 27 ++++++++++++++++++++++++--- polyapi/poly_schemas.py | 11 ++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index fc605df..069f178 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -76,7 +76,7 @@ def resolve_poly_refs(obj, schema_index): def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): spec_idxs_to_remove = [] for idx, spec in enumerate(specs): - if spec.get("type") in ("apiFunction", "customFunction", "serverFunction"): + if spec.get("type") in ("apiFunction", "customFunction", "serverFunction", "schema"): func = spec.get("function") if func: try: @@ -95,6 +95,25 @@ def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): return specs +def replace_poly_refs_in_schemas(specs: List[SchemaSpecDto], schema_index): + spec_idxs_to_remove = [] + for idx, spec in enumerate(specs): + try: + spec["definition"] = resolve_poly_refs(spec["definition"], schema_index) + except Exception: + # print() + print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") + spec_idxs_to_remove.append(idx) + + # reverse the list so we pop off later indexes first + spec_idxs_to_remove.reverse() + + for idx in spec_idxs_to_remove: + specs.pop(idx) + + return specs + + def parse_function_specs( specs: List[SpecificationDto], limit_ids: List[str] | None = None, # optional list of ids to limit to @@ -184,10 +203,12 @@ def generate() -> None: functions = parse_function_specs(specs, limit_ids=limit_ids) schemas = get_schemas() + schema_index = build_schema_index(schemas) if schemas: - generate_schemas(schemas) + schema_limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug + schemas = replace_poly_refs_in_schemas(schemas, schema_index) + generate_schemas(schemas, limit_ids=schema_limit_ids) - schema_index = build_schema_index(schemas) functions = replace_poly_refs_in_functions(functions, schema_index) if functions: diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index 01aa0a4..6b42ec7 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -20,9 +20,14 @@ """ -def generate_schemas(specs: List[SchemaSpecDto]): - for spec in specs: - create_schema(spec) +def generate_schemas(specs: List[SchemaSpecDto], limit_ids: List[str] = None): + if limit_ids: + for spec in specs: + if spec["id"] in limit_ids: + create_schema(spec) + else: + for spec in specs: + create_schema(spec) def add_schema_file( From 2169e156804ded073f2179b09fe1f121cd303a24 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 15 Apr 2025 07:29:54 -0700 Subject: [PATCH 54/72] release 0.3.3.dev10 --- polyapi/generate.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 069f178..521dc43 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -76,7 +76,7 @@ def resolve_poly_refs(obj, schema_index): def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): spec_idxs_to_remove = [] for idx, spec in enumerate(specs): - if spec.get("type") in ("apiFunction", "customFunction", "serverFunction", "schema"): + if spec.get("type") in ("apiFunction", "customFunction", "serverFunction"): func = spec.get("function") if func: try: diff --git a/pyproject.toml b/pyproject.toml index 6240ded..4790dd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev9" +version = "0.3.3.dev10" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From c2db3e9d288ac02f3dba91bce7ad1d7011430c89 Mon Sep 17 00:00:00 2001 From: Eric Neumann Date: Fri, 2 May 2025 06:03:33 -0700 Subject: [PATCH 55/72] EN #3943 update to support SFX serverSideAsync True by setting correct return type (#39) --- polyapi/generate.py | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 521dc43..6873450 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -136,6 +136,10 @@ def parse_function_specs( # poly libraries only support client functions of same language continue + # Functions with serverSideAsync True will always return a Dict with execution ID + if spec.get('serverSideAsync'): + spec['function']['returnType'] = {'kind': 'plain', 'value': 'object'} + functions.append(spec) return functions diff --git a/pyproject.toml b/pyproject.toml index 4790dd6..b9f2926 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev10" +version = "0.3.3.dev11" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 2fc69eacf54b7bda3e181754aefd631174cd69ad Mon Sep 17 00:00:00 2001 From: Nahuel Rebollo Neira Date: Tue, 6 May 2025 15:30:58 -0300 Subject: [PATCH 56/72] deploying version 0.3.3 for R22 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b9f2926..8257c37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3.dev11" +version = "0.3.3" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From a4a48720b3f4ad2e1606065f84dbb92cd197461e Mon Sep 17 00:00:00 2001 From: Bboydozzy96 Date: Tue, 6 May 2025 16:05:43 -0300 Subject: [PATCH 57/72] upgrade version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8257c37..9f7ab3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.3" +version = "0.3.4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From a6ba2f3e0589045659cb33c4dc80c9a579e16e18 Mon Sep 17 00:00:00 2001 From: Shina Akinboboye Date: Wed, 7 May 2025 08:00:05 -0400 Subject: [PATCH 58/72] 4084 - revert strippping none values from function arguments during execution --- polyapi/execute.py | 6 +----- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/polyapi/execute.py b/polyapi/execute.py index 26b5948..ab529d3 100644 --- a/polyapi/execute.py +++ b/polyapi/execute.py @@ -8,14 +8,10 @@ def execute(function_type, function_id, data) -> Response: """ execute a specific function id/type """ - data_without_None = data - if isinstance(data, Dict): - data_without_None = {k: v for k, v in data.items() if v is not None} - api_key, api_url = get_api_key_and_url() headers = {"Authorization": f"Bearer {api_key}"} url = f"{api_url}/functions/{function_type}/{function_id}/execute" - resp = requests.post(url, json=data_without_None, headers=headers) + resp = requests.post(url, json=data, headers=headers) # print(resp.status_code) # print(resp.headers["content-type"]) if resp.status_code < 200 or resp.status_code >= 300: diff --git a/pyproject.toml b/pyproject.toml index 9f7ab3b..15a0ad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.4" +version = "0.3.4.dev1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 9c1a2a163fc8c75487a5c6467824798ac550c06f Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 7 May 2025 14:47:21 +0200 Subject: [PATCH 59/72] P2) Update clients and specs endpoint so when generating with no-types argument all schemas get excluded (#38) * added no type option * version updated --- polyapi/cli.py | 3 ++- polyapi/generate.py | 9 +++++---- polyapi/rendered_spec.py | 5 +++-- pyproject.toml | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 67866d7..347dc1a 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -44,10 +44,11 @@ def setup(args): ########################################################################### # Generate command generate_parser = subparsers.add_parser("generate", help="Generates Poly library") + generate_parser.add_argument("--no-types", action="store_true", help="Generate SDK without type definitions") def generate_command(args): initialize_config() - generate() + generate(no_types=args.no_types) generate_parser.set_defaults(command=generate_command) diff --git a/polyapi/generate.py b/polyapi/generate.py index 6873450..cfc6cfa 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -36,12 +36,13 @@ path:''' -def get_specs() -> List: +def get_specs(no_types: bool = False) -> List: api_key, api_url = get_api_key_and_url() assert api_key headers = get_auth_headers(api_key) url = f"{api_url}/specs" - resp = requests.get(url, headers=headers) + params = {"noTypes": str(no_types).lower()} + resp = requests.get(url, headers=headers, params=params) if resp.status_code == 200: return resp.json() else: @@ -196,11 +197,11 @@ def remove_old_library(): shutil.rmtree(path) -def generate() -> None: +def generate(no_types: bool = False) -> None: print("Generating Poly Python SDK...", end="", flush=True) remove_old_library() - specs = get_specs() + specs = get_specs(no_types=no_types) cache_specs(specs) limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug diff --git a/polyapi/rendered_spec.py b/polyapi/rendered_spec.py index 7de7cad..2206417 100644 --- a/polyapi/rendered_spec.py +++ b/polyapi/rendered_spec.py @@ -35,11 +35,12 @@ def update_rendered_spec(spec: SpecificationDto): assert resp.status_code == 201, (resp.text, resp.status_code) -def _get_spec(spec_id: str) -> Optional[SpecificationDto]: +def _get_spec(spec_id: str, no_types: bool = False) -> Optional[SpecificationDto]: api_key, base_url = get_api_key_and_url() url = f"{base_url}/specs" headers = {"Authorization": f"Bearer {api_key}"} - resp = requests.get(url, headers=headers) + params = {"noTypes": str(no_types).lower()} + resp = requests.get(url, headers=headers, params=params) if resp.status_code == 200: specs = resp.json() for spec in specs: diff --git a/pyproject.toml b/pyproject.toml index 15a0ad9..0df59df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.4.dev1" +version = "0.3.4.dev2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From d20794bcfbff292f865508998604da0eaacd0779 Mon Sep 17 00:00:00 2001 From: Shina Akinboboye <60622084+akinboboye@users.noreply.github.com> Date: Thu, 8 May 2025 13:34:29 -0400 Subject: [PATCH 60/72] 4010 generate contexts (#43) --- .gitignore | 6 ++---- polyapi/cli.py | 10 ++++++++-- polyapi/config.py | 4 +--- polyapi/function_cli.py | 7 ++++++- polyapi/generate.py | 15 ++++++++++----- polyapi/utils.py | 16 ++++++---------- pyproject.toml | 2 +- 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 49ae8f4..135534e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ -.env -.env* -.venv/ -.venv/* +*env +*venv .DS_Store # Pip diff --git a/polyapi/cli.py b/polyapi/cli.py index 347dc1a..7299841 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -45,10 +45,12 @@ def setup(args): # Generate command generate_parser = subparsers.add_parser("generate", help="Generates Poly library") generate_parser.add_argument("--no-types", action="store_true", help="Generate SDK without type definitions") + generate_parser.add_argument("--contexts", type=str, required=False, help="Contexts to generate") def generate_command(args): initialize_config() - generate(no_types=args.no_types) + contexts = args.contexts.split(",") if args.contexts else None + generate(contexts=contexts, no_types=args.no_types) generate_parser.set_defaults(command=generate_command) @@ -69,6 +71,7 @@ def generate_command(args): fn_add_parser.add_argument("--logs", choices=["enabled", "disabled"], default=None, help="Enable or disable logs for the function.") fn_add_parser.add_argument("--execution-api-key", required=False, default="", help="API key for execution (for server functions only).") fn_add_parser.add_argument("--disable-ai", "--skip-generate", action="store_true", help="Pass --disable-ai skip AI generation of missing descriptions") + fn_add_parser.add_argument("--generate-contexts", type=str, help="Server function only – only include certain contexts to speed up function execution") def add_function(args): initialize_config() @@ -80,6 +83,8 @@ def add_function(args): err = "You must specify `--server` or `--client`." elif logs_enabled and not args.server: err = "Option `logs` is only for server functions (--server)." + elif args.generate_contexts and not args.server: + err = "Option `generate-contexts` is only for server functions (--server)." if err: print_red("ERROR") @@ -95,7 +100,8 @@ def add_function(args): server=args.server, logs_enabled=logs_enabled, generate=not args.disable_ai, - execution_api_key=args.execution_api_key + execution_api_key=args.execution_api_key, + generate_contexts=args.generate_contexts ) fn_add_parser.set_defaults(command=add_function) diff --git a/polyapi/config.py b/polyapi/config.py index 2d8dedf..4b0e856 100644 --- a/polyapi/config.py +++ b/polyapi/config.py @@ -3,7 +3,7 @@ import configparser from typing import Tuple -from polyapi.utils import is_valid_polyapi_url, is_valid_uuid, print_green, print_yellow +from polyapi.utils import is_valid_polyapi_url, print_green, print_yellow # cached values API_KEY = None @@ -78,8 +78,6 @@ def initialize_config(force=False): errors = [] if not is_valid_polyapi_url(url): errors.append(f"{url} is not a valid Poly API Base URL") - if not is_valid_uuid(key): - errors.append(f"{key} is not a valid Poly App Key or User Key") if errors: print_yellow("\n".join(errors)) sys.exit(1) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index bc836fa..b888d08 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -24,6 +24,7 @@ def function_add_or_update( client: bool, server: bool, logs_enabled: Optional[bool], + generate_contexts: Optional[str], generate: bool = True, execution_api_key: str = "" ): @@ -59,6 +60,9 @@ def function_add_or_update( "logsEnabled": logs_enabled, } + if generate_contexts: + data["generateContexts"] = generate_contexts.split(",") + if server and parsed["dependencies"]: print_yellow( "\nPlease note that deploying your functions will take a few minutes because it makes use of libraries other than polyapi." @@ -87,7 +91,8 @@ def function_add_or_update( function_id = resp.json()["id"] print(f"Function ID: {function_id}") if generate: - generate_library() + contexts=generate_contexts.split(",") if generate_contexts else None + generate_library(contexts=contexts) else: print("Error adding function.") print(resp.status_code) diff --git a/polyapi/generate.py b/polyapi/generate.py index cfc6cfa..20b9b36 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -2,7 +2,7 @@ import requests import os import shutil -from typing import List, Tuple, cast +from typing import List, Optional, Tuple, cast from .auth import render_auth_function from .client import render_client_function @@ -36,12 +36,16 @@ path:''' -def get_specs(no_types: bool = False) -> List: +def get_specs(contexts=Optional[List[str]], no_types: bool = False) -> List: api_key, api_url = get_api_key_and_url() assert api_key headers = get_auth_headers(api_key) url = f"{api_url}/specs" params = {"noTypes": str(no_types).lower()} + + if contexts: + params["contexts"] = contexts + resp = requests.get(url, headers=headers, params=params) if resp.status_code == 200: return resp.json() @@ -197,11 +201,12 @@ def remove_old_library(): shutil.rmtree(path) -def generate(no_types: bool = False) -> None: - print("Generating Poly Python SDK...", end="", flush=True) +def generate(contexts: Optional[List[str]], no_types: bool = False) -> None: + generate_msg = f"Generating Poly Python SDK for contexts ${contexts}..." if contexts else "Generating Poly Python SDK..." + print(generate_msg, end="", flush=True) remove_old_library() - specs = get_specs(no_types=no_types) + specs = get_specs(no_types=no_types, contexts=contexts) cache_specs(specs) limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug diff --git a/polyapi/utils.py b/polyapi/utils.py index e09383d..642ec51 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -2,6 +2,7 @@ import re import os import uuid +from urllib.parse import urlparse from typing import Tuple, List from colorama import Fore, Style from polyapi.constants import BASIC_PYTHON_TYPES @@ -261,21 +262,16 @@ def rewrite_arg_name(s: str): def is_valid_polyapi_url(_url: str): + # in dev allow localhost (and 127.0.0.1) over http *or* https + parsed = urlparse(_url) + if parsed.scheme in ("http", "https") and parsed.hostname in ("localhost", "127.0.0.1"): + return True + # Join the subdomains into a pattern subdomain_pattern = "|".join(valid_subdomains) pattern = rf"^https://({subdomain_pattern})\.polyapi\.io$" return re.match(pattern, _url) is not None - -def is_valid_uuid(uuid_string, version=4): - try: - uuid_obj = uuid.UUID(uuid_string, version=version) - except ValueError: - return False - - return str(uuid_obj) == uuid_string - - def return_type_already_defined_in_args(return_type_name: str, args_def: str) -> bool: """ Checks if the return_type_name preceded optionally by 'class ' and followed by ' =' exists in args_def. diff --git a/pyproject.toml b/pyproject.toml index 0df59df..8b3e587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.4.dev2" +version = "0.3.5.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From d02bae31c5d76d905b66f6c750b1373a4ff5f795 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 12 May 2025 07:50:45 -0700 Subject: [PATCH 61/72] make contexts truly optional --- polyapi/generate.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 20b9b36..f6aeceb 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -201,7 +201,7 @@ def remove_old_library(): shutil.rmtree(path) -def generate(contexts: Optional[List[str]], no_types: bool = False) -> None: +def generate(contexts: Optional[List[str]] = None, no_types: bool = False) -> None: generate_msg = f"Generating Poly Python SDK for contexts ${contexts}..." if contexts else "Generating Poly Python SDK..." print(generate_msg, end="", flush=True) remove_old_library() diff --git a/pyproject.toml b/pyproject.toml index 8b3e587..5aee5b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.5.dev0" +version = "0.3.5.dev1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From bf238b27778b496682c09a862e02fa531e88fbd5 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 13 May 2025 12:11:12 +0200 Subject: [PATCH 62/72] P3) (Optoro) Allow variable to be secret in the UI, but gettable in functions, and prevent secret variables from being made non-secret (#42) * secret -> secrecy - updated python client * comment fixed --- polyapi/typedefs.py | 6 +++++- polyapi/variables.py | 7 ++++--- pyproject.toml | 2 +- tests/test_variables.py | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index 3a6d84a..6d6ff18 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -40,11 +40,15 @@ class SpecificationDto(TypedDict): language: str +# Enum for variable secrecy levels +Secrecy = Literal['SECRET', 'OBSCURED', 'NONE'] + + class VariableSpecification(TypedDict): environmentId: str value: Any valueType: PropertyType - secret: bool + secrecy: Secrecy class VariableSpecDto(TypedDict): diff --git a/polyapi/variables.py b/polyapi/variables.py index 95f7d2f..673a195 100644 --- a/polyapi/variables.py +++ b/polyapi/variables.py @@ -2,11 +2,11 @@ from typing import List from polyapi.schema import map_primitive_types -from polyapi.typedefs import PropertyType, VariableSpecDto +from polyapi.typedefs import PropertyType, VariableSpecDto, Secrecy from polyapi.utils import add_import_to_init, init_the_init -# GET is only included if the variable is not a secret +# GET is only included if the variable is not SECRET GET_TEMPLATE = """ @staticmethod def get() -> {variable_type}: @@ -76,9 +76,10 @@ def generate_variables(variables: List[VariableSpecDto]): def render_variable(variable: VariableSpecDto): variable_type = _get_variable_type(variable["variable"]["valueType"]) + # Only include get() method if secrecy is not SECRET get_method = ( "" - if variable["variable"]["secret"] + if variable["variable"]["secrecy"] == "SECRET" else GET_TEMPLATE.format( variable_id=variable["id"], variable_type=variable_type ) diff --git a/pyproject.toml b/pyproject.toml index 5aee5b5..6a114d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.5.dev1" +version = "0.3.6.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ diff --git a/tests/test_variables.py b/tests/test_variables.py index beea44d..72eac81 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -12,7 +12,7 @@ }, "variable": { "environmentId": "123818231", - "secret": False, + "secrecy": "NONE", "valueType": { "kind": "primitive", "type": "string" From 78d4b15a5f008600e1166639fd43f252956f70af Mon Sep 17 00:00:00 2001 From: Shina Akinboboye <60622084+akinboboye@users.noreply.github.com> Date: Thu, 15 May 2025 15:28:19 -0400 Subject: [PATCH 63/72] add generate contexts (#45) --- polyapi/function_cli.py | 3 +-- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index b888d08..bff9a90 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -91,8 +91,7 @@ def function_add_or_update( function_id = resp.json()["id"] print(f"Function ID: {function_id}") if generate: - contexts=generate_contexts.split(",") if generate_contexts else None - generate_library(contexts=contexts) + generate_library() else: print("Error adding function.") print(resp.status_code) diff --git a/pyproject.toml b/pyproject.toml index 6a114d2..6881a0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.6.dev0" +version = "0.3.7.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From ebf6e6b707b8359d8661193a833b70b0a17cc0b0 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 22 May 2025 17:23:00 +0200 Subject: [PATCH 64/72] adds mtls and direct execute options (#44) * adds mtls and direct execute support * support for direct execute from client * fixed mtls * removed unused dep --- polyapi/api.py | 12 ++++++++-- polyapi/config.py | 31 ++++++++++++++++++++++++- polyapi/execute.py | 56 +++++++++++++++++++++++++++++++++++++++++---- polyapi/generate.py | 6 ++++- polyapi/utils.py | 2 +- pyproject.toml | 2 +- 6 files changed, 98 insertions(+), 11 deletions(-) diff --git a/polyapi/api.py b/polyapi/api.py index 9533b21..fc243ba 100644 --- a/polyapi/api.py +++ b/polyapi/api.py @@ -23,8 +23,16 @@ def {function_name}( Function ID: {function_id} \""" - resp = execute("{function_type}", "{function_id}", {data}) - return {api_response_type}(resp.json()) # type: ignore + if get_direct_execute_config(): + resp = direct_execute("{function_type}", "{function_id}", {data}) + return {api_response_type}({{ + "status": resp.status_code, + "headers": dict(resp.headers), + "data": resp.json() + }}) # type: ignore + else: + resp = execute("{function_type}", "{function_id}", {data}) + return {api_response_type}(resp.json()) # type: ignore """ diff --git a/polyapi/config.py b/polyapi/config.py index 4b0e856..60eb16f 100644 --- a/polyapi/config.py +++ b/polyapi/config.py @@ -8,6 +8,10 @@ # cached values API_KEY = None API_URL = None +API_FUNCTION_DIRECT_EXECUTE = None +MTLS_CERT_PATH = None +MTLS_KEY_PATH = None +MTLS_CA_PATH = None def get_config_file_path() -> str: @@ -45,6 +49,13 @@ def get_api_key_and_url() -> Tuple[str | None, str | None]: API_KEY = key API_URL = url + # Read and cache MTLS and direct execute settings + global API_FUNCTION_DIRECT_EXECUTE, MTLS_CERT_PATH, MTLS_KEY_PATH, MTLS_CA_PATH + API_FUNCTION_DIRECT_EXECUTE = config.get("polyapi", "api_function_direct_execute", fallback="false").lower() == "true" + MTLS_CERT_PATH = config.get("polyapi", "mtls_cert_path", fallback=None) + MTLS_KEY_PATH = config.get("polyapi", "mtls_key_path", fallback=None) + MTLS_CA_PATH = config.get("polyapi", "mtls_ca_path", fallback=None) + return key, url @@ -104,4 +115,22 @@ def clear_config(): path = get_config_file_path() if os.path.exists(path): - os.remove(path) \ No newline at end of file + os.remove(path) + + +def get_mtls_config() -> Tuple[bool, str | None, str | None, str | None]: + """Return MTLS configuration settings""" + global MTLS_CERT_PATH, MTLS_KEY_PATH, MTLS_CA_PATH + if MTLS_CERT_PATH is None or MTLS_KEY_PATH is None or MTLS_CA_PATH is None: + # Force a config read if values aren't cached + get_api_key_and_url() + return bool(MTLS_CERT_PATH and MTLS_KEY_PATH and MTLS_CA_PATH), MTLS_CERT_PATH, MTLS_KEY_PATH, MTLS_CA_PATH + + +def get_direct_execute_config() -> bool: + """Return whether direct execute is enabled""" + global API_FUNCTION_DIRECT_EXECUTE + if API_FUNCTION_DIRECT_EXECUTE is None: + # Force a config read if value isn't cached + get_api_key_and_url() + return bool(API_FUNCTION_DIRECT_EXECUTE) \ No newline at end of file diff --git a/polyapi/execute.py b/polyapi/execute.py index ab529d3..d066574 100644 --- a/polyapi/execute.py +++ b/polyapi/execute.py @@ -1,22 +1,68 @@ -from typing import Dict +from typing import Dict, Optional import requests from requests import Response -from polyapi.config import get_api_key_and_url +from polyapi.config import get_api_key_and_url, get_mtls_config from polyapi.exceptions import PolyApiException +def direct_execute(function_type, function_id, data) -> Response: + """ execute a specific function id/type + """ + api_key, api_url = get_api_key_and_url() + headers = {"Authorization": f"Bearer {api_key}"} + url = f"{api_url}/functions/{function_type}/{function_id}/direct-execute" + + endpoint_info = requests.post(url, json=data, headers=headers) + if endpoint_info.status_code < 200 or endpoint_info.status_code >= 300: + raise PolyApiException(f"{endpoint_info.status_code}: {endpoint_info.content.decode('utf-8', errors='ignore')}") + + endpoint_info_data = endpoint_info.json() + request_params = endpoint_info_data.copy() + request_params.pop("url", None) + + if "maxRedirects" in request_params: + request_params["allow_redirects"] = request_params.pop("maxRedirects") > 0 + + has_mtls, cert_path, key_path, ca_path = get_mtls_config() + + if has_mtls: + resp = requests.request( + url=endpoint_info_data["url"], + cert=(cert_path, key_path), + verify=ca_path, + **request_params + ) + else: + resp = requests.request( + url=endpoint_info_data["url"], + verify=False, + **request_params + ) + + if resp.status_code < 200 or resp.status_code >= 300: + error_content = resp.content.decode("utf-8", errors="ignore") + raise PolyApiException(f"{resp.status_code}: {error_content}") + + return resp def execute(function_type, function_id, data) -> Response: """ execute a specific function id/type """ api_key, api_url = get_api_key_and_url() headers = {"Authorization": f"Bearer {api_key}"} + url = f"{api_url}/functions/{function_type}/{function_id}/execute" - resp = requests.post(url, json=data, headers=headers) - # print(resp.status_code) - # print(resp.headers["content-type"]) + + # Make the request + resp = requests.post( + url, + json=data, + headers=headers, + ) + if resp.status_code < 200 or resp.status_code >= 300: error_content = resp.content.decode("utf-8", errors="ignore") raise PolyApiException(f"{resp.status_code}: {error_content}") + return resp diff --git a/polyapi/generate.py b/polyapi/generate.py index f6aeceb..1558b9e 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -14,7 +14,7 @@ from .server import render_server_function from .utils import add_import_to_init, get_auth_headers, init_the_init, print_green, to_func_namespace from .variables import generate_variables -from .config import get_api_key_and_url +from .config import get_api_key_and_url, get_direct_execute_config SUPPORTED_FUNCTION_TYPES = { "apiFunction", @@ -46,6 +46,10 @@ def get_specs(contexts=Optional[List[str]], no_types: bool = False) -> List: if contexts: params["contexts"] = contexts + # Add apiFunctionDirectExecute parameter if direct execute is enabled + if get_direct_execute_config(): + params["apiFunctionDirectExecute"] = "true" + resp = requests.get(url, headers=headers, params=params) if resp.status_code == 200: return resp.json() diff --git a/polyapi/utils.py b/polyapi/utils.py index 642ec51..2ea11a2 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -16,7 +16,7 @@ # this string should be in every __init__ file. # it contains all the imports needed for the function or variable code to run -CODE_IMPORTS = "from typing import List, Dict, Any, Optional, Callable\nfrom typing_extensions import TypedDict, NotRequired\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" +CODE_IMPORTS = "from typing import List, Dict, Any, Optional, Callable\nfrom typing_extensions import TypedDict, NotRequired\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url, get_direct_execute_config\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update, direct_execute\n\n" def init_the_init(full_path: str, code_imports="") -> None: diff --git a/pyproject.toml b/pyproject.toml index 6881a0d..61f6eec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev0" +version = "0.3.7.dev1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 938dfdd94c21885651b86e8b6a62aef831d3ca5a Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 26 May 2025 10:13:04 +0200 Subject: [PATCH 65/72] polyCustom - prevent rewrites of executionId (#46) --- polyapi/__init__.py | 83 +++++++++++++++++++++++++++++++++++++++++---- pyproject.toml | 2 +- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/polyapi/__init__.py b/polyapi/__init__.py index 2a30c36..7935843 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -1,7 +1,8 @@ import os import sys import truststore -from typing import Dict, Any +from typing import Any, Optional, overload, Literal +from typing_extensions import TypedDict truststore.inject_into_ssl() from .cli import CLI_COMMANDS @@ -15,9 +16,77 @@ sys.exit(1) -polyCustom: Dict[str, Any] = { - "executionId": None, - "executionApiKey": None, - "responseStatusCode": 200, - "responseContentType": None, -} \ No newline at end of file +class PolyCustomDict(TypedDict, total=False): + """Type definition for polyCustom dictionary.""" + executionId: Optional[str] # Read-only + executionApiKey: Optional[str] + responseStatusCode: int + responseContentType: Optional[str] + + +class _PolyCustom: + def __init__(self): + self._internal_store = { + "executionId": None, + "executionApiKey": None, + "responseStatusCode": 200, + "responseContentType": None, + } + self._execution_id_locked = False + + def set_once(self, key: str, value: Any) -> None: + if key == "executionId" and self._execution_id_locked: + # Silently ignore attempts to overwrite locked executionId + return + self._internal_store[key] = value + if key == "executionId": + # Lock executionId after setting it + self.lock_execution_id() + + def get(self, key: str, default: Any = None) -> Any: + return self._internal_store.get(key, default) + + def lock_execution_id(self) -> None: + self._execution_id_locked = True + + def unlock_execution_id(self) -> None: + self._execution_id_locked = False + + @overload + def __getitem__(self, key: Literal["executionId"]) -> Optional[str]: ... + + @overload + def __getitem__(self, key: Literal["executionApiKey"]) -> Optional[str]: ... + + @overload + def __getitem__(self, key: Literal["responseStatusCode"]) -> int: ... + + @overload + def __getitem__(self, key: Literal["responseContentType"]) -> Optional[str]: ... + + def __getitem__(self, key: str) -> Any: + return self.get(key) + + @overload + def __setitem__(self, key: Literal["executionApiKey"], value: Optional[str]) -> None: ... + + @overload + def __setitem__(self, key: Literal["responseStatusCode"], value: int) -> None: ... + + @overload + def __setitem__(self, key: Literal["responseContentType"], value: Optional[str]) -> None: ... + + def __setitem__(self, key: str, value: Any) -> None: + self.set_once(key, value) + + def __repr__(self) -> str: + return f"PolyCustom({self._internal_store})" + + def copy(self) -> '_PolyCustom': + new = _PolyCustom() + new._internal_store = self._internal_store.copy() + new._execution_id_locked = self._execution_id_locked + return new + + +polyCustom: PolyCustomDict = _PolyCustom() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 61f6eec..bb639af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev1" +version = "0.3.7.dev2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From db5381b8dcb736258b176f2a461ff02a4a0d8e84 Mon Sep 17 00:00:00 2001 From: Shina Akinboboye <60622084+akinboboye@users.noreply.github.com> Date: Tue, 27 May 2025 15:25:08 -0400 Subject: [PATCH 66/72] 4292 (#47) --- polyapi/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/polyapi/utils.py b/polyapi/utils.py index 2ea11a2..b9ffa7f 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -151,8 +151,11 @@ def get_type_and_def( title = f"List[{title}]" return wrapped_generate_schema_types(schema, title, "List") + elif schema.get("properties"): + result = wrapped_generate_schema_types(schema, "ResponseType", "Dict") # type: ignore + return result else: - return "Any", "" + return "Dict", "" else: return "Dict", "" elif type_spec["kind"] == "function": From 93cff007b14e9067ec956b883b96c9f6a377b837 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 28 May 2025 15:35:26 +0200 Subject: [PATCH 67/72] create mock schemas to fit everything when using no types flag (#48) --- polyapi/function_cli.py | 2 +- polyapi/generate.py | 133 ++++++++++++++++++++------ polyapi/utils.py | 57 ++++++++--- pyproject.toml | 2 +- tests/test_generate.py | 206 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 359 insertions(+), 41 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index bff9a90..bc99f2b 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -86,7 +86,7 @@ def function_add_or_update( headers = get_auth_headers(api_key) resp = requests.post(url, headers=headers, json=data) - if resp.status_code == 201: + if resp.status_code in [200, 201]: print_green("DEPLOYED") function_id = resp.json()["id"] print(f"Function ID: {function_id}") diff --git a/polyapi/generate.py b/polyapi/generate.py index 1558b9e..df848a1 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -129,24 +129,26 @@ def parse_function_specs( ) -> List[SpecificationDto]: functions = [] for spec in specs: - if not spec or "function" not in spec: - continue - - if not spec["function"]: - continue - - if limit_ids and spec["id"] not in limit_ids: + if not spec: continue + # For no_types mode, we might not have function data, but we still want to include the spec + # if it's a supported function type if spec["type"] not in SUPPORTED_FUNCTION_TYPES: continue - if spec["type"] == "customFunction" and spec["language"] != "python": - # poly libraries only support client functions of same language + # Skip if we have a limit and this spec is not in it + if limit_ids and spec.get("id") not in limit_ids: continue + # For customFunction, check language if we have function data + if spec["type"] == "customFunction": + if spec.get("language") and spec["language"] != "python": + # poly libraries only support client functions of same language + continue + # Functions with serverSideAsync True will always return a Dict with execution ID - if spec.get('serverSideAsync'): + if spec.get('serverSideAsync') and spec.get("function"): spec['function']['returnType'] = {'kind': 'plain', 'value': 'object'} functions.append(spec) @@ -205,6 +207,63 @@ def remove_old_library(): shutil.rmtree(path) +def create_empty_schemas_module(): + """Create an empty schemas module for no-types mode so user code can still import from polyapi.schemas""" + currdir = os.path.dirname(os.path.abspath(__file__)) + schemas_path = os.path.join(currdir, "schemas") + + # Create the schemas directory + if not os.path.exists(schemas_path): + os.makedirs(schemas_path) + + # Create an __init__.py file with dynamic schema resolution + init_path = os.path.join(schemas_path, "__init__.py") + with open(init_path, "w") as f: + f.write('''"""Empty schemas module for no-types mode""" +from typing import Any, Dict + +class _GenericSchema(Dict[str, Any]): + """Generic schema type that acts like a Dict for no-types mode""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + +class _SchemaModule: + """Dynamic module that returns itself for attribute access, allowing infinite nesting""" + + def __getattr__(self, name: str): + # For callable access (like schemas.Response()), return the generic schema class + # For further attribute access (like schemas.random.random2), return self to allow nesting + return _NestedSchemaAccess() + + def __call__(self, *args, **kwargs): + # If someone tries to call the module itself, return a generic schema + return _GenericSchema(*args, **kwargs) + + def __dir__(self): + # Return common schema names for introspection + return ['Response', 'Request', 'Error', 'Data', 'Result'] + +class _NestedSchemaAccess: + """Handles nested attribute access and final callable resolution""" + + def __getattr__(self, name: str): + # Continue allowing nested access + return _NestedSchemaAccess() + + def __call__(self, *args, **kwargs): + # When finally called, return a generic schema instance + return _GenericSchema(*args, **kwargs) + + def __class_getitem__(cls, item): + # Support type annotations like schemas.Response[str] + return _GenericSchema + +# Replace this module with our dynamic module +import sys +sys.modules[__name__] = _SchemaModule() +''') + + def generate(contexts: Optional[List[str]] = None, no_types: bool = False) -> None: generate_msg = f"Generating Poly Python SDK for contexts ${contexts}..." if contexts else "Generating Poly Python SDK..." print(generate_msg, end="", flush=True) @@ -216,14 +275,23 @@ def generate(contexts: Optional[List[str]] = None, no_types: bool = False) -> No limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug functions = parse_function_specs(specs, limit_ids=limit_ids) - schemas = get_schemas() - schema_index = build_schema_index(schemas) - if schemas: - schema_limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug - schemas = replace_poly_refs_in_schemas(schemas, schema_index) - generate_schemas(schemas, limit_ids=schema_limit_ids) - - functions = replace_poly_refs_in_functions(functions, schema_index) + # Only process schemas if no_types is False + if not no_types: + schemas = get_schemas() + schema_index = build_schema_index(schemas) + if schemas: + schema_limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug + schemas = replace_poly_refs_in_schemas(schemas, schema_index) + generate_schemas(schemas, limit_ids=schema_limit_ids) + + functions = replace_poly_refs_in_functions(functions, schema_index) + else: + # When no_types is True, we still need to process functions but without schema resolution + # Use an empty schema index to avoid poly-ref resolution + schema_index = {} + + # Create an empty schemas module so user code can still import from polyapi.schemas + create_empty_schemas_module() if functions: generate_functions(functions) @@ -233,10 +301,11 @@ def generate(contexts: Optional[List[str]] = None, no_types: bool = False) -> No ) exit() - variables = get_variables() - if variables: - generate_variables(variables) - + # Only process variables if no_types is False + if not no_types: + variables = get_variables() + if variables: + generate_variables(variables) # indicator to vscode extension that this is a polyapi-python project file_path = os.path.join(os.getcwd(), ".polyapi-python") @@ -266,11 +335,19 @@ def render_spec(spec: SpecificationDto) -> Tuple[str, str]: arguments: List[PropertySpecification] = [] return_type = {} - if spec["function"]: - arguments = [ - arg for arg in spec["function"]["arguments"] - ] - return_type = spec["function"]["returnType"] + if spec.get("function"): + # Handle cases where arguments might be missing or None + if spec["function"].get("arguments"): + arguments = [ + arg for arg in spec["function"]["arguments"] + ] + + # Handle cases where returnType might be missing or None + if spec["function"].get("returnType"): + return_type = spec["function"]["returnType"] + else: + # Provide a fallback return type when missing + return_type = {"kind": "any"} if function_type == "apiFunction": func_str, func_type_defs = render_api_function( @@ -284,7 +361,7 @@ def render_spec(spec: SpecificationDto) -> Tuple[str, str]: elif function_type == "customFunction": func_str, func_type_defs = render_client_function( function_name, - spec["code"], + spec.get("code", ""), arguments, return_type, ) diff --git a/polyapi/utils.py b/polyapi/utils.py index b9ffa7f..470c4b2 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -97,20 +97,32 @@ def get_type_and_def( ) -> Tuple[str, str]: """ returns type and type definition for a given PropertyType """ + # Handle cases where type_spec might be None or empty + if not type_spec: + return "Any", "" + + # Handle cases where kind might be missing + if "kind" not in type_spec: + return "Any", "" + if type_spec["kind"] == "plain": - value = type_spec["value"] + value = type_spec.get("value", "") if value.endswith("[]"): primitive = map_primitive_types(value[:-2]) return f"List[{primitive}]", "" else: return map_primitive_types(value), "" elif type_spec["kind"] == "primitive": - return map_primitive_types(type_spec["type"]), "" + return map_primitive_types(type_spec.get("type", "any")), "" elif type_spec["kind"] == "array": if type_spec.get("items"): items = type_spec["items"] if items.get("$ref"): - return wrapped_generate_schema_types(type_spec, "ResponseType", "Dict") # type: ignore + # For no-types mode, avoid complex schema generation + try: + return wrapped_generate_schema_types(type_spec, "ResponseType", "Dict") # type: ignore + except: + return "List[Dict]", "" else: item_type, _ = get_type_and_def(items) title = f"List[{item_type}]" @@ -130,13 +142,20 @@ def get_type_and_def( return "List", "" elif title: assert isinstance(title, str) - return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore + # For no-types mode, avoid complex schema generation + try: + return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore + except: + return "Dict", "" elif schema.get("allOf") and len(schema["allOf"]): # we are in a case of a single allOf, lets strip off the allOf and move on! # our library doesn't handle allOf well yet allOf = schema["allOf"][0] title = allOf.get("title", allOf.get("name", title_fallback)) - return wrapped_generate_schema_types(allOf, title, "Dict") + try: + return wrapped_generate_schema_types(allOf, title, "Dict") + except: + return "Dict", "" elif schema.get("items"): # fallback to schema $ref name if no explicit title items = schema.get("items") # type: ignore @@ -150,7 +169,10 @@ def get_type_and_def( return "List", "" title = f"List[{title}]" - return wrapped_generate_schema_types(schema, title, "List") + try: + return wrapped_generate_schema_types(schema, title, "List") + except: + return "List[Dict]", "" elif schema.get("properties"): result = wrapped_generate_schema_types(schema, "ResponseType", "Dict") # type: ignore return result @@ -190,9 +212,13 @@ def get_type_and_def( def _maybe_add_fallback_schema_name(a: PropertySpecification): - if a["type"]["kind"] == "object" and a["type"].get("schema"): + # Handle cases where type might be missing + if not a.get("type"): + return + + if a["type"].get("kind") == "object" and a["type"].get("schema"): schema = a["type"].get("schema", {}) - if not schema.get("title") and not schema.get("name") and a["name"]: + if not schema.get("title") and not schema.get("name") and a.get("name"): schema["title"] = a["name"].title() @@ -203,14 +229,23 @@ def parse_arguments( arg_string = "" for idx, a in enumerate(arguments): _maybe_add_fallback_schema_name(a) - arg_type, arg_def = get_type_and_def(a["type"]) + + # Handle cases where type might be missing + arg_type_spec = a.get("type", {"kind": "any"}) + arg_type, arg_def = get_type_and_def(arg_type_spec) if arg_def: args_def.append(arg_def) - a["name"] = rewrite_arg_name(a["name"]) + + # Handle cases where name might be missing + arg_name = a.get("name", f"arg{idx}") + a["name"] = rewrite_arg_name(arg_name) + arg_string += ( f" {a['name']}: {add_type_import_path(function_name, arg_type)}" ) - if not a["required"]: + + # Handle cases where required might be missing + if not a.get("required", True): arg_string += " = None" description = a.get("description", "") diff --git a/pyproject.toml b/pyproject.toml index bb639af..f5ff503 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev2" +version = "0.3.7.dev3" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ diff --git a/tests/test_generate.py b/tests/test_generate.py index 2e6bcb2..b738ba6 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,5 +1,9 @@ import unittest +import os +import shutil +import importlib.util from polyapi.utils import get_type_and_def, rewrite_reserved +from polyapi.generate import render_spec, create_empty_schemas_module OPENAPI_FUNCTION = { "kind": "function", @@ -72,6 +76,28 @@ }, } +# Test spec with missing function data (simulating no_types=true) +NO_TYPES_SPEC = { + "id": "test-id-123", + "type": "serverFunction", + "context": "test", + "name": "testFunction", + "description": "A test function for no-types mode", + # Note: no "function" field, simulating no_types=true response +} + +# Test spec with minimal function data +MINIMAL_FUNCTION_SPEC = { + "id": "test-id-456", + "type": "apiFunction", + "context": "test", + "name": "minimalFunction", + "description": "A minimal function spec", + "function": { + # Note: no "arguments" or "returnType" fields + } +} + class T(unittest.TestCase): def test_get_type_and_def(self): @@ -81,3 +107,183 @@ def test_get_type_and_def(self): def test_rewrite_reserved(self): rv = rewrite_reserved("from") self.assertEqual(rv, "_from") + + def test_render_spec_no_function_data(self): + """Test that render_spec handles specs with no function data gracefully""" + func_str, func_type_defs = render_spec(NO_TYPES_SPEC) + + # Should generate a function even without function data + self.assertIsNotNone(func_str) + self.assertIsNotNone(func_type_defs) + self.assertIn("testFunction", func_str) + self.assertIn("test-id-123", func_str) + + def test_render_spec_minimal_function_data(self): + """Test that render_spec handles specs with minimal function data""" + func_str, func_type_defs = render_spec(MINIMAL_FUNCTION_SPEC) + + # Should generate a function with fallback types + self.assertIsNotNone(func_str) + self.assertIsNotNone(func_type_defs) + self.assertIn("minimalFunction", func_str) + self.assertIn("test-id-456", func_str) + # Should use Any as fallback return type in the type definitions + self.assertIn("Any", func_type_defs) + + def test_create_empty_schemas_module(self): + """Test that create_empty_schemas_module creates the necessary files""" + # Clean up any existing schemas directory + schemas_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "polyapi", "schemas") + if os.path.exists(schemas_path): + shutil.rmtree(schemas_path) + + # Create empty schemas module + create_empty_schemas_module() + + # Verify the directory and __init__.py file were created + self.assertTrue(os.path.exists(schemas_path)) + init_path = os.path.join(schemas_path, "__init__.py") + self.assertTrue(os.path.exists(init_path)) + + # Verify the content of __init__.py includes dynamic schema handling + with open(init_path, "r") as f: + content = f.read() + self.assertIn("Empty schemas module for no-types mode", content) + self.assertIn("_GenericSchema", content) + self.assertIn("_SchemaModule", content) + self.assertIn("__getattr__", content) + + # Clean up + shutil.rmtree(schemas_path) + + def test_no_types_workflow(self): + """Test the complete no-types workflow including schema imports and function parsing""" + import tempfile + import sys + from unittest.mock import patch + + # Clean up any existing schemas directory + schemas_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "polyapi", "schemas") + if os.path.exists(schemas_path): + shutil.rmtree(schemas_path) + + # Mock get_specs to return empty list (simulating no functions) + with patch('polyapi.generate.get_specs', return_value=[]): + try: + # This should exit with SystemExit due to no functions + from polyapi.generate import generate + generate(no_types=True) + except SystemExit: + pass # Expected when no functions exist + + # Verify schemas module was created + self.assertTrue(os.path.exists(schemas_path)) + init_path = os.path.join(schemas_path, "__init__.py") + self.assertTrue(os.path.exists(init_path)) + + # Test that we can import schemas and use arbitrary schema names + from polyapi import schemas + + # Test various schema access + Response = schemas.Response + CustomType = schemas.CustomType + AnyName = schemas.SomeArbitrarySchemaName + + # All should return the same generic schema class type + self.assertEqual(type(Response).__name__, '_NestedSchemaAccess') + self.assertEqual(type(CustomType).__name__, '_NestedSchemaAccess') + self.assertEqual(type(AnyName).__name__, '_NestedSchemaAccess') + + # Test creating instances + response_instance = Response() + custom_instance = CustomType() + + self.assertIsInstance(response_instance, dict) + self.assertIsInstance(custom_instance, dict) + + # Test that function code with schema references can be parsed + test_code = ''' +from polyapi import polyCustom, schemas + +def test_function() -> schemas.Response: + polyCustom["executionId"] = "123" + return polyCustom +''' + + # Create a temporary file with the test code + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write(test_code) + temp_file = f.name + + try: + # Test that the parser can handle this code + from polyapi.parser import parse_function_code + result = parse_function_code(test_code, 'test_function', 'test_context') + + self.assertEqual(result['name'], 'test_function') + self.assertEqual(result['context'], 'test_context') + # Return type should be Any since we're in no-types mode + self.assertEqual(result['types']['returns']['type'], 'Any') + + finally: + # Clean up temp file + os.unlink(temp_file) + + # Clean up schemas directory + shutil.rmtree(schemas_path) + + def test_nested_schema_access(self): + """Test that nested schema access like schemas.random.random2.random3 works""" + # Clean up any existing schemas directory + schemas_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "polyapi", "schemas") + if os.path.exists(schemas_path): + shutil.rmtree(schemas_path) + + # Create empty schemas module + create_empty_schemas_module() + + # Test that we can import and use nested schemas + from polyapi import schemas + + # Test various levels of nesting + simple = schemas.Response + nested = schemas.random.random2 + deep_nested = schemas.api.v1.user.profile.data + very_deep = schemas.some.very.deep.nested.schema.access + + # All should be _NestedSchemaAccess instances + self.assertEqual(type(simple).__name__, '_NestedSchemaAccess') + self.assertEqual(type(nested).__name__, '_NestedSchemaAccess') + self.assertEqual(type(deep_nested).__name__, '_NestedSchemaAccess') + self.assertEqual(type(very_deep).__name__, '_NestedSchemaAccess') + + # Test that they can be called and return generic schemas + simple_instance = simple() + nested_instance = nested() + deep_instance = deep_nested() + very_deep_instance = very_deep() + + # All should be dictionaries + self.assertIsInstance(simple_instance, dict) + self.assertIsInstance(nested_instance, dict) + self.assertIsInstance(deep_instance, dict) + self.assertIsInstance(very_deep_instance, dict) + + # Test that function code with nested schemas can be parsed + test_code = ''' +from polyapi import polyCustom, schemas + +def test_nested_function() -> schemas.api.v1.user.profile: + return schemas.api.v1.user.profile() +''' + + from polyapi.parser import parse_function_code + result = parse_function_code(test_code, 'test_nested_function', 'test_context') + + self.assertEqual(result['name'], 'test_nested_function') + self.assertEqual(result['context'], 'test_context') + # Return type should be Any since we're in no-types mode + self.assertEqual(result['types']['returns']['type'], 'Any') + + # Clean up schemas directory + shutil.rmtree(schemas_path) From 050853e5c80db1193101c48dc983d3b874051aea Mon Sep 17 00:00:00 2001 From: Eric Neumann Date: Wed, 28 May 2025 08:29:51 -0700 Subject: [PATCH 68/72] EN #4348 flatten new-lines in arg descriptions (#50) * EN #4348 flatten new-lines in arg descriptions * EN #4348 bump version to 0.3.7.dev4 --- polyapi/utils.py | 10 ++++++++-- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/polyapi/utils.py b/polyapi/utils.py index 470c4b2..19217a4 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -222,6 +222,12 @@ def _maybe_add_fallback_schema_name(a: PropertySpecification): schema["title"] = a["name"].title() +def _clean_description(text: str) -> str: + """Flatten new-lines and collapse excess whitespace.""" + text = text.replace("\\n", " ").replace("\n", " ") + return re.sub(r"\s+", " ", text).strip() + + def parse_arguments( function_name: str, arguments: List[PropertySpecification] ) -> Tuple[str, str]: @@ -248,8 +254,8 @@ def parse_arguments( if not a.get("required", True): arg_string += " = None" - description = a.get("description", "") - description = description.replace("\n", " ") + description = _clean_description(a.get("description", "")) + if description: if idx == len(arguments) - 1: arg_string += f" # {description}\n" diff --git a/pyproject.toml b/pyproject.toml index f5ff503..f5c2514 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev3" +version = "0.3.7.dev4" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From ee50a54cc840f5f4256fd1d3a2337a90f8e1e14c Mon Sep 17 00:00:00 2001 From: Eric Neumann Date: Wed, 28 May 2025 08:39:58 -0700 Subject: [PATCH 69/72] EN #4360 fix return types for TS funcs (#49) --- polyapi/schema.py | 6 +++++- polyapi/utils.py | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/polyapi/schema.py b/polyapi/schema.py index 1dccd22..6ca3391 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -126,4 +126,8 @@ def clean_title(title: str) -> str: def map_primitive_types(type_: str) -> str: # Define your mapping logic here - return JSONSCHEMA_TO_PYTHON_TYPE_MAP.get(type_, "Any") \ No newline at end of file + return JSONSCHEMA_TO_PYTHON_TYPE_MAP.get(type_, "Any") + + +def is_primitive(type_: str) -> bool: + return type_ in JSONSCHEMA_TO_PYTHON_TYPE_MAP diff --git a/polyapi/utils.py b/polyapi/utils.py index 19217a4..3b61f5e 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -11,6 +11,7 @@ wrapped_generate_schema_types, clean_title, map_primitive_types, + is_primitive ) @@ -140,11 +141,17 @@ def get_type_and_def( # TODO fix me # we don't use ReturnType as name for the list type here, we use _ReturnTypeItem return "List", "" + elif title and title == "ReturnType" and schema.get("type"): + assert isinstance(title, str) + schema_type = schema.get("type", "Any") + root_type, generated_code = wrapped_generate_schema_types(schema, schema_type, "Dict") # type: ignore + return (map_primitive_types(root_type), "") if is_primitive(root_type) else (root_type, generated_code) # type: ignore elif title: assert isinstance(title, str) # For no-types mode, avoid complex schema generation try: - return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore + root_type, generated_code = wrapped_generate_schema_types(schema, title, "Dict") # type: ignore + return ("Any", "") if root_type == "ReturnType" else wrapped_generate_schema_types(schema, title, "Dict") # type: ignore except: return "Dict", "" elif schema.get("allOf") and len(schema["allOf"]): From bd3359353730c29ba996c15c1a614b6b89f6fb50 Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Wed, 28 May 2025 14:19:55 -0700 Subject: [PATCH 70/72] adding ability for python client server and client functions to add custom headers (#51) --- polyapi/__init__.py | 11 ++++++++++- polyapi/webhook.py | 1 + pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/polyapi/__init__.py b/polyapi/__init__.py index 7935843..0d811a3 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -1,5 +1,6 @@ import os import sys +import copy import truststore from typing import Any, Optional, overload, Literal from typing_extensions import TypedDict @@ -22,6 +23,7 @@ class PolyCustomDict(TypedDict, total=False): executionApiKey: Optional[str] responseStatusCode: int responseContentType: Optional[str] + responseHeaders: Dict[str, str] class _PolyCustom: @@ -31,6 +33,7 @@ def __init__(self): "executionApiKey": None, "responseStatusCode": 200, "responseContentType": None, + "responseHeaders": {}, } self._execution_id_locked = False @@ -63,6 +66,9 @@ def __getitem__(self, key: Literal["responseStatusCode"]) -> int: ... @overload def __getitem__(self, key: Literal["responseContentType"]) -> Optional[str]: ... + + @overload + def __getitem__(self, key: Literal["responseHeaders"]) -> Dict[str, str]: ... def __getitem__(self, key: str) -> Any: return self.get(key) @@ -75,6 +81,9 @@ def __setitem__(self, key: Literal["responseStatusCode"], value: int) -> None: . @overload def __setitem__(self, key: Literal["responseContentType"], value: Optional[str]) -> None: ... + + @overload + def __setitem__(self, key: Literal["responseHeaders"], value: Dict[str, str]) -> None: ... def __setitem__(self, key: str, value: Any) -> None: self.set_once(key, value) @@ -84,7 +93,7 @@ def __repr__(self) -> str: def copy(self) -> '_PolyCustom': new = _PolyCustom() - new._internal_store = self._internal_store.copy() + new._internal_store = copy.deepcopy(self._internal_store) new._execution_id_locked = self._execution_id_locked return new diff --git a/polyapi/webhook.py b/polyapi/webhook.py index 855c300..b27987d 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -65,6 +65,7 @@ async def handleEvent(data): "data": resp, "statusCode": polyCustom.get("responseStatusCode", 200), "contentType": polyCustom.get("responseContentType", None), + "headers": polyCustom.get("responseHeaders", {{}}), }}, }}, namespace="/events") diff --git a/pyproject.toml b/pyproject.toml index f5c2514..93e189b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev4" +version = "0.3.7.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From bc8e09635ab1c5874aacca8db3b5224aa21a1e3c Mon Sep 17 00:00:00 2001 From: Aaron Goin Date: Wed, 28 May 2025 15:08:16 -0700 Subject: [PATCH 71/72] fix type error! --- polyapi/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/__init__.py b/polyapi/__init__.py index 0d811a3..583d1f3 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -2,7 +2,7 @@ import sys import copy import truststore -from typing import Any, Optional, overload, Literal +from typing import Any, Dict, Optional, overload, Literal from typing_extensions import TypedDict truststore.inject_into_ssl() from .cli import CLI_COMMANDS diff --git a/pyproject.toml b/pyproject.toml index 93e189b..3d4a292 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev5" +version = "0.3.7.dev6" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 569ec608d243b5de91ec36300a8f30188b1ce478 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 30 May 2025 06:56:43 -0700 Subject: [PATCH 72/72] try simple upgrade --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3d4a292..4ac7031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.7.dev6" +version = "0.3.7.dev7" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [