AI Components PR Review

.tool-versions

@@ -1,2 +1,4 @@
 nodejs 14.19.0
 pnpm 7.13.4
+python 3.11.5
+poetry 1.6.1

packages/component_code_gen/.env.example

@@ -0,0 +1,17 @@
+BROWSERLESS_API_KEY=your-browserless-api-key
+
+SUPABASE_URL=https://your-supabase-url.supabase.co
+SUPABASE_API_KEY=your-supabase-service-role-key
+
+OPENAI_API_TYPE=openai
+OPENAI_API_KEY=your-openai-api-key
+OPENAI_MODEL=gpt-4
+
+# comment or remove these below if using openai api
+OPENAI_API_TYPE=azure
+OPENAI_DEPLOYMENT_NAME=deployment-name
+OPENAI_API_VERSION=2023-05-15
+OPENAI_API_BASE=https://resource-name.openai.azure.com
+OPENAI_API_KEY=azure-api-key
+OPENAI_MODEL=gpt-4-32k
+OPENAI_TEMPERATURE=0.5

packages/component_code_gen/.gitignore

@@ -1,5 +1,8 @@
 .env
+instructions.md
 .vscode
 ve/
 node_modules/
 __pycache__/
+tests/**/output
+*.mjs

packages/component_code_gen/README.md

@@ -1,36 +1,76 @@
 # AI Code Gen for Pipedream Components
 
-Generate components using OpenAI GPT-4.
+Generate components using OpenAI GPT.
 
 
-### Run
+### Installation
 
 ```
-SCRIPT=generate_action.py
-APP=slack
-PROMPT="how to send myself a direct message?"
-poetry run python3 "$SCRIPT" --app "$APP" "$PROMPT"
-poetry run python3 "$SCRIPT" --app "$APP" "$PROMPT" --verbose # print debug logs
+asdf plugin-add poetry
+asdf install
+cd packages/component_code_gen
+poetry install
 ```
 
+### Setup
 
-### Installation
+#### Create a `.env` file
+
+```
+cd packages/component_code_gen
+cp .env.example .env
+```
+
+#### Modify the `.env` file to use your own keys:
+
+1. Add these API Keys to your new `.env` file:
+
+    - BROWSERLESS_API_KEY=api_key # not required
+    - SUPABASE_URL=https://your-project-url.supabase.co # get this from Supabase Project Settings -> API
+    - SUPABASE_API_KEY=service_role_key # get this from Supabase Project Settings -> API
+
+2. Add OpenAI keys
+
+    - OPENAI_API_TYPE=openai
+    - OPENAI_API_KEY=your-openai-api-key
+    - OPENAI_MODEL=gpt-4
+
+3. Or use a Azure OpenAI deployment (gpt-4-32k)
 
-1. Install poetry: follow instructions at https://python-poetry.org/docs/#installation
+    - OPENAI_API_TYPE=azure
+    - OPENAI_DEPLOYMENT_NAME=deployment-name
+    - OPENAI_API_VERSION=2023-05-15
+    - OPENAI_API_BASE=https://resource-name.openai.azure.com
+    - OPENAI_API_KEY=azure-api-key
+    - OPENAI_MODEL=gpt-4-32k
+
+5. Create a file named `instructions.md` with the same structure as the `instructions.md.example` file:
 
-2. Run install:
 ```
-poetry install
+## Prompt
+
+... your prompt here ...
+
+## API docs
+
+... copy and paste relevant parts of the api docs here ...
 ```
 
 
-### Setup
+### Run
 
-Create a `.env` file
+```
+poetry run python main.py --type action --app slack --instructions instructions.md --verbose
+```
+
+
+### Tests
 
-Add these API Keys:
+To run a suite of tests (e.g. webhook_sources):
+
+```
+poetry run python -m tests.test --type webhook_sources
+```
 
-    - BROWSERLESS_API_KEY=api_key
-    - OPENAI_API_KEY=API_KEY
-    - SUPABASE_URL=https://url.supabase.co
-    - SUPABASE_API_KEY=service_role_key
+This script will generate code for some selected apps/components for comparison with registry components
+Compare `./tests/webhook_sources/output/*` with `./tests/webhook_sources/reference/*`

packages/component_code_gen/code_gen/generate_component_code.py

@@ -1,46 +1,87 @@
+import config.logging_config as logging_config
+from config.config import config
+import helpers.supabase_helpers as supabase_helpers
+import helpers.langchain_helpers as langchain_helpers
 from dotenv import load_dotenv
 load_dotenv()
 
-import helpers.langchain_helpers as langchain_helpers
-import helpers.supabase_helpers as supabase_helpers
 
-import config.logging_config as logging_config
 logger = logging_config.getLogger(__name__)
 
 
-def main(app, prompt, templates):
+def generate_code(app, prompt, templates, tries):
+    validate_inputs(app, prompt, templates, tries)
     db = supabase_helpers.SupabaseConnector()
-
     docs_meta = db.get_app_docs_meta(app)
-
-    if 'docs_url' in docs_meta:
-        contents = db.get_docs_contents(app)
-        if contents:
-            docs =  { row['url']: row['content'] for row in contents }
-            return with_docs(app, prompt, docs, 'api reference', templates)
-
-    if 'openapi_url' in docs_meta:
-        contents = db.get_openapi_contents(app)
-        if contents:
-            docs = { row['path']: row['content'] for row in contents }
-            return with_docs(app, prompt, docs, 'openapi', templates)
-
-    return no_docs(app, prompt, templates)
-
-
-def no_docs(app, prompt, templates):
-    logger.debug('no docs, calling openai directly')
-
-    return langchain_helpers.no_docs(app, prompt, templates)
-
-
-def with_docs(app, prompt, docs, docs_type, templates):
-    logger.debug(f"using {docs_type} docs")
-
-    result = langchain_helpers.ask_agent(prompt, docs, templates)
+    results = []
+
+    auth_example = None
+    auth_meta = db.get_app_auth_meta(app)
+    if auth_meta.get('component_code_scaffold_raw'):
+        auth_example = f"Here's how authentication is done in {app}:\n\n{auth_meta['component_code_scaffold_raw']}\n\n"
+
+    for i in range(tries):
+        logger.debug(f'Attempt {i+1} of {tries}')
+
+        # Initialize a flag to track if we obtained any results with docs
+        has_docs_result = False
+
+        if 'docs_url' in docs_meta:
+            contents = db.get_docs_contents(app)
+            if contents:
+                docs = {row['url']: row['content'] for row in contents}
+                results.append(call_langchain(
+                    app, prompt, templates, auth_example, docs, 'api reference'))
+                has_docs_result = True
+
+        if 'openapi_url' in docs_meta:
+            contents = db.get_openapi_contents(app)
+            if contents:
+                docs = {row['path']: row['content'] for row in contents}
+                results.append(call_langchain(
+                    app, prompt, templates, auth_example, docs, 'openapi'))
+                has_docs_result = True
+
+        # If we haven't obtained any results using docs
+        if not has_docs_result:
+            results.append(call_langchain(app, prompt, templates, auth_example))
+
+    # Create a new prompt string
+    new_prompt = "I've asked other GPT agents to generate the following code based on the prompt and the instructions below. One set of code (or all) may not follow the rules laid out in the prompt or in the instructions below, so you'll need to review it for accuracy. Try to evaluate the examples according to the rules, combine the best parts of each example, and generate a final set of code that solves the problem posed by the prompt and follows all of the rules below. Here are the attempts + code:\n\n---\n\n"
+    for idx, result in enumerate(results, 1):
+        new_prompt += f"Try {idx}:\n\n${result}\n\n---\n\n"
+    new_prompt += "---\n\n" + prompt
+
+    # Call the model again with the new prompt to get the final result
+    logger.debug(f"Calling the model a final time to summarize the attempts")
+    return call_langchain(app, new_prompt, templates, auth_example)
+
+
+def call_langchain(app, prompt, templates, auth_example, docs=None, docs_type=None, attempts=0, max_attempts=3):
+    logger.debug(f"Calling langchain")
+    # If we don't have docs, or if we can't reach OpenAI to get the parsed docs
+    if not docs:
+        logger.debug('No docs available, calling the model directly')
+        return langchain_helpers.no_docs(app, prompt, templates, auth_example)
+
+    if attempts >= max_attempts:
+        logger.debug('Max attempts reached, calling the model directly')
+        return langchain_helpers.no_docs(app, prompt, templates, auth_example)
+
+    # else if we have docs, call the model with docs
+    logger.debug(f"Using {docs_type} docs")
+
+    result = langchain_helpers.ask_agent(prompt, docs, templates, auth_example)
 
     if result != "I don't know":
         return result
 
-    logger.debug("trying again without docs")
-    return no_docs(app, prompt, templates)
+    logger.debug("Trying again without docs")
+    return call_langchain(app, prompt, templates, auth_example, attempts=attempts+1)
+
+
+def validate_inputs(app, prompt, templates, tries):
+    assert app and type(app) == str
+    assert prompt and type(prompt) == str
+    assert tries and type(tries) == int
+    assert templates.system_instructions

packages/component_code_gen/config/config.py

@@ -1,21 +1,39 @@
+import os
 from dotenv import load_dotenv
 load_dotenv()
 
-import os
+
+def get_env_var(var_name, required=False, default=None):
+    if os.environ.get(var_name):
+        return os.environ.get(var_name)
+    if default is not None:
+        return default
+    if required:
+        raise Exception(f"Environment variable {var_name} is required")
 
 
 config = {
+    "temperature": get_env_var("OPENAI_TEMPERATURE", default=0.5),
+    "openai_api_type": get_env_var("OPENAI_API_TYPE", default="azure"),
     "openai": {
-        "api_key": os.environ.get('OPENAI_API_KEY'),
+        "api_key": get_env_var("OPENAI_API_KEY", required=True),
+        "model": get_env_var("OPENAI_MODEL", default="gpt-4"),
+    },
+    "azure": {
+        "deployment_name": get_env_var("OPENAI_DEPLOYMENT_NAME", required=True),
+        "api_version": get_env_var("OPENAI_API_VERSION", default="2023-05-15"),
+        "api_base": get_env_var("OPENAI_API_BASE", required=True),
+        "api_key": get_env_var("OPENAI_API_KEY", required=True),
+        "model": get_env_var("OPENAI_MODEL", default="gpt-4-32k"),
     },
     "browserless": {
-        "api_key": os.environ.get('BROWSERLESS_API_KEY'),
+        "api_key": get_env_var("BROWSERLESS_API_KEY"),
     },
     "supabase": {
-        "url": os.environ.get('SUPABASE_URL'),
-        "api_key": os.environ.get('SUPABASE_API_KEY'),
+        "url": get_env_var("SUPABASE_URL", required=True),
+        "api_key": get_env_var("SUPABASE_API_KEY", required=True),
     },
     "logging": {
-        "level": "DEBUG" if os.environ.get('DEBUG') == "1" else "WARN",
+        "level": get_env_var("LOGGING_LEVEL", default="WARN"),
     },
 }