diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index b505582f2a957..08b4fcaccfa1a 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -44,7 +44,7 @@ cp .env.example .env - OPENAI_API_KEY=azure-api-key - OPENAI_MODEL=gpt-4-32k -4. Create a file named `instructions.md` with the same structure as the `instructions.md.exaple` file: +5. Create a file named `instructions.md` with the same structure as the `instructions.md.example` file: ``` ## Prompt diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index b90c1dd546051..61d2d0a8f3c5c 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -10,19 +10,19 @@ def generate_code(app, prompt, templates, tries): + validate_inputs(app, prompt, templates, tries) + db = supabase_helpers.SupabaseConnector() + docs_meta = db.get_app_docs_meta(app) results = [] + auth_example = None + auth_meta = db.get_app_auth_meta(app) + if auth_meta.get('component_code_scaffold_raw'): + auth_example = f"Here's how authentication is done in {app}:\n\n{auth_meta['component_code_scaffold_raw']}\n\n" + for i in range(tries): logger.debug(f'Attempt {i+1} of {tries}') - validate_inputs(app, prompt, templates, tries) - - db = supabase_helpers.SupabaseConnector() - - auth_meta = db.get_app_auth_meta(app) - # TODO: is this needed only for actions? - # add_code_example(templates, auth_meta['component_code_scaffold_raw']) - docs_meta = db.get_app_docs_meta(app) # Initialize a flag to track if we obtained any results with docs has_docs_result = False @@ -31,7 +31,7 @@ def generate_code(app, prompt, templates, tries): if contents: docs = {row['url']: row['content'] for row in contents} results.append(call_langchain( - app, prompt, templates, docs, 'api reference')) + app, prompt, templates, auth_example, docs, 'api reference')) has_docs_result = True if 'openapi_url' in docs_meta: @@ -39,12 +39,12 @@ def generate_code(app, prompt, templates, tries): if contents: docs = {row['path']: row['content'] for row in contents} results.append(call_langchain( - app, prompt, templates, docs, 'openapi')) + app, prompt, templates, auth_example, docs, 'openapi')) has_docs_result = True # If we haven't obtained any results using docs if not has_docs_result: - results.append(call_langchain(app, prompt, templates)) + results.append(call_langchain(app, prompt, templates, auth_example)) # Create a new prompt string new_prompt = "I've asked other GPT agents to generate the following code based on the prompt and the instructions below. One set of code (or all) may not follow the rules laid out in the prompt or in the instructions below, so you'll need to review it for accuracy. Try to evaluate the examples according to the rules, combine the best parts of each example, and generate a final set of code that solves the problem posed by the prompt and follows all of the rules below. Here are the attempts + code:\n\n---\n\n" @@ -54,42 +54,34 @@ def generate_code(app, prompt, templates, tries): # Call the model again with the new prompt to get the final result logger.debug(f"Calling the model a final time to summarize the attempts") - return call_langchain(app, new_prompt, templates) + return call_langchain(app, new_prompt, templates, auth_example) -def call_langchain(app, prompt, templates, docs=None, docs_type=None, attempts=0, max_attempts=3): +def call_langchain(app, prompt, templates, auth_example, docs=None, docs_type=None, attempts=0, max_attempts=3): logger.debug(f"Calling langchain") # If we don't have docs, or if we can't reach OpenAI to get the parsed docs if not docs: logger.debug('No docs available, calling the model directly') - return langchain_helpers.no_docs(app, prompt, templates) + return langchain_helpers.no_docs(app, prompt, templates, auth_example) if attempts >= max_attempts: logger.debug('Max attempts reached, calling the model directly') - return langchain_helpers.no_docs(app, prompt, templates) + return langchain_helpers.no_docs(app, prompt, templates, auth_example) # else if we have docs, call the model with docs logger.debug(f"Using {docs_type} docs") - result = langchain_helpers.ask_agent(prompt, docs, templates) + result = langchain_helpers.ask_agent(prompt, docs, templates, auth_example) if result != "I don't know": return result logger.debug("Trying again without docs") - return call_langchain(app, prompt, templates, attempts=attempts+1) - - -def add_code_example(templates, example): - return templates.no_docs_system_instructions % example + return call_langchain(app, prompt, templates, auth_example, attempts=attempts+1) def validate_inputs(app, prompt, templates, tries): assert app and type(app) == str assert prompt and type(prompt) == str assert tries and type(tries) == int - assert templates.no_docs_user_prompt - assert templates.no_docs_system_instructions - assert templates.with_docs_system_instructions - assert templates.suffix - assert templates.format_instructions + assert templates.system_instructions diff --git a/packages/component_code_gen/config/config.py b/packages/component_code_gen/config/config.py index 88f9cc17764e4..9aafa2ee15ae7 100644 --- a/packages/component_code_gen/config/config.py +++ b/packages/component_code_gen/config/config.py @@ -3,37 +3,28 @@ load_dotenv() -DEFAULTS = { - "OPENAI_API_TYPE": "openai", - "OPENAI_MODEL": "gpt-4", - "OPENAI_API_VERSION": "2023-05-15", - "LOGGING_LEVEL": "WARN", - "ENABLE_DOCS": False -} - - -def get_env_var(var_name, required=False): +def get_env_var(var_name, required=False, default=None): if os.environ.get(var_name): return os.environ.get(var_name) - if required and var_name not in DEFAULTS: + if default is not None: + return default + if required: raise Exception(f"Environment variable {var_name} is required") - if var_name in DEFAULTS: - return DEFAULTS[var_name] config = { - "temperature": get_env_var("OPENAI_TEMPERATURE") or 0.5, - "openai_api_type": get_env_var("OPENAI_API_TYPE"), + "temperature": get_env_var("OPENAI_TEMPERATURE", default=0.5), + "openai_api_type": get_env_var("OPENAI_API_TYPE", default="azure"), "openai": { "api_key": get_env_var("OPENAI_API_KEY", required=True), - "model": get_env_var("OPENAI_MODEL"), + "model": get_env_var("OPENAI_MODEL", default="gpt-4"), }, "azure": { - "deployment_name": get_env_var("OPENAI_DEPLOYMENT_NAME"), - "api_version": get_env_var("OPENAI_API_VERSION"), - "api_base": get_env_var("OPENAI_API_BASE"), + "deployment_name": get_env_var("OPENAI_DEPLOYMENT_NAME", required=True), + "api_version": get_env_var("OPENAI_API_VERSION", default="2023-05-15"), + "api_base": get_env_var("OPENAI_API_BASE", required=True), "api_key": get_env_var("OPENAI_API_KEY", required=True), - "model": get_env_var("OPENAI_MODEL"), + "model": get_env_var("OPENAI_MODEL", default="gpt-4-32k"), }, "browserless": { "api_key": get_env_var("BROWSERLESS_API_KEY"), @@ -43,7 +34,6 @@ def get_env_var(var_name, required=False): "api_key": get_env_var("SUPABASE_API_KEY", required=True), }, "logging": { - "level": get_env_var("LOGGING_LEVEL"), + "level": get_env_var("LOGGING_LEVEL", default="WARN"), }, - "enable_docs": get_env_var("ENABLE_DOCS") or False, } diff --git a/packages/component_code_gen/helpers/langchain_helpers.py b/packages/component_code_gen/helpers/langchain_helpers.py index 0413f257232f5..459d4f6a13275 100644 --- a/packages/component_code_gen/helpers/langchain_helpers.py +++ b/packages/component_code_gen/helpers/langchain_helpers.py @@ -1,3 +1,6 @@ +from templates.common.suffix import suffix +from templates.common.format_instructions import format_instructions +from templates.common.docs_system_instructions import docs_system_instructions from langchain.schema import ( # AIMessage, HumanMessage, @@ -24,19 +27,25 @@ def create_tools(docs): class PipedreamOpenAPIAgent: - def __init__(self, docs, templates): + def __init__(self, docs, templates, auth_example): + system_instructions = format_template( + f"{templates.system_instructions(auth_example)}\n{docs_system_instructions}") + tools = OpenAPIExplorerTool.create_tools(docs) tool_names = [tool.name for tool in tools] + prompt_template = ZeroShotAgent.create_prompt( tools=tools, - prefix=format_template(templates.with_docs_system_instructions), - suffix=templates.suffix, - format_instructions=templates.format_instructions, + prefix=system_instructions, + suffix=suffix, + format_instructions=format_instructions, input_variables=['input', 'agent_scratchpad'] ) + llm_chain = LLMChain(llm=get_llm(), prompt=prompt_template) agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names) verbose = True if config['logging']['level'] == 'DEBUG' else False + self.agent_executor = AgentExecutor.from_agent_and_tools( agent=agent, tools=tools, verbose=verbose) @@ -66,26 +75,27 @@ def format_result(result): def get_llm(): if config['openai_api_type'] == "azure": azure_config = config["azure"] - llm = AzureChatOpenAI(deployment_name=azure_config['deployment_name'], - model_name=azure_config["model"], temperature=config["temperature"], request_timeout=300) + return AzureChatOpenAI(deployment_name=azure_config['deployment_name'], + model_name=azure_config["model"], temperature=config["temperature"], request_timeout=300) else: openai_config = config["openai"] - llm = ChatOpenAI( + return ChatOpenAI( model_name=openai_config["model"], temperature=config["temperature"], request_timeout=300) - return llm -def ask_agent(user_prompt, docs, templates): - agent = PipedreamOpenAPIAgent(docs, templates) +def ask_agent(user_prompt, docs, templates, auth_example): + agent = PipedreamOpenAPIAgent(docs, templates, auth_example) result = agent.run(user_prompt) return result -def no_docs(app, prompt, templates): +def no_docs(app, prompt, templates, auth_example): + user_prompt = f"{prompt}.The app is {app}." + system_instructions = format_template(templates.system_instructions(auth_example)) + result = get_llm()(messages=[ - SystemMessage(content=format_template( - templates.no_docs_system_instructions)), - HumanMessage(content=templates.no_docs_user_prompt % (prompt, app)), + SystemMessage(content=system_instructions), + HumanMessage(content=user_prompt), ]) return format_result(result.content) diff --git a/packages/component_code_gen/templates/actions/additional_rules.py b/packages/component_code_gen/templates/actions/additional_rules.py new file mode 100644 index 0000000000000..9b3a8e1d2ece1 --- /dev/null +++ b/packages/component_code_gen/templates/actions/additional_rules.py @@ -0,0 +1,32 @@ +additional_rules = """## Additional rules for actions + +1. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it. + +2. Always use this signature for the run method: + +async run({steps, $}) { + // your code here +} + +Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. + +9. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs. + +For example, do this: + +const response = await axios(this, { + url: `https://api.stability.ai/v1/engines/list`, + headers: { + Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, + }, +}); +// process the response data. response.data is undefined + +not this: + +const { data } = await axios(this, { + url: `https://api.stability.ai/v1/engines/list`, + headers: { + Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, + }, +});""" diff --git a/packages/component_code_gen/templates/actions/export_summary.py b/packages/component_code_gen/templates/actions/export_summary.py new file mode 100644 index 0000000000000..f73c4fa5c9d05 --- /dev/null +++ b/packages/component_code_gen/templates/actions/export_summary.py @@ -0,0 +1 @@ +export_summary = """A short summary should be exported before the end so that the user can quickly read what has happened. This is done by calling `$.export("$summary", "Your summary here")`. This is not optional.""" diff --git a/packages/component_code_gen/templates/actions/introduction.py b/packages/component_code_gen/templates/actions/introduction.py new file mode 100644 index 0000000000000..e50056635d2f5 --- /dev/null +++ b/packages/component_code_gen/templates/actions/introduction.py @@ -0,0 +1,12 @@ +# Experimenting with prompt from an AI researcher: https://twitter.com/jeremyphoward/status/1689464589191454720 +introduction = """## Instructions + +You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. + +Your goal is to create Pipedream Action Components. Your code should solve the requirements provided below. + +Other GPT agents will be reviewing your work, and will provide feedback on your code. You will be rewarded for code that is accurate, factual, thoughtful, nuanced, and solves the requirements provided in the instructions. + +## Pipedream components + +All Pipedream components are Node.js modules that have a default export: `defineComponent`. `defineComponent` is provided to the environment as a global — you do not need to import `defineComponent`. `defineComponent` is a function that takes an object — a Pipedream component — as its single argument.""" diff --git a/packages/component_code_gen/templates/actions/main_example.py b/packages/component_code_gen/templates/actions/main_example.py new file mode 100644 index 0000000000000..aa1b59b6c7c02 --- /dev/null +++ b/packages/component_code_gen/templates/actions/main_example.py @@ -0,0 +1,28 @@ +main_example = """Here's an example component: + +```javascript +import { axios } from "@pipedream/platform" +export default defineComponent({ + key: "openai-list-models", + name: "List Models", + description: "Lists all models available to the user.", + version: "0.0.{{ts}}", + type: "action", + props: { + openai: { + type: "app", + app: "openai", + } + }, + async run({steps, $}) { + const response = await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, + }) + $.export("$summary", "Successfully listed models") + return response + }, +}) +```""" diff --git a/packages/component_code_gen/templates/actions/other_example.py b/packages/component_code_gen/templates/actions/other_example.py new file mode 100644 index 0000000000000..a1b4f92874901 --- /dev/null +++ b/packages/component_code_gen/templates/actions/other_example.py @@ -0,0 +1,69 @@ +other_example = """Here's an example Pipedream component that makes a test request against the Slack API: + +export default defineComponent({ + key: "slack-send-message", + name: "Send Message", + version: "0.0.{{ts}}", + description: "Sends a message to a channel. [See docs here]()", + type: "action", + props: { + slack: { + type: "app", + app: "slack", + }, + channel: { + type: "string", + label: "Channel", + description: "The channel to post the message to", + }, + text: { + type: "string", + label: "Message Text", + description: "The text of the message to post", + }, + }, + async run({ steps, $ }) { + const response = await axios($, { + method: "POST", + url: `https://slack.com/api/chat.postMessage`, + headers: { + Authorization: `Bearer ${this.slack.$auth.oauth_access_token}`, + }, + data: { + channel: this.channel, + text: this.text, + }, + }) + $.export("$summary", "Sent message successfully") + return response + }, +}); + +Notice this section: + +data: { + channel: this.channel, + text: this.text, +}, + +This shows you how to pass the values of props (e.g. this.channel and this.text) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See rule #2 below for more detail. + +The code you generate should be placed within the `run` method of the Pipedream component: + +import { axios } from "@pipedream/platform"; + +export default defineComponent({ + props: { + the_app_name_slug: { + type: "app", + app: "the_app_name_slug", + }, + }, + async run({ steps, $ }) { + const response = await axios($, { + // Add the axios configuration object to make the HTTP request here + }) + $.export("$summary", "Your summary here") + return response + }, +});""" diff --git a/packages/component_code_gen/templates/apps/additional_rules.py b/packages/component_code_gen/templates/apps/additional_rules.py new file mode 100644 index 0000000000000..d34ace201e68d --- /dev/null +++ b/packages/component_code_gen/templates/apps/additional_rules.py @@ -0,0 +1 @@ +additional_rules = """""" diff --git a/packages/component_code_gen/templates/apps/auth.py b/packages/component_code_gen/templates/apps/auth.py new file mode 100644 index 0000000000000..09614f248dba6 --- /dev/null +++ b/packages/component_code_gen/templates/apps/auth.py @@ -0,0 +1,7 @@ +auth = """This lets the user connect their app account to the step, authorizing requests to the app API. + +`this` exposes the user's app credentials in the object `this.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header.""" diff --git a/packages/component_code_gen/templates/apps/introduction.py b/packages/component_code_gen/templates/apps/introduction.py new file mode 100644 index 0000000000000..68f72ae8fb1f4 --- /dev/null +++ b/packages/component_code_gen/templates/apps/introduction.py @@ -0,0 +1,6 @@ +introduction = """You are an agent designed to create Pipedream App Code. + +## Pipedream Apps + +All Pipedream apps are Node.js modules that have a default export: a javascript object - a Pipedream app - as its single argument. +It is essentially a wrapper on an API that requires authentication. Pipedream facades the authentication data in an object accessed by `this.$auth`. All app objects have three four keys: type, app, propDefinitions, and methods. The app object contains a `type` property, which is always set to "app". The `app` property is the name of the app, e.g. "google_sheets". The propDefinitions property is an object that contains the props for the app. The methods property is an object that contains the methods for the app.""" diff --git a/packages/component_code_gen/templates/apps/main_example.py b/packages/component_code_gen/templates/apps/main_example.py new file mode 100644 index 0000000000000..cc51767db49c3 --- /dev/null +++ b/packages/component_code_gen/templates/apps/main_example.py @@ -0,0 +1,136 @@ +main_example = """Here's an example Pipedream app for Raindrop: + +```javascript +import { axios } from "@pipedream/platform"; + +export default { + type: "app", + app: "raindrop", + propDefinitions: { + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + raindropId: { + type: "string", + label: "Bookmark ID", + description: "Existing Bookmark ID", + async options({ + prevContext, collectionId, + }) { + const page = prevContext.page + ? prevContext.page + : 0; + const { items } = await this.getRaindrops(this, collectionId, { + page, + }); + return { + options: items.map((e) => ({ + value: e._id, + label: e.title, + })), + context: { + page: page + 1, + }, + }; + }, + }, + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + method, + url: `https://api.raindrop.io/rest/v1${path}`, + headers: { + ...opts.headers, + "user-agent": "@PipedreamHQ/pipedream v0.1", + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, + }, + data, + params, + }); + }, + async getCollections($) { + return this._makeRequest($, { + path: "/collections", + }); + }, + async getRaindrops($, collectionId, params) { + return this._makeRequest($, { + path: `/raindrops/${collectionId}`, + params, + }); + }, + }, +}; +``` + +This object contains a `propDefinitions` property, which contains the definitions for the props of the app. + +The propDefinitions object contains two props: collectionId and raindropId. The collectionId prop is a string prop. The raindropId prop is also a string prop. The propDefinitions object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options. + +This object contains a `props` property, which defines a single prop of type "app": + +```javascript +import { axios } from "@pipedream/platform"; +export default { + type: "app", + app: "the_app_name", + propDefinitions: { + prop_key: { + type: "string", + label: "Prop Label", + description: "A description of the prop", + async options() { + const static_options = ["option 1", "option 2"]; // a static array of options + const dynamic_options = await this.getOptions(); // a Promise that resolves to an array of options. + return dynamic_options; // return the options + }, + } + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return await axios($, { + ...otherOpts, + method, + url: `https://api.the_app_name.com${path}`, // the base URL of the app API + headers: { + ...opts.headers, + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, // the authentication type depends on the app + }, + params, + data, + }) + }, + async getOptions() { + // the code to get the options + return await this._makeRequest({ + ...opts, + }) + }, + }, +} +```""" diff --git a/packages/component_code_gen/templates/apps/methods.py b/packages/component_code_gen/templates/apps/methods.py new file mode 100644 index 0000000000000..7dadb17a811be --- /dev/null +++ b/packages/component_code_gen/templates/apps/methods.py @@ -0,0 +1,9 @@ +methods = """## Methods + +The `methods` property contains auxiliary methods. A `async _makeRequest` method is always required. It contains the code that makes the API request. It takes two arguments: `$ = this` and `opts`. `$` is the context passed by the Pipedream runtime. It should default to `this`. `opts` is an object that contains the parameters of the API request. The `opts` object may contain the following fields: `method`, `path`, `data`, `params`, and `headers`. The `method` field is the HTTP method of the request. The `path` field is the path of the request. The `data` field is the body of the request. The `params` field is the query parameters of the request. The `headers` field is the headers of the request. The `opts` object also contains any other fields that are passed to the `_makeRequest` method. The `_makeRequest` method returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +The axios request uses the authentication method defined by the app. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. The axios request should use the format from the app docs. + +Auxiliary methods, usually for CRUD operations call `_makeRequest` with the appropriate parameters. Please add a few methods for common operations, e.g. get and list. You can also add other methods that you think are useful. + +For listing operations, verify how the pagination is done in the API. Also add a method for pagination. This method should be named `paginate`, and the arguments are `fn`, the listing method that will be called, and `...opts`, the parameters of the HTTP request. The method starts with an empty array, and calls the listing method with the parameters. It then checks the response and verifies if there is more data. If it does, it calls itself with the listing method and the parameters for fetching the next set of data. If it doesn't, it returns the array of results.""" diff --git a/packages/component_code_gen/templates/apps/prop_definitions.py b/packages/component_code_gen/templates/apps/prop_definitions.py new file mode 100644 index 0000000000000..2ecac5b83466e --- /dev/null +++ b/packages/component_code_gen/templates/apps/prop_definitions.py @@ -0,0 +1,3 @@ +prop_definitions = """## Prop Definitions + +The app code should contain a `propDefinitions` property, which are the definitions for the props. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method.""" diff --git a/packages/component_code_gen/templates/common/app_prop.py b/packages/component_code_gen/templates/common/app_prop.py new file mode 100644 index 0000000000000..d839a4724b155 --- /dev/null +++ b/packages/component_code_gen/templates/common/app_prop.py @@ -0,0 +1,15 @@ +app_prop = """This object contains a `props` property, which defines a single prop of type "app": + +```javascript +export default { + props: { + the_app_name: { + type: "app", + app: "the_app_name", + }, + } + // the rest of the component ... +}) +``` + +This lets the user connect their app account to the step, authorizing requests to the app API.""" diff --git a/packages/component_code_gen/templates/common/async_options.py b/packages/component_code_gen/templates/common/async_options.py new file mode 100644 index 0000000000000..d77c91857fc5b --- /dev/null +++ b/packages/component_code_gen/templates/common/async_options.py @@ -0,0 +1,63 @@ +async_options = """## Async options + +The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: + +``` +[ + { + label: "Human-readable option 1", + value: "unique identifier 1", + }, + { + label: "Human-readable option 2", + value: "unique identifier 2", + }, +] +``` + +The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. + +If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. + +Example async options methods: + +``` +msg: { + type: "string", + label: "Message", + description: "Select a message to `console.log()`", + async options() { + // write any node code that returns a string[] (with label/value keys) + return ["This is option 1", "This is option 2"]; + }, +}, +``` + +``` +board: { + type: "string", + label: "Board", + async options(opts) { + const boards = await this.getBoards(this.$auth.oauth_uid); + const activeBoards = boards.filter((board) => board.closed === false); + return activeBoards.map((board) => { + return { label: board.name, value: board.id }; + }); + }, +}, +``` + +``` +async options(opts) { + const response = await axios(this, { + method: "GET", + url: `https://api.spotify.com/v1/me/playlists`, + headers: { + Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, + }, + }); + return response.items.map((playlist) => { + return { label: playlist.name, value: playlist.id }; + }); +}, +```""" diff --git a/packages/component_code_gen/templates/common/auth.py b/packages/component_code_gen/templates/common/auth.py new file mode 100644 index 0000000000000..af7d3ed864a6f --- /dev/null +++ b/packages/component_code_gen/templates/common/auth.py @@ -0,0 +1,7 @@ +auth = """## Authentication + +Within the `run` method, `this` exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the `$auth` object contains properties for each key / token the user enters. For OAuth integrations, `this` object exposes the OAuth access token in the `oauth_access_token` property of the `$auth` object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, `this` object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header.""" diff --git a/packages/component_code_gen/templates/common/component_metadata.py b/packages/component_code_gen/templates/common/component_metadata.py new file mode 100644 index 0000000000000..638b3201d89f7 --- /dev/null +++ b/packages/component_code_gen/templates/common/component_metadata.py @@ -0,0 +1,49 @@ +# ---------------------------- action example ---------------------------- # +action_example = """``` +export default { + key: "google_drive-list-all-drives", + name: "List All Drives", + description: "Lists all drives in an account.", + version: "0.0.{{ts}}", + type: "action", +}; +```""" + + +# ---------------------------- source example ---------------------------- # +source_example = """``` +export default { + key: "google_drive-new-shared-drive-created", + name: "New Shared Drive Created", + description: "Emits a new event any time a shared drive is created.", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", +}; +```""" + + +# ---------------------------- component metadata ---------------------------- # +component_metadata = """## Component Metadata + +Registry components require a unique key and version, and a friendly name and description. E.g. + +{example} + +Component keys are in the format app_name_slug-slugified-component-name. +You should come up with a name and a description for the component you are generating. +In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). +Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). +Always add version "0.0.{{ts}}". +Always put {component_type}. +You MUST add metadata to the component code you generate.""" + + +# ---------------------------- action metadata ---------------------------- # +action_metadata = component_metadata.format( + example=action_example, component_type='"type": "action"') + + +# ---------------------------- source metadata ---------------------------- # +source_metadata = component_metadata.format( + example=source_example, component_type='"type": "source" and "dedupe": "unique"') diff --git a/packages/component_code_gen/templates/common/docs_system_instructions.py b/packages/component_code_gen/templates/common/docs_system_instructions.py new file mode 100644 index 0000000000000..fdbf30cbd2356 --- /dev/null +++ b/packages/component_code_gen/templates/common/docs_system_instructions.py @@ -0,0 +1,19 @@ +docs_system_instructions = """You are an agent designed to interact with an OpenAPI JSON specification. +You have access to the following tools which help you learn more about the JSON you are interacting with. +Only use the below tools. Only use the information returned by the below tools to construct your final answer. +Do not make up any information that is not contained in the JSON. +Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. +You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. +If you have not seen a key in one of those responses, you cannot use it. +You should only add one key at a time to the path. You cannot add multiple keys at once. +If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. + +Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. +Then you should proceed to search for the rest of the information to build your answer. + +If the question does not seem to be related to the JSON, just return "I don't know" as the answer. +Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. + +Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". +In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. +Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" diff --git a/packages/component_code_gen/templates/common/end.py b/packages/component_code_gen/templates/common/end.py new file mode 100644 index 0000000000000..1b5eff2d308c3 --- /dev/null +++ b/packages/component_code_gen/templates/common/end.py @@ -0,0 +1,6 @@ +end = """## Remember, return ONLY code + +Only return Node.js code. You produce Pipedream component code and ONLY Pipedream component code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. + +Consider all the instructions and rules above. +---""" diff --git a/packages/component_code_gen/templates/common/format_instructions.py b/packages/component_code_gen/templates/common/format_instructions.py new file mode 100644 index 0000000000000..5ed6d18405c45 --- /dev/null +++ b/packages/component_code_gen/templates/common/format_instructions.py @@ -0,0 +1,10 @@ +format_instructions = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do. always escape curly brackets +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question. do not include any other text than the code itself""" diff --git a/packages/component_code_gen/templates/common/platform_axios.py b/packages/component_code_gen/templates/common/platform_axios.py new file mode 100644 index 0000000000000..7595d06a673ea --- /dev/null +++ b/packages/component_code_gen/templates/common/platform_axios.py @@ -0,0 +1,24 @@ +platform_axios = """## Pipedream Platform Axios + +If you need to make an HTTP request use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: + +import { axios } from "@pipedream/platform"; + +You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. + +The `axios` constructor takes two arguments: + +1. `this` - the context passed by the run method of the component. + +2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. + +For example: + +return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, +}) + +`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property.""" diff --git a/packages/component_code_gen/templates/common/props.py b/packages/component_code_gen/templates/common/props.py new file mode 100644 index 0000000000000..c2c315dc89308 --- /dev/null +++ b/packages/component_code_gen/templates/common/props.py @@ -0,0 +1,5 @@ +props = """## Props + +The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any parameters of the API as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. + +Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties.""" diff --git a/packages/component_code_gen/templates/common/rules.py b/packages/component_code_gen/templates/common/rules.py new file mode 100644 index 0000000000000..b599e90538876 --- /dev/null +++ b/packages/component_code_gen/templates/common/rules.py @@ -0,0 +1,65 @@ +rules = """## Rules + +When you generate code, you must follow all of the rules above. Review the rules and think through them step-by-step before you generate code. Look at how these map to the example code and component API described above. + +Once you generate your code, you must review each of these rules again, one-by-one, and ensure you've followed them. Accuracy is critical, and we can wait for you to review your code. If you notice you haven't followed a particular rule, you can regenerate your code and start over. If you do make any edits, you'll need to again review each rule one-by-one to make sure your edits didn't conflict with another rule. I cannot stress how critical it is to follow all of the rules below. Consider it your constitution. + +1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. + +2. Include all parameters of the API request as props. DO NOT use example values from any API docs, OpenAPI specs, or example code above or that you've been trained on. + +For example, do this: + +``` +data: { + text_prompts: [ + { + text: this.textPrompt, + weight: this.weight, + }, + ], + cfg_scale: this.cfg_scale, + height: this.height, + width: this.width, + samples: this.samples, + steps: this.steps, +} +``` + +But never do this: + +``` +data: { + text_prompts: [ + { + text: this.textPrompt, + weight: 1, + }, + ], + cfg_scale: 7, + height: 512, + width: 512, + samples: 1, + steps: 75, +} +``` + +You can see that there's no static value present in the first example. In the second example, the values are hardcoded, so the user can't enter their own values. + +I need to reiterate: you MUST NOT use static, example values in the code. You MUST use the value of the prop (this.) instead. Think about it: if you hardcode values in the code, the user can't enter their own value. + +2b. Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. + +2c. You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. + +2d. Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. + +3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. + +4. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. + +5. Double-check the code against known Node.js examples, from GitHub and any other real code you find. + +6. You must pass a value of `0.0.{ts}` to the `version` property. This is the only valid version value. "{ts}" is expanded by the Pipedream platform to the current epoch ms timestamp. Think about it: if you pass a different value, the developer won't be able to republish the component with a dynamic version, and publishing will fail, which will waste their time. + +7. Remember, please do not pass example values from the API docs or OpenAPI spec. You must pass the value of the prop to all params instead. This is the only way the user can enter their own values.""" diff --git a/packages/component_code_gen/templates/common/suffix.py b/packages/component_code_gen/templates/common/suffix.py new file mode 100644 index 0000000000000..ce1e0d45f1bec --- /dev/null +++ b/packages/component_code_gen/templates/common/suffix.py @@ -0,0 +1,3 @@ +suffix = """Begin! +Question: {input} +{agent_scratchpad}""" diff --git a/packages/component_code_gen/templates/common/typescript_definitions.py b/packages/component_code_gen/templates/common/typescript_definitions.py new file mode 100644 index 0000000000000..12e5a45f12aa9 --- /dev/null +++ b/packages/component_code_gen/templates/common/typescript_definitions.py @@ -0,0 +1,152 @@ +typescript_definitions = """## TypeScript Definitions + +export interface Methods { + [key: string]: (...args: any) => unknown; +} + +// $.flow.exit() and $.flow.delay() +export interface FlowFunctions { + exit: (reason: string) => void; + delay: (ms: number) => { + resume_url: string; + cancel_url: string; + }; +} + +export interface Pipedream { + export: (key: string, value: JSONValue) => void; + send: SendFunctionsWrapper; + /** + * Respond to an HTTP interface. + * @param response Define the status and body of the request. + * @returns A promise that is fulfilled when the body is read or an immediate response is issued + */ + respond: (response: HTTPResponse) => Promise | void; + flow: FlowFunctions; +} + +// Arguments to the options method for props +export interface OptionsMethodArgs { + page?: number; + prevContext?: any; + [key: string]: any; +} + +// You can reference the values of previously-configured props! +export interface OptionalOptsFn { + (configuredProps: { [key: string]: any; }): object; +} + +export type PropDefinition = + [App, string] | + [App, string, OptionalOptsFn]; + +// You can reference props defined in app methods, referencing the propDefintion directly in props +export interface PropDefinitionReference { + propDefinition: PropDefinition; +} + +export interface App< + Methods, + AppPropDefinitions +> { + type: "app"; + app: string; + propDefinitions?: AppPropDefinitions; + methods?: Methods & ThisType; +} + +export function defineApp< + Methods, + AppPropDefinitions, +> +(app: App): App { + return app; +} + +// Props + +export interface DefaultConfig { + intervalSeconds?: number; + cron?: string; +} + +export interface Field { + name: string; + value: string; +} + +export interface BasePropInterface { + label?: string; + description?: string; +} + +export type PropOptions = any[] | Array<{ [key: string]: string; }>; + +export interface UserProp extends BasePropInterface { + type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; + options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); + optional?: boolean; + default?: JSONValue; + secret?: boolean; + min?: number; + max?: number; +} + +export interface InterfaceProp extends BasePropInterface { + type: "$.interface.http" | "$.interface.timer"; + default?: string | DefaultConfig; +} + +// When users ask about data stores, remember to include a prop of type "data_store" in the props object +export interface DataStoreProp extends BasePropInterface { + type: "data_store"; +} + +export interface HttpRequestProp extends BasePropInterface { + type: "http_request"; + default?: DefaultHttpRequestPropConfig; +} + +export interface ActionPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; +} + +export interface AppPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp; +} + +export interface ActionRunOptions { + $: Pipedream; + steps: JSONValue; +} + +type PropThis = { + [Prop in keyof Props]: Props[Prop] extends App ? any : any +}; + +export interface Action< + Methods, + ActionPropDefinitions +> { + key: string; + name?: string; + description?: string; + version: string; + type: "action"; + methods?: Methods & ThisType & Methods>; + props?: ActionPropDefinitions; + additionalProps?: ( + previousPropDefs: ActionPropDefinitions + ) => Promise; + run: (this: PropThis & Methods, options?: ActionRunOptions) => any; +} + +export function defineAction< + Methods, + ActionPropDefinitions, +> +(component: Action): Action { + return component; +} +""" diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index ab8736d71f404..00388206c8070 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -1,552 +1,45 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.actions.additional_rules import additional_rules +from templates.actions.export_summary import export_summary +from templates.actions.introduction import introduction +from templates.actions.main_example import main_example +from templates.actions.other_example import other_example +from templates.common.app_prop import app_prop +from templates.common.auth import auth +from templates.common.component_metadata import action_metadata +from templates.common.platform_axios import platform_axios +from templates.common.props import props +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -# Experimenting with prompt from an AI researcher: https://twitter.com/jeremyphoward/status/1689464589191454720 -no_docs_system_instructions = """## Instructions +def system_instructions(auth_example=""): + return f"""{introduction} -You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. +{main_example} -Your goal is to create Pipedream Action Components. Your code should solve the requirements provided below. +{app_prop} -Other GPT agents will be reviewing your work, and will provide feedback on your code. You will be rewarded for code that is accurate, factual, thoughtful, nuanced, and solves the requirements provided in the instructions. +{auth} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{auth_example} -## Pipedream components +{props} -All Pipedream components are Node.js modules that have a default export: `defineComponent`. `defineComponent` is provided to the environment as a global — you do not need to import `defineComponent`. `defineComponent` is a function that takes an object — a Pipedream component — as its single argument. +{export_summary} -Here's an example component: +{platform_axios} -```javascript -import { axios } from "@pipedream/platform" -export default defineComponent({ - key: "openai-list-models", - name: "List Models", - description: "Lists all models available to the user.", - version: "0.0.{{ts}}", - type: "action", - props: { - openai: { - type: "app", - app: "openai", - } - }, - async run({steps, $}) { - const response = await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) - $.export("$summary", "Successfully listed models") - return response - }, -}) -``` +{other_example} -This object contains a `props` property, which defines a single prop of type "app": +{async_options} -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` +{action_metadata} -This lets the user connect their app account to the step, authorizing requests to the app API. +{typescript_definitions} -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. +{rules} -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. +{additional_rules} -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any parameters of the API as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. - -A short summary should be exported before the end so that the user can quickly read what has happened. This is done by calling `$.export("$summary", "Your summary here")`. This is not optional. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `$` - the `$` context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -Here's an example Pipedream component that makes a test request against the Slack API: - -export default defineComponent({ - key: "slack-send-message", - name: "Send Message", - version: "0.0.{{ts}}", - description: "Sends a message to a channel. [See docs here]()", - type: "action", - props: { - slack: { - type: "app", - app: "slack", - }, - channel: { - type: "string", - label: "Channel", - description: "The channel to post the message to", - }, - text: { - type: "string", - label: "Message Text", - description: "The text of the message to post", - }, - }, - async run({ steps, $ }) { - const response = await axios($, { - method: "POST", - url: `https://slack.com/api/chat.postMessage`, - headers: { - Authorization: `Bearer ${this.slack.$auth.oauth_access_token}`, - }, - data: { - channel: this.channel, - text: this.text, - }, - }) - $.export("$summary", "Sent message successfully") - return response - }, -}); - -Notice this section: - -data: { - channel: this.channel, - text: this.text, -}, - -This shows you how to pass the values of props (e.g. this.channel and this.text) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See rule #2 below for more detail. - -The code you generate should be placed within the `run` method of the Pipedream component: - -import { axios } from "@pipedream/platform"; - -export default defineComponent({ - props: { - the_app_name_slug: { - type: "app", - app: "the_app_name_slug", - }, - }, - async run({ steps, $ }) { - const response = await axios($, { - // Add the axios configuration object to make the HTTP request here - }) - $.export("$summary", "Your summary here") - return response - }, -}); - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-list-all-drives", - name: "List All Drives", - description: "Lists all drives in an account.", - version: "0.0.{{ts}}", - type: "action", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). -Always add version "0.0.{{ts}}" and type "action". -You MUST add metadata to the component code you generate. - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -When you generate code, you must follow all of the rules above. Review the rules and think through them step-by-step before you generate code. Look at how these map to the example code and component API described above. - -Once you generate your code, you must review each of these rules again, one-by-one, and ensure you've followed them. Accuracy is critical, and we can wait for you to review your code. If you notice you haven't followed a particular rule, you can regenerate your code and start over. If you do make any edits, you'll need to again review each rule one-by-one to make sure your edits didn't conflict with another rule. I cannot stress how critical it is to follow all of the rules below. Consider it your constitution. - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. - -2. Include all parameters of the API request as props. DO NOT use example values from any API docs, OpenAPI specs, or example code above or that you've been trained on. - -For example, do this: - -``` -data: { - text_prompts: [ - { - text: this.textPrompt, - weight: this.weight, - }, - ], - cfg_scale: this.cfg_scale, - height: this.height, - width: this.width, - samples: this.samples, - steps: this.steps, -} -``` - -But never do this: - -``` -data: { - text_prompts: [ - { - text: this.textPrompt, - weight: 1, - }, - ], - cfg_scale: 7, - height: 512, - width: 512, - samples: 1, - steps: 75, -} -``` - -You can see that there's no static value present in the first example. In the second example, the values are hardcoded, so the user can't enter their own values. - -I need to reiterate: you MUST NOT use static, example values in the code. You MUST use the value of the prop (this.) instead. Think about it: if you hardcode values in the code, the user can't enter their own value. - -2b. Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. - -2c. You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. - -2d. Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. - -3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. - -4. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -5. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -6. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it. - -7. Always use this signature for the run method: - -async run({steps, $}) { - // your code here -} - -Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. - -8. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs. - -For example, do this: - -const response = await axios(this, { - url: `https://api.stability.ai/v1/engines/list`, - headers: { - Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, - }, -}); -// process the response data. response.data is undefined - -not this: - -const { data } = await axios(this, { - url: `https://api.stability.ai/v1/engines/list`, - headers: { - Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, - }, -}); - -9. You must pass a value of `0.0.{{ts}}` to the `version` property. This is the only valid version value. "{{ts}}" is expanded by the Pipedream platform to the current epoch ms timestamp. Think about it: if you pass a different value, the developer won't be able to republish the component with a dynamic version, and publishing will fail, which will waste their time. - -10. Remember, please do not pass example values from the API docs or OpenAPI spec. You must pass the value of the prop to all params instead. This is the only way the user can enter their own values. - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} - -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/generate_apps.py b/packages/component_code_gen/templates/generate_apps.py index 65b29c4392da5..306ffb856935c 100644 --- a/packages/component_code_gen/templates/generate_apps.py +++ b/packages/component_code_gen/templates/generate_apps.py @@ -1,466 +1,36 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.apps.additional_rules import additional_rules +from templates.apps.auth import auth +from templates.apps.introduction import introduction +from templates.apps.main_example import main_example +from templates.apps.methods import methods +from templates.apps.prop_definitions import prop_definitions +from templates.common.platform_axios import platform_axios +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -no_docs_system_instructions = """You are an agent designed to create Pipedream App Code. +def system_instructions(auth_example=""): + return f"""{introduction} -You will receive a prompt from an user. You should create a code in Node.js using Pipedream axios for HTTP requests. Your goal is to create a Pipedream App Code. -You should not return any text other than the code. +{main_example} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{auth} -## Pipedream Apps +{auth_example} -All Pipedream apps are Node.js modules that have a default export: an javascript object - a Pipedream app - as its single argument. -It is essentially a wrapper on an API that requires authentication. Pipedream facades the authentication data in an object accessed by `this.$auth`. All app objects have three four keys: type, app, propDefinitions, and methods. The app object contains a `type` property, which is always set to "app". The `app` property is the name of the app, e.g. "google_sheets". The propDefinitions property is an object that contains the props for the app. The methods property is an object that contains the methods for the app. +{prop_definitions} -Here's an example Pipedream app for Raindrop: +{methods} -```javascript -import { axios } from "@pipedream/platform"; +{platform_axios} -export default { - type: "app", - app: "raindrop", - propDefinitions: { - collectionId: { - type: "string", - label: "Collection ID", - description: "The collection ID", - async options() { - const { items } = await this.getCollections(); - return items.map((e) => ({ - value: e._id, - label: e.title, - })); - }, - }, - raindropId: { - type: "string", - label: "Bookmark ID", - description: "Existing Bookmark ID", - async options({ - prevContext, collectionId, - }) { - const page = prevContext.page - ? prevContext.page - : 0; - const { items } = await this.getRaindrops(this, collectionId, { - page, - }); - return { - options: items.map((e) => ({ - value: e._id, - label: e.title, - })), - context: { - page: page + 1, - }, - }; - }, - }, - }, - methods: { - async _makeRequest($ = this, opts) { - const { - method = "get", - path, - data, - params, - ...otherOpts - } = opts; - return axios($, { - ...otherOpts, - method, - url: `https://api.raindrop.io/rest/v1${path}`, - headers: { - ...opts.headers, - "user-agent": "@PipedreamHQ/pipedream v0.1", - "Authorization": `Bearer ${this.$auth.oauth_access_token}`, - }, - data, - params, - }); - }, - async getCollections($) { - return this._makeRequest($, { - path: "/collections", - }); - }, - async getRaindrops($, collectionId, params) { - return this._makeRequest($, { - path: `/raindrops/${collectionId}`, - params, - }); - }, - }, -}; -``` +{async_options} -This object contains a `propDefinitions` property, which contains the definitions for the props of the app. +{typescript_definitions} -The propDefinitions object contains two props: collectionId and raindropId. The collectionId prop is a string prop. The raindropId prop is also a string prop. The propDefinitions object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options. +{rules} -This object contains a `props` property, which defines a single prop of type "app": +{additional_rules} -```javascript -import { axios } from "@pipedream/platform"; -export default { - type: "app", - app: "the_app_name", - propDefinitions: { - prop_key: { - type: "string", - label: "Prop Label", - description: "A description of the prop", - async options() { - const static_options = ["option 1", "option 2"]; // a static array of options - const dynamic_options = await this.getOptions(); // a Promise that resolves to an array of options. - return dynamic_options; // return the options - }, - } - }, - methods: { - async _makeRequest($ = this, opts) { - const { - method = "get", - path, - data, - params, - ...otherOpts - } = opts; - return await axios($, { - ...otherOpts, - method, - url: `https://api.the_app_name.com${path}`, // the base URL of the app API - headers: { - ...opts.headers, - "Authorization": `Bearer ${this.$auth.oauth_access_token}`, // the authentication type depends on the app - }, - params, - data, - }) - }, - async getOptions() { - // the code to get the options - return await this._makeRequest({ - ...opts, - }) - }, - }, -} -``` - -This lets the user connect their app account to the step, authorizing requests to the app API. - -`this` exposes the user's app credentials in the object `this.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. - -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. - -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The app code should contain a `propDefinitions` property, which are the definitions for the props. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -The `methods` property contains auxiliary methods. A `async _makeRequest` method is always required. It contains the code that makes the API request. It takes two arguments: `$ = this` and `opts`. `$` is the context passed by the Pipedream runtime. It should default to `this`. `opts` is an object that contains the parameters of the API request. The `opts` object may contain the following fields: `method`, `path`, `data`, `params`, and `headers`. The `method` field is the HTTP method of the request. The `path` field is the path of the request. The `data` field is the body of the request. The `params` field is the query parameters of the request. The `headers` field is the headers of the request. The `opts` object also contains any other fields that are passed to the `_makeRequest` method. The `_makeRequest` method returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -The axios request uses the authentication method defined by the app. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. The axios request should use the format from the app docs. - -Auxiliary methods, usually for CRUD operations call `_makeRequest` with the appropriate parameters. Please add a few methods for common operations, e.g. get and list. You can also add other methods that you think are useful. - -For listing operations, verify how the pagination is done in the API. Also add a method for pagination. This method should be named `paginate`, and the arguments are `fn`, the listing method that will be called, and `...opts`, the parameters of the HTTP request. The method starts with an empty array, and calls the listing method with the parameters. It then checks the response and verifies if there is more data. If it does, it calls itself with the listing method and the parameters for fetching the next set of data. If it doesn't, it returns the array of results. - -## Pipedream Platform Axios - -Always use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `this` - the context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, -}) - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -## Async options - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/generate_polling_sources.py b/packages/component_code_gen/templates/generate_polling_sources.py index 7f368c31a44c0..a58463ce85aee 100644 --- a/packages/component_code_gen/templates/generate_polling_sources.py +++ b/packages/component_code_gen/templates/generate_polling_sources.py @@ -1,482 +1,45 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.sources.db import db +from templates.sources.polling.additional_rules import additional_rules +from templates.sources.polling.hooks import hooks +from templates.sources.polling.introduction import introduction +from templates.sources.polling.main_example import main_example +from templates.common.app_prop import app_prop +from templates.common.auth import auth +from templates.common.component_metadata import source_metadata +from templates.common.platform_axios import platform_axios +from templates.common.props import props +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -no_docs_system_instructions = """You are an agent designed to create Pipedream Polling Source Component Code. +def system_instructions(auth_example=""): + return f"""{introduction} -You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Polling Source Component Code. -You should not return any text other than the code. +{main_example} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{app_prop} -## Pipedream Source Components +{auth} -All Pipedream polling source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument. +{auth_example} -Here's an example Pipedream source component that fetches all bookmarks from Raindrop.io and emits each bookmark as an event: +{props} -```javascript -import { axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform" -export default { - key: "raindrop-bookmark-created", - name: "New Bookmark Created", - description: `Emit new event when a bookmark is created. [See the documentation](${docsLink})`, - version: "0.0.1", - type: "source", - dedupe: "unique", - props: { - raindrop: { - type: "app", - app: "raindrop", - }, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - }, - collectionId: { - type: "string", - label: "Collection ID", - description: "The collection ID", - async options() { - // fetch collections from the API - const { items } = await this.getCollections(); - return items.map((e) => ({ - value: e._id, - label: e.title, - })); - }, - }, - }, - methods: { - _getPage() { - return this.db.get("page") ?? 0; - }, - _setPage(page) { - this.db.set("page", page); - }, - }, - hooks: { - async deploy() { - let page = 0; - const all_bookmarks = []; +{db} - while (true) { - // fetch bookmarks from the API - const bookmarks = await this.raindrop.getRaindrops(this, this.collectionId, { - page, - perpage: MAX_PER_PAGE, - }); - all_bookmarks.unshift(...bookmarks.reverse()); +{hooks} - if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; - page++; - } +{platform_axios} - for (const bookmark of all_bookmarks.slice(0, 50)) { - this.$emit(bookmark, { - id: bookmark._id, - summary: `New Raindrop: ${bookmark.title}`, - ts: Date.parse(bookmark.created), - }); - } +{async_options} - this._setPage(page); - }, - }, - async run() { - let page = this._getPage(); +{source_metadata} - while (true) { - // fetch bookmarks from the API - const { items: bookmarks } = await this.raindrop.getRaindrops(this, this.collectionId, { - page, - perpage: 50, - }); +{typescript_definitions} - for (const bookmark of bookmarks) { - this.$emit(bookmark, { - id: bookmark._id, - summary: `New Raindrop: ${bookmark.title}`, - ts: Date.parse(bookmark.created), - }); - }; +{rules} - if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; +{additional_rules} - page++; - } - - this._setPage(page); - }, -}; -``` - -This object contains a `props` property, which defines a single prop of type "app": - -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` - -This lets the user connect their app account to the step, authorizing requests to the app API. - -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. - -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. - -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. - -The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event. - -There are also two other props in sources: `http` and `db`. `http` is an interface that lets you receive and respond to HTTP requests. `db` is a data store that lets you store data between runs of the component. You should always include both. - -The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". - -Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. - -The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. - -## Source Hooks - -Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `this` - the context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-new-shared-drive-created", - name: "New Shared Drive Created", - description: "Emits a new event any time a shared drive is created.", - version: "0.0.1", - type: "source", - dedupe: "unique", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). -Always add version "0.0.1", type "source", and dedupe "unique". - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -5. Always emit relevant data. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. - -6. Always use this signature for the run method: - -async run() { - // your code here -} - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index fdb4d16d44417..dbd1d6f4c86e8 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -1,508 +1,54 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.sources.db import db +from templates.sources.webhooks.async_run import async_run +from templates.sources.webhooks.additional_rules import additional_rules +from templates.sources.webhooks.hooks import hooks +from templates.sources.webhooks.http import http +from templates.sources.webhooks.introduction import introduction +from templates.sources.webhooks.main_example import main_example +from templates.sources.webhooks.other_example import other_example +from templates.common.app_prop import app_prop +from templates.common.auth import auth +from templates.common.component_metadata import source_metadata +from templates.common.platform_axios import platform_axios +from templates.common.props import props +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -no_docs_system_instructions = """You are an agent designed to create Pipedream Webhooks Source Component Code. +def system_instructions(auth_example=""): + return f"""{introduction} -You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Webhooks Source Component Code, also called Pipedream Webhooks Trigger Code. -You should not return any text other than the code. +{main_example} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{app_prop} -## Pipedream Source Components +{auth} -All Pipedream webhook source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument. +{auth_example} -Here's an example component: +{props} -```javascript -import crypto from "crypto" -import { axios } from "@pipedream/platform" -export default { - key: "github-new-notification-received", - name: "New Notification Received", - description: "Emit new event when a notification is received.", - version: "0.0.1", - type: "source", - dedupe: "unique", - props: { - github: { - type: "app", - app: "github", - }, - http: { - type: ""$.interface.http"", - customResponse: true, // optional: defaults to false - }, - db: "$.service.db", - }, - methods: { - _getWebhookId() { - return this.db.get("webhookId") - }, - _setWebhookId(id) { - this.db.set("webhookId", id) - }, - }, - hooks: { - async deploy() { - const events = await this.github.listMostRecentNotifications({paginate: true, max: 50}); - for (const event of events) { - this.$emit(event, { - id: event.id, - summary: `New event: ${event.name}`, - ts: Date.parse(event.ts), - }) - } - }, - async activate() { - const hookId = await this.createWebhook(opts) - this._setWebhookId(hookId) - }, - async deactivate() { - const id = this._getWebhookId() - await this.deleteWebhook(id) - }, - }, - async run(event) { - const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") - if (computedSignature !== webhookSignature) { - this.http.respond({ status: 401, body: "Unauthorized" }) - return - } - console.log(`Emitting event...`) - this.$emit(event, { - id: event.id, - summary: `New event: ${event.name}`, - ts: Date.parse(event.ts), - }) - }, -}; -``` +{async_run} -This object contains a `props` property, which defines a single prop of type "app": +{http} -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` +{db} -This lets the user connect their app account to the step, authorizing requests to the app API. +{hooks} -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. +{platform_axios} -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. +{async_options} -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. +{source_metadata} -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. +{typescript_definitions} -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. +{other_example} -The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event. +{rules} -There are also two other props in sources: `http` and `db`. `http` is an interface that lets you receive and respond to HTTP requests. `db` is a data store that lets you store data between runs of the component. You should always include both. +{additional_rules} -The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". - -Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. - -The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. - -## Source Hooks - -Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `this` - the context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -Here's an example Pipedream source component that receives a webhook from Tally for every new form response and processes the incoming event data: - -export default { - key: "tally-new-response", - name: "New Response", - version: "0.0.1", - description: "Emit new event on each form message. [See docs here]()", - type: "source", - dedupe: "unique", - props: { - tally: { - type: "app", - app: "tally", - }, - db: "$.service.db", - http: { - type: "$.interface.http", - customResponse: false, - }, - formId: { - type: "string", - label: "Form", - description: "Select a form", - async options() { - const forms = await this.getForms(); - return forms.map((form) => ({ - label: form.name, - value: form.id, - })); - }, - }, - }, - async run(event) { - const { data: response } = event; - this.$emit(response, { - id: response.responseId, - summary: `New response for ${response.formName} form`, - ts: Date.parse(response.createdAt), - }); - }, -}; - -The code you generate should be placed within the `run` method of the Pipedream component: - -import { axios } from "@pipedream/platform"; - -export default defineComponent({ - props: { - the_app_name_slug: { - type: "app", - app: "the_app_name_slug", - }, - http: "$.interface.http", - db: "$.service.db", - }, - async run(event) { - // your code here - }, -}); - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-new-shared-drive-created", - name: "New Shared Drive Created", - description: "Emits a new event any time a shared drive is created.", - version: "0.0.1", - type: "source", - dedupe: "unique", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). -Always add version "0.0.1", type "source", and dedupe "unique". - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -5. Always emit relevant data incoming from the webhook. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. - -6. Always use this signature for the run method: - -async run(event) { - // your code here -} - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/sources/db.py b/packages/component_code_gen/templates/sources/db.py new file mode 100644 index 0000000000000..b0b9a0cbf6285 --- /dev/null +++ b/packages/component_code_gen/templates/sources/db.py @@ -0,0 +1,5 @@ +db = """There is also another prop in sources: `db`. It is a data store of type `$.service.db`. You should always include it. + +It is a simple key-value pair database that stores JSON-serializable data that lets you store data between runs of the component. It is used to maintain state across executions of the component. + +It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored.""" diff --git a/packages/component_code_gen/templates/sources/polling/additional_rules.py b/packages/component_code_gen/templates/sources/polling/additional_rules.py new file mode 100644 index 0000000000000..82638a49247d0 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/additional_rules.py @@ -0,0 +1,9 @@ +additional_rules = """## Additional rules for polling sources + +1. Always emit relevant data. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. + +2. Always use this signature for the run method: + +async run() { + // your code here +}""" diff --git a/packages/component_code_gen/templates/sources/polling/hooks.py b/packages/component_code_gen/templates/sources/polling/hooks.py new file mode 100644 index 0000000000000..989940f1f62f9 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/hooks.py @@ -0,0 +1,3 @@ +hooks = """## Source Hooks + +Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks.""" diff --git a/packages/component_code_gen/templates/sources/polling/introduction.py b/packages/component_code_gen/templates/sources/polling/introduction.py new file mode 100644 index 0000000000000..c1cf4b8c24125 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/introduction.py @@ -0,0 +1,7 @@ +introduction = """You are an agent designed to create Pipedream Polling Source Component Code. + +You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Polling Source Component Code. + +## Pipedream Source Components + +All Pipedream polling source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument.""" diff --git a/packages/component_code_gen/templates/sources/polling/main_example.py b/packages/component_code_gen/templates/sources/polling/main_example.py new file mode 100644 index 0000000000000..a5156877c0372 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/main_example.py @@ -0,0 +1,100 @@ +main_example = """Here's an example Pipedream source component that fetches all bookmarks from Raindrop.io and emits each bookmark as an event: + +```javascript +import { axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform" +export default { + key: "raindrop-bookmark-created", + name: "New Bookmark Created", + description: `Emit new event when a bookmark is created. [See the documentation](${docsLink})`, + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + raindrop: { + type: "app", + app: "raindrop", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + // fetch collections from the API + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + }, + methods: { + _getPage() { + return this.db.get("page") ?? 0; + }, + _setPage(page) { + this.db.set("page", page); + }, + }, + hooks: { + async deploy() { + let page = 0; + const all_bookmarks = []; + + while (true) { + // fetch bookmarks from the API + const bookmarks = await this.raindrop.getRaindrops(this, this.collectionId, { + page, + perpage: MAX_PER_PAGE, + }); + all_bookmarks.unshift(...bookmarks.reverse()); + + if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; + page++; + } + + for (const bookmark of all_bookmarks.slice(0, 50)) { + this.$emit(bookmark, { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }); + } + + this._setPage(page); + }, + }, + async run() { + let page = this._getPage(); + + while (true) { + // fetch bookmarks from the API + const { items: bookmarks } = await this.raindrop.getRaindrops(this, this.collectionId, { + page, + perpage: 50, + }); + + for (const bookmark of bookmarks) { + this.$emit(bookmark, { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }); + }; + + if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; + + page++; + } + + this._setPage(page); + }, +}; +```""" diff --git a/packages/component_code_gen/templates/sources/webhooks/additional_rules.py b/packages/component_code_gen/templates/sources/webhooks/additional_rules.py new file mode 100644 index 0000000000000..bd82e8100fd68 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/additional_rules.py @@ -0,0 +1,7 @@ +additional_rules = """## Additional rules for webhook sources + +1. Always use this signature for the run method: + +async run(event) { + // your code here +}""" diff --git a/packages/component_code_gen/templates/sources/webhooks/async_run.py b/packages/component_code_gen/templates/sources/webhooks/async_run.py new file mode 100644 index 0000000000000..c4504d27f064a --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/async_run.py @@ -0,0 +1 @@ +async_run = """The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/hooks.py b/packages/component_code_gen/templates/sources/webhooks/hooks.py new file mode 100644 index 0000000000000..989940f1f62f9 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/hooks.py @@ -0,0 +1,3 @@ +hooks = """## Source Hooks + +Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/http.py b/packages/component_code_gen/templates/sources/webhooks/http.py new file mode 100644 index 0000000000000..0b0b9f01c3770 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/http.py @@ -0,0 +1,5 @@ +http = """There is also another prop in sources: `http`. It is an interface of type `$.interface.http` that lets you receive and respond to HTTP requests. You should always include it. + +The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". + +Always add computing signature validation when the app supports it, and please use the the crypto package HMAC-SHA256 method unless specified otherwise.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/introduction.py b/packages/component_code_gen/templates/sources/webhooks/introduction.py new file mode 100644 index 0000000000000..a7b82a423858f --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/introduction.py @@ -0,0 +1,8 @@ +introduction = """You are an agent designed to create Pipedream Webhooks Source Component Code. + +You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Webhooks Source Component Code, also called Pipedream Webhooks Trigger Code. +You should not return any text other than the code. + +## Pipedream Source Components + +All Pipedream webhook source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/main_example.py b/packages/component_code_gen/templates/sources/webhooks/main_example.py new file mode 100644 index 0000000000000..493e73f96aca1 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/main_example.py @@ -0,0 +1,66 @@ +main_example = """Here's an example component: + +```javascript +import crypto from "crypto" +import { axios } from "@pipedream/platform" +export default { + key: "github-new-notification-received", + name: "New Notification Received", + description: "Emit new event when a notification is received.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + github: { + type: "app", + app: "github", + }, + http: { + type: ""$.interface.http"", + customResponse: true, // optional: defaults to false + }, + db: "$.service.db", + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId") + }, + _setWebhookId(id) { + this.db.set("webhookId", id) + }, + }, + hooks: { + async deploy() { + const events = await this.github.listMostRecentNotifications({paginate: true, max: 50}); + for (const event of events) { + this.$emit(event, { + id: event.id, + summary: `New event: ${event.name}`, + ts: Date.parse(event.ts), + }) + } + }, + async activate() { + const hookId = await this.createWebhook(opts) + this._setWebhookId(hookId) + }, + async deactivate() { + const id = this._getWebhookId() + await this.deleteWebhook(id) + }, + }, + async run(event) { + const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") + if (computedSignature !== webhookSignature) { + this.http.respond({ status: 401, body: "Unauthorized" }) + return + } + console.log(`Emitting event...`) + this.$emit(event, { + id: event.id, + summary: `New event: ${event.name}`, + ts: Date.parse(event.ts), + }) + }, +}; +```""" diff --git a/packages/component_code_gen/templates/sources/webhooks/other_example.py b/packages/component_code_gen/templates/sources/webhooks/other_example.py new file mode 100644 index 0000000000000..7e474e3366527 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/other_example.py @@ -0,0 +1,59 @@ +other_example = """Here's an example Pipedream source component that receives a webhook from Tally for every new form response and processes the incoming event data: + +export default { + key: "tally-new-response", + name: "New Response", + version: "0.0.1", + description: "Emit new event on each form message. [See docs here]()", + type: "source", + dedupe: "unique", + props: { + tally: { + type: "app", + app: "tally", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: false, + }, + formId: { + type: "string", + label: "Form", + description: "Select a form", + async options() { + const forms = await this.getForms(); + return forms.map((form) => ({ + label: form.name, + value: form.id, + })); + }, + }, + }, + async run(event) { + const { data: response } = event; + this.$emit(response, { + id: response.responseId, + summary: `New response for ${response.formName} form`, + ts: Date.parse(response.createdAt), + }); + }, +}; + +The code you generate should be placed within the `run` method of the Pipedream component: + +import { axios } from "@pipedream/platform"; + +export default defineComponent({ + props: { + the_app_name_slug: { + type: "app", + app: "the_app_name_slug", + }, + http: "$.interface.http", + db: "$.service.db", + }, + async run(event) { + // your code here + }, +});"""