From db921fc85268583e9c7e8c4de5d48d1e766f1589 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 28 Jul 2023 09:30:57 -0300 Subject: [PATCH 01/44] add generate webhook sample --- .../generate_webhook_sample.py | 24 ++++++++++ .../templates/webhook_samples.py | 46 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 packages/component_code_gen/generate_webhook_sample.py create mode 100644 packages/component_code_gen/templates/webhook_samples.py diff --git a/packages/component_code_gen/generate_webhook_sample.py b/packages/component_code_gen/generate_webhook_sample.py new file mode 100644 index 0000000000000..29bf9a209fedb --- /dev/null +++ b/packages/component_code_gen/generate_webhook_sample.py @@ -0,0 +1,24 @@ +import os +import argparse +import templates.webhook_samples + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--app', '-a', help='the app_name_slug', required=True) + parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + + if args.verbose: + os.environ['DEBUG'] = '1' + + # this is here so that the DEBUG environment variable is set before the imports + from code_gen.generate_component_code import main + + result = main(args.app, args.prompt, templates.webhook_samples) + print(result) + + +if __name__ == '__main__': + main() diff --git a/packages/component_code_gen/templates/webhook_samples.py b/packages/component_code_gen/templates/webhook_samples.py new file mode 100644 index 0000000000000..159139e784213 --- /dev/null +++ b/packages/component_code_gen/templates/webhook_samples.py @@ -0,0 +1,46 @@ +no_docs_user_prompt = """I want a webhook example. %s. The app is %s.""" + +no_docs_system_instructions = """You are an agent that generates correct JSON data for a webhook event for a specific app. +The JSON object should have fictitious data, but should have the correct structure. +You should not return any text other than the JSON object.""" + +with_docs_system_instructions = """You are an agent designed to interact with an OpenAPI JSON specification. +Your goal is to return a JSON webhook object that is fired when a specific event happens. +You should not return any text other than the JSON object. + +You have access to the following tools which help you learn more about the JSON you are interacting with. +Only use the below tools. Only use the information returned by the below tools to construct your final answer. +Do not make up any information that is not contained in the JSON. +Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. +You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. +If you have not seen a key in one of those responses, you cannot use it. +You should only add one key at a time to the path. You cannot add multiple keys at once. +If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. + +Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. +Then you should proceed to search for the rest of the information to build your answer. + +If the question does not seem to be related to the JSON, just return "I don't know" as the answer. +Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. + +Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". +In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. +Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" + +suffix = """Begin! +Remember, DO NOT include any other text in your response other than the Pipedream Component code. +DO NOT return ``` or any other code formatting characters in your response. + +Question: {input} +{agent_scratchpad}""" + +format_instructions = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do. always espace curly brackets +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question. do not include any other text than the code itself""" From f910c8dd87f953cc51980ecafdbb6a1374c377db Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 1 Aug 2023 09:02:34 -0300 Subject: [PATCH 02/44] add generate webhook source --- .../generate_webhook_source.py | 24 + .../templates/generate_webhook_sources.py | 441 ++++++++++++++++++ 2 files changed, 465 insertions(+) create mode 100644 packages/component_code_gen/generate_webhook_source.py create mode 100644 packages/component_code_gen/templates/generate_webhook_sources.py diff --git a/packages/component_code_gen/generate_webhook_source.py b/packages/component_code_gen/generate_webhook_source.py new file mode 100644 index 0000000000000..2c773393493ed --- /dev/null +++ b/packages/component_code_gen/generate_webhook_source.py @@ -0,0 +1,24 @@ +import os +import argparse +import templates.generate_webhook_sources + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--app', '-a', help='the app_name_slug', required=True) + parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + + if args.verbose: + os.environ['DEBUG'] = '1' + + # this is here so that the DEBUG environment variable is set before the imports + from code_gen.generate_component_code import main + + result = main(args.app, args.prompt, templates.generate_webhook_sources) + print(result) + + +if __name__ == '__main__': + main() diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py new file mode 100644 index 0000000000000..143c0620c5a50 --- /dev/null +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -0,0 +1,441 @@ +no_docs_user_prompt = """%s. The app is %s.""" + +no_docs_system_instructions = """You are an agent designed to create Pipedream Webhooks Source Component Code. + +You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Webhooks Source Component Code, also called Pipedream Webhooks Trigger Code. +You should not return any text other than the code. + +output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! + +## Pipedream Source Components + +All Pipedream webhook source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument. + +Here's an example component: + +```javascript +export default { + type: "source", + props: { + github: { + type: "app", + app: "github", + }, + http: { + type: ""$.interface.http"", + customResponse: true, // optional: defaults to false + }, + db: "$.service.db", + }, + async run(event) { + console.log(`Emitting event...`); + this.$emit(event, { + id: event.id, + summary: `New event: ${event.name}`, + ts: Date.parse(event.ts), + }); + }, +}; +``` + +This object contains a `props` property, which defines a single prop of type "app": + +```javascript +export default defineComponent({ + props: { + the_app_name: { + type: "app", + app: "the_app_name", + }, + } + // the rest of the component ... +}) +``` + +This lets the user connect their app account to the step, authorizing requests to the app API. + +Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. + +The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. + +Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. + +The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event. + +There are also two other props in sources: `http` and `db`. `http` is an interface that lets you receive and respond to HTTP requests. `db` is a data store that lets you store data between runs of the component. You should always include both. + +The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". + +The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. + +## Pipedream Platform Axios + +If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: + +import { axios } from "@pipedream/platform"; + +You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. + +The `axios` constructor takes two arguments: + +1. `this` - the context passed by the run method of the component. + +2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. + +For example: + +async run({steps, $}) { + return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, + }) +}, + +`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +Here's an example Pipedream source component that receives a webhook from Tally for every new form response and processes the incoming event data: + +export default { + key: "tally-new-response", + name: "New Response", + version: "0.0.1", + description: "Emit new event on each form message. [See docs here]()", + type: "source", + dedupe: "unique", + props: { + tally: { + type: "app", + app: "tally", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: false, + }, + formId: { + type: "string", + label: "Form", + description: "Select a form", + async options() { + const forms = await this.getForms(); + return forms.map((form) => ({ + label: form.name, + value: form.id, + })); + }, + }, + }, + async run(event) { + const { data: response } = event; + this.$emit(response, { + id: response.responseId, + summary: `New response for ${response.formName} form`, + ts: Date.parse(response.createdAt), + }); + }, +}; + +The code you generate should be placed within the `run` method of the Pipedream component: + +import { axios } from "@pipedream/platform"; + +export default defineComponent({ + props: { + the_app_name_slug: { + type: "app", + app: "the_app_name_slug", + }, + http: "$.interface.http", + db: "$.service.db", + }, + async run(event) { + // your code here + }, +}); + +## Async options props + +The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: + +``` +[ + { + label: "Human-readable option 1", + value: "unique identifier 1", + }, + { + label: "Human-readable option 2", + value: "unique identifier 2", + }, +] +``` + +The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. + +If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. + +Example async options methods: + +``` +msg: { + type: "string", + label: "Message", + description: "Select a message to `console.log()`", + async options() { + // write any node code that returns a string[] (with label/value keys) + return ["This is option 1", "This is option 2"]; + }, +}, +``` + +``` +board: { + type: "string", + label: "Board", + async options(opts) { + const boards = await this.getBoards(this.$auth.oauth_uid); + const activeBoards = boards.filter((board) => board.closed === false); + return activeBoards.map((board) => { + return { label: board.name, value: board.id }; + }); + }, +}, +``` + +``` +async options(opts) { + const response = await axios(this, { + method: "GET", + url: `https://api.spotify.com/v1/me/playlists`, + headers: { + Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, + }, + }); + return response.items.map((playlist) => { + return { label: playlist.name, value: playlist.id }; + }); +}, +``` + +## TypeScript Definitinos + +export interface Methods { + [key: string]: (...args: any) => unknown; +} + +// $.flow.exit() and $.flow.delay() +export interface FlowFunctions { + exit: (reason: string) => void; + delay: (ms: number) => { + resume_url: string; + cancel_url: string; + }; +} + +export interface Pipedream { + export: (key: string, value: JSONValue) => void; + send: SendFunctionsWrapper; + /** + * Respond to an HTTP interface. + * @param response Define the status and body of the request. + * @returns A promise that is fulfilled when the body is read or an immediate response is issued + */ + respond: (response: HTTPResponse) => Promise | void; + flow: FlowFunctions; +} + +// Arguments to the options method for props +export interface OptionsMethodArgs { + page?: number; + prevContext?: any; + [key: string]: any; +} + +// You can reference the values of previously-configured props! +export interface OptionalOptsFn { + (configuredProps: { [key: string]: any; }): object; +} + +export type PropDefinition = + [App, string] | + [App, string, OptionalOptsFn]; + +// You can reference props defined in app methods, referencing the propDefintion directly in props +export interface PropDefinitionReference { + propDefinition: PropDefinition; +} + +export interface App< + Methods, + AppPropDefinitions +> { + type: "app"; + app: string; + propDefinitions?: AppPropDefinitions; + methods?: Methods & ThisType; +} + +export function defineApp< + Methods, + AppPropDefinitions, +> +(app: App): App { + return app; +} + +// Props + +export interface DefaultConfig { + intervalSeconds?: number; + cron?: string; +} + +export interface Field { + name: string; + value: string; +} + +export interface BasePropInterface { + label?: string; + description?: string; +} + +export type PropOptions = any[] | Array<{ [key: string]: string; }>; + +export interface UserProp extends BasePropInterface { + type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; + options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); + optional?: boolean; + default?: JSONValue; + secret?: boolean; + min?: number; + max?: number; +} + +export interface InterfaceProp extends BasePropInterface { + type: "$.interface.http" | "$.interface.timer"; + default?: string | DefaultConfig; +} + +// When users ask about data stores, remember to include a prop of type "data_store" in the props object +export interface DataStoreProp extends BasePropInterface { + type: "data_store"; +} + +export interface HttpRequestProp extends BasePropInterface { + type: "http_request"; + default?: DefaultHttpRequestPropConfig; +} + +export interface ActionPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; +} + +export interface AppPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp; +} + +export interface ActionRunOptions { + $: Pipedream; + steps: JSONValue; +} + +type PropThis = { + [Prop in keyof Props]: Props[Prop] extends App ? any : any +}; + +export interface Action< + Methods, + ActionPropDefinitions +> { + key: string; + name?: string; + description?: string; + version: string; + type: "action"; + methods?: Methods & ThisType & Methods>; + props?: ActionPropDefinitions; + additionalProps?: ( + previousPropDefs: ActionPropDefinitions + ) => Promise; + run: (this: PropThis & Methods, options?: ActionRunOptions) => any; +} + +export function defineAction< + Methods, + ActionPropDefinitions, +> +(component: Action): Action { + return component; +} + +## Additional rules + +1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. + +2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. + +3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. + +4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. + +5. Always emit relevant data incoming from the webhook. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. + +6. Always use this signature for the run method: + +async run(event) { + // your code here +} + +## Remember, return ONLY code + +Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. +--- + +Your code: +""" + +with_docs_system_instructions = f"""{no_docs_system_instructions} +You are an agent designed to interact with an OpenAPI JSON specification. +You have access to the following tools which help you learn more about the JSON you are interacting with. +Only use the below tools. Only use the information returned by the below tools to construct your final answer. +Do not make up any information that is not contained in the JSON. +Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. +You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. +If you have not seen a key in one of those responses, you cannot use it. +You should only add one key at a time to the path. You cannot add multiple keys at once. +If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. + +Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. +Then you should proceed to search for the rest of the information to build your answer. + +If the question does not seem to be related to the JSON, just return "I don't know" as the answer. +Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. + +Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". +In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. +Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" + +suffix = """Begin! +Remember, DO NOT include any other text in your response other than the code. +DO NOT return ``` or any other code formatting characters in your response. + +Question: {input} +{agent_scratchpad}""" + +format_instructions = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do. always escape curly brackets +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question. do not include any other text than the code itself""" From 6fda90f12f2914b37186475e69522ecfd0731a0a Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 1 Aug 2023 12:55:07 -0300 Subject: [PATCH 03/44] modify logic for main --- .../component_code_gen/generate_action.py | 30 ++++++++++------ .../generate_component_code.py | 36 +++++++++++++++++++ .../generate_webhook_sample.py | 30 ++++++++++------ .../generate_webhook_source.py | 30 ++++++++++------ 4 files changed, 96 insertions(+), 30 deletions(-) create mode 100644 packages/component_code_gen/generate_component_code.py diff --git a/packages/component_code_gen/generate_action.py b/packages/component_code_gen/generate_action.py index 8ef7470708dd7..760523a97ea0c 100644 --- a/packages/component_code_gen/generate_action.py +++ b/packages/component_code_gen/generate_action.py @@ -4,24 +4,34 @@ import templates.transform_to_action -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() +def main(app, prompt, verbose=False): + validate_inputs(app, prompt) - if args.verbose: + if verbose: os.environ['DEBUG'] = '1' # this is here so that the DEBUG environment variable is set before the imports from code_gen.generate_component_code import main from code_gen.transform_code import transform - code = main(args.app, args.prompt, templates.generate_actions) + code = main(app, prompt, templates.generate_actions) result = transform(code, templates.transform_to_action) - print(result) + return result + + +def validate_inputs(app, prompt): + if not (bool(app) and bool(prompt)): + raise Exception('app and prompt are required') + + if type(app) != str and type(prompt) != str: + raise Exception('app and prompt should be strings') if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--app', '-a', help='the app_name_slug', required=True) + parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + result = main(args.app, args.prompt, args.verbose) + print(result) diff --git a/packages/component_code_gen/generate_component_code.py b/packages/component_code_gen/generate_component_code.py new file mode 100644 index 0000000000000..f3da238ee6b81 --- /dev/null +++ b/packages/component_code_gen/generate_component_code.py @@ -0,0 +1,36 @@ +import argparse +import generate_action +import generate_webhook_sample +import generate_webhook_source + + +def main(action, app, prompt, verbose=False): + validate_inputs(app, prompt) + result = actions[action](app, prompt, verbose) + return result + + +def validate_inputs(app, prompt): + if not (bool(app) and bool(prompt)): + raise Exception('app and prompt are required') + + if type(app) != str and type(prompt) != str: + raise Exception('app and prompt should be strings') + + +actions = { + 'generate_action': generate_action.main, + 'generate_webhook_sample': generate_webhook_sample.main, + 'generate_webhook_source': generate_webhook_source.main, +} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--action', help='which kind of code you want to generate?', choices=actions.keys(), required=True) + parser.add_argument('--app', help='the app_name_slug', required=True) + parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + result = main(args.action, args.app, args.prompt, args.verbose) + print(result) diff --git a/packages/component_code_gen/generate_webhook_sample.py b/packages/component_code_gen/generate_webhook_sample.py index 29bf9a209fedb..33df4dd3ee12e 100644 --- a/packages/component_code_gen/generate_webhook_sample.py +++ b/packages/component_code_gen/generate_webhook_sample.py @@ -3,22 +3,32 @@ import templates.webhook_samples -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() +def main(app, prompt, verbose=False): + validate_inputs(app, prompt) - if args.verbose: + if verbose: os.environ['DEBUG'] = '1' # this is here so that the DEBUG environment variable is set before the imports from code_gen.generate_component_code import main - result = main(args.app, args.prompt, templates.webhook_samples) - print(result) + result = main(app, prompt, templates.webhook_samples) + return result + + +def validate_inputs(app, prompt): + if not (bool(app) and bool(prompt)): + raise Exception('app and prompt are required') + + if type(app) != str and type(prompt) != str: + raise Exception('app and prompt should be strings') if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--app', '-a', help='the app_name_slug', required=True) + parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + result = main(args.app, args.prompt, args.verbose) + print(result) diff --git a/packages/component_code_gen/generate_webhook_source.py b/packages/component_code_gen/generate_webhook_source.py index 2c773393493ed..38fa5f6f71358 100644 --- a/packages/component_code_gen/generate_webhook_source.py +++ b/packages/component_code_gen/generate_webhook_source.py @@ -3,22 +3,32 @@ import templates.generate_webhook_sources -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() +def main(app, prompt, verbose=False): + validate_inputs(app, prompt) - if args.verbose: + if verbose: os.environ['DEBUG'] = '1' # this is here so that the DEBUG environment variable is set before the imports from code_gen.generate_component_code import main - result = main(args.app, args.prompt, templates.generate_webhook_sources) - print(result) + result = main(app, prompt, templates.generate_webhook_sources) + return result + + +def validate_inputs(app, prompt): + if not (bool(app) and bool(prompt)): + raise Exception('app and prompt are required') + + if type(app) != str and type(prompt) != str: + raise Exception('app and prompt should be strings') if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--app', '-a', help='the app_name_slug', required=True) + parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + result = main(args.app, args.prompt, args.verbose) + print(result) From 1a722e8836b43644d93ee7a75d52feaf35fc496d Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 1 Aug 2023 09:03:56 -0300 Subject: [PATCH 04/44] add tests --- packages/component_code_gen/.gitignore | 1 + .../reference/accelo-new-task-assigned.mjs | 69 ++++++++++ .../webhooks/reference/asana-new-project.mjs | 87 ++++++++++++ .../reference/fibery-entity-created.mjs | 81 +++++++++++ .../webhooks/reference/github-new-commit.mjs | 130 ++++++++++++++++++ .../postmark-new-inbound-email-received.mjs | 52 +++++++ .../process_street-workflow-run.completed.mjs | 90 ++++++++++++ .../reference/quaderno-payment-received.mjs | 117 ++++++++++++++++ .../shipcloud-new-shipment-status.mjs | 89 ++++++++++++ .../webhooks/reference/stripe-new-payment.mjs | 88 ++++++++++++ .../webhooks/reference/tally-new-response.mjs | 69 ++++++++++ .../webhooks/reference/twilio-new-call.mjs | 125 +++++++++++++++++ .../reference/woocommerce-new-order-event.mjs | 116 ++++++++++++++++ .../reference/zenkit-new-notification.mjs | 76 ++++++++++ .../component_code_gen/tests/webhooks/test.sh | 31 +++++ 15 files changed, 1221 insertions(+) create mode 100644 packages/component_code_gen/tests/webhooks/reference/accelo-new-task-assigned.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/asana-new-project.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/fibery-entity-created.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/github-new-commit.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/postmark-new-inbound-email-received.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/process_street-workflow-run.completed.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/quaderno-payment-received.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/shipcloud-new-shipment-status.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/stripe-new-payment.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/tally-new-response.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/twilio-new-call.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/woocommerce-new-order-event.mjs create mode 100644 packages/component_code_gen/tests/webhooks/reference/zenkit-new-notification.mjs create mode 100755 packages/component_code_gen/tests/webhooks/test.sh diff --git a/packages/component_code_gen/.gitignore b/packages/component_code_gen/.gitignore index 3e4c818783dec..6065c74698e1a 100644 --- a/packages/component_code_gen/.gitignore +++ b/packages/component_code_gen/.gitignore @@ -3,3 +3,4 @@ ve/ node_modules/ __pycache__/ +tests/**/output diff --git a/packages/component_code_gen/tests/webhooks/reference/accelo-new-task-assigned.mjs b/packages/component_code_gen/tests/webhooks/reference/accelo-new-task-assigned.mjs new file mode 100644 index 0000000000000..1b39f7c924835 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/accelo-new-task-assigned.mjs @@ -0,0 +1,69 @@ +export default { + key: "accelo-new-task-assigned", + name: "New Task Assigned (Instant)", + description: "Emit new event on each new task assigned.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + accelo: { + type: "app", + app: "accelo", + }, + db: "$.service.db", + http: "$.interface.http", + }, + hooks: { + async deploy() { + const { response: tasks } = await this.accelo.getTasks({ + params: { + _filters: "order_by_desc(date_created)", + _limit: 10, + }, + }); + + for (const task of tasks.slice(0, 10).reverse()) { + await this.emitEvent(task); + } + }, + async activate() { + const { response } = await this.accelo.createWebhook({ + data: { + trigger_url: this.http.endpoint, + event_id: this.getWebhookEventType(), + }, + }); + + this._setWebhookId(response.subscription.subscription_id); + }, + async deactivate() { + const webhookId = this._getWebhookId(); + await this.accelo.removeWebhook(webhookId); + }, + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId"); + }, + _setWebhookId(webhookId) { + this.db.set("webhookId", webhookId); + }, + getWebhookEventType() { + return "assign_task"; + }, + async emitEvent(data) { + const task = await this.accelo.getTask({ + taskId: data.id, + }); + + this.$emit(task, { + id: data.id, + summary: `New task assigned with ID ${data.id}`, + ts: Date.parse(data.date_created), + }); + }, + }, + async run(event) { + await this.emitEvent(event.body); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/asana-new-project.mjs b/packages/component_code_gen/tests/webhooks/reference/asana-new-project.mjs new file mode 100644 index 0000000000000..128abfcf3b08b --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/asana-new-project.mjs @@ -0,0 +1,87 @@ +export default { + key: "asana-new-project", + name: "New Project Added To Workspace (Instant)", + description: "Emit new event for each new project added to a workspace.", + version: "0.1.2", + type: "source", + dedupe: "unique", + props: { + asana: { + type: "app", + app: "asana", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: true, + }, + workspace: { + type: "string", + label: "Workspace", + description: "Gid of a workspace.", + }, + }, + hooks: { + async activate() { + const response = await this.asana.createWebHook({ + data: { + ...this.getWebhookFilter(), + target: this.http.endpoint, + }, + }); + + this._setWebhookId(response.gid); + }, + async deactivate() { + const webhookId = this._getWebhookId(); + await this.asana.deleteWebhook(webhookId); + }, + }, + methods: { + async _respondWebHook(http, event) { + http.respond({ + status: 200, + headers: { + "x-hook-secret": event.headers["x-hook-secret"], + }, + }); + }, + _getWebhookId() { + return this.db.get("webhookId"); + }, + _setWebhookId(webhookId) { + this.db.set("webhookId", webhookId); + }, + getWebhookFilter() { + return { + filters: [ + { + action: "added", + resource_type: "project", + }, + ], + resource: this.workspace, + }; + }, + async emitEvent(event) { + const { body } = event; + + if (!body || !body.events) return; + + for (const e of body.events) { + const project = await this.asana.getProject(e.resource.gid); + + this.$emit(project, { + id: project.gid, + summary: project.name, + ts: Date.now(), + }); + } + }, + }, + async run(event) { + await this._respondWebHook(this.http, event); + + await this.emitEvent(event); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/fibery-entity-created.mjs b/packages/component_code_gen/tests/webhooks/reference/fibery-entity-created.mjs new file mode 100644 index 0000000000000..e2760f4d0516f --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/fibery-entity-created.mjs @@ -0,0 +1,81 @@ +export default { + key: "fibery-entity-created", + name: "New Entity Created", + description: "Emit new event for every created entity of a certain type. [See the docs here](https://api.fibery.io/#webhooks)", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + fibery: { + type: "app", + app: "fibery", + }, + db: "$.service.db", + http: "$.interface.http", + type: { + type: "string", + label: "Type", + description: "A custom type in your Fibery account", + async options() { + const types = await this.listTypes(); + return types.map((t) => ({ + label: t["fibery/name"], + value: t["fibery/id"], + })); + }, + withLabel: true, + }, + }, + hooks: { + async deploy() { + const response = await this.fibery.listHistoricalEntities({ + type: this.type.label, + }); + response.result.forEach((entity) => { + this.$emit(entity, { + id: entity["fibery/id"], + summary: `Historical entity: ${this.getEntityId(entity)}`, + ts: entity["fibery/creation-date"], + }); + }); + }, + async activate() { + const response = await this.fibery.createWebhook({ + data: { + url: this.http.endpoint, + type: this.type.value, + }, + }); + this._setWebhookId(response.id); + }, + async deactivate() { + const webhookId = this._getWebhookId(); + await this.fibery.deleteWebhook({ + webhookId, + }); + }, + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId"); + }, + _setWebhookId(webhookId) { + this.db.set("webhookId", webhookId); + }, + getEntityId(entity) { + return entity["fibery/id"] || entity["id"]; + }, + }, + async run(event) { + console.log(`Received new event with ${event.body.effects.length} sequence(s)`); + event.body.effects + .filter(({ effect }) => effect === "fibery.entity/create") + .forEach((entity) => { + this.$emit(entity, { + id: entity.id, + summary: `New created entity: ${this.getEntityId(entity)}`, + ts: entity.values["fibery/creation-date"], + }); + }); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/github-new-commit.mjs b/packages/component_code_gen/tests/webhooks/reference/github-new-commit.mjs new file mode 100644 index 0000000000000..0f84276f543d0 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/github-new-commit.mjs @@ -0,0 +1,130 @@ +export default { + key: "github-new-commit", + name: "New Commit (Instant)", + description: "Emit new events on new commits to a repo or branch", + version: "0.1.9", + type: "source", + dedupe: "unique", + props: { + github: { + type: "app", + app: "github", + }, + http: "$.interface.http", + db: "$.service.db", + repoFullname: { + type: "string", + label: "Repository", + description: "The name of the repository. The name is not case sensitive", + async options({ org }) { + const repositories = await this.getRepos({ + org, + }); + + return repositories.map((repository) => repository.full_name); + }, + }, + branch: { + type: "string", + label: "Branch", + description: "Branch to monitor for new commits. Defaults to master", + optional: true, + withLabel: true, + async options({ + page, repoFullname, + }) { + const branches = await this.getBranches({ + repoFullname, + params: { + page: page + 1, + }, + }); + + return branches.map((branch) => ({ + label: branch.name, + value: `${branch.commit.sha}/${branch.name}`, + })); + }, + }, + }, + hooks: { + async deploy() { + if (this.branch) { + this.branch = { + label: this.branch.split("/")[1], + value: this.branch.split("/")[0], + }; + } + + const commitInfo = await this.github.getCommits({ + repoFullname: this.repoFullname, + sha: this.branch + ? this.branch.value + : undefined, + per_page: 25, + }); + const commits = commitInfo.map((info) => ({ + id: info.commit.url.split("/").pop(), + timestamp: info.commit.committer.date, + ...info.commit, + })); + this.processCommits(commits); + }, + async activate() { + const response = await this.github.createWebhook({ + repoFullname: this.repoFullname, + data: { + name: "web", + config: { + url: this.http.endpoint, + content_type: "json", + }, + events: this.getWebhookEvents(), + }, + }); + this._setWebhookId(response.id); + }, + async deactivate() { + const webhookId = this._getWebhookId(); + await this.github.removeWebhook({ + repoFullname: this.repoFullname, + webhookId, + }); + }, + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId"); + }, + _setWebhookId(webhookId) { + this.db.set("webhookId", webhookId); + }, + getWebhookEvents() { + return [ + "push", + ]; + }, + }, + async run(event) { + const { body } = event; + + // skip initial response from Github + if (body?.zen) { + console.log(body.zen); + return; + } + + const branch = body.ref.split("refs/heads/").pop(); + if (!(!this.branch || branch === this.branch.label)) { + return; + } + + for (const commit of body.commits) { + this.$emit(commit, { + id: commit.id, + summary: commit.message, + ts: Date.parse(commit.timestamp), + }); + } + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/postmark-new-inbound-email-received.mjs b/packages/component_code_gen/tests/webhooks/reference/postmark-new-inbound-email-received.mjs new file mode 100644 index 0000000000000..9f8b3c99d8649 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/postmark-new-inbound-email-received.mjs @@ -0,0 +1,52 @@ +export default { + key: "postmark-new-inbound-email-received", + name: "New Inbound Email Received", + description: "Emit new event when an email is received by the Postmark server [(See docs here)](https://postmarkapp.com/developer/webhooks/inbound-webhook)", + version: "0.0.1", + type: "source", + props: { + postmark: { + type: "app", + app: "postmark", + }, + http: { + type: "$.interface.http", + customResponse: true, + }, + }, + hooks: { + async activate() { + return this.postmark.setServerInfo({ + [this.getWebhookType()]: this.http.endpoint, + ...this.getWebhookProps(), + }); + }, + async deactivate() { + return this.postmark.setServerInfo({ + [this.getWebhookType()]: "", + }); + }, + }, + methods: { + getWebhookType() { + return "InboundHookUrl"; + }, + }, + async run(data) { + this.http.respond({ + status: 200, + }); + + let dateParam = data.ReceivedAt ?? data.Date ?? Date.now(); + let dateObj = new Date(dateParam); + + let msgId = data.MessageID; + let id = `${msgId}-${dateObj.toISOString()}`; + + this.$emit(data, { + id, + summary: data.Subject, + ts: dateObj.valueOf(), + }); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/process_street-workflow-run.completed.mjs b/packages/component_code_gen/tests/webhooks/reference/process_street-workflow-run.completed.mjs new file mode 100644 index 0000000000000..21d00bd170361 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/process_street-workflow-run.completed.mjs @@ -0,0 +1,90 @@ +export default { + key: "process_street-workflow-run-completed", + name: "Workflow Run Completed", + description: "Emit new event for every completed workflow", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + processStreet: { + type: "app", + app: "process_street", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + }, + workflowId: { + type: "string", + label: "Workflow ID", + description: "The ID of the Workflow", + async options() { + const { workflows } = await this.listWorkflows(); + return workflows.map((workflow) => ({ + label: workflow.name, + value: workflow.id, + })); + }, + optional: true, + }, + }, + hooks: { + async deploy() { + console.log("Retrieving workflow runs..."); + const { workflowRuns } = await this.processStreet.listWorkflowRuns({ + workflowId: this.workflowId, + }); + + const filteredWorkflowRuns = workflowRuns + .filter((workflowRun) => workflowRun.status === "Completed") + .slice(-25); + + for (const workflowRun of filteredWorkflowRuns) { + this.emitEvent(workflowRun); + } + }, + async activate() { + console.log("Creating webhook..."); + const webhookTypes = this.getWebhookTypes(); + const { id } = await this.processStreet.createWebhook({ + data: { + url: this.http.endpoint, + workflowId: this.workflowId, + triggers: webhookTypes, + }, + }); + this._setWebhookId(id); + }, + async deactivate() { + console.log("Deleting webhook..."); + await this.processStreet.deleteWebhook({ + id: this._getWebhookId(), + }); + }, + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId"); + }, + _setWebhookId(id) { + this.db.set("webhookId", id); + }, + getWebhookTypes() { + return [ + "WorkflowRunCompleted", + ]; + }, + emitEvent(data) { + this.$emit(data, { + id: data.id, + summary: `Completed workflow run: ${data.name}`, + ts: data.audit.updatedDate, + }); + }, + }, + async run(event) { + console.log("Webhook received"); + const data = event.body.data; + this.emitEvent(data); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/quaderno-payment-received.mjs b/packages/component_code_gen/tests/webhooks/reference/quaderno-payment-received.mjs new file mode 100644 index 0000000000000..97baa86f5019a --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/quaderno-payment-received.mjs @@ -0,0 +1,117 @@ +import { createHmac } from "crypto"; + +export default { + key: "quaderno-payment-received", + name: "New Payment Received", + description: "Emit new event when a payment is successfully processed in Quaderno. [See the Documentation](https://developers.quaderno.io/api/#tag/Webhooks/operation/createWebhook).", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + quaderno: { + type: "app", + app: "quaderno", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: true, + }, + }, + hooks: { + async activate() { + const response = + await this.createWebhook({ + data: { + url: this.http.endpoint, + events_types: this.getEventName(), + }, + }); + + this.setWebhookId(response.id); + this.setAuthKey(response.auth_key); + }, + async deactivate() { + const webhookId = this.getWebhookId(); + if (webhookId) { + await this.deleteWebhook({ + webhookId, + }); + } + }, + }, + methods: { + createWebhook(args = {}) { + return this.app.post({ + path: "/webhooks", + ...args, + }); + }, + deleteWebhook({ + webhookId, ...args + } = {}) { + return this.app.delete({ + path: `/webhooks/${webhookId}`, + ...args, + }); + }, + setWebhookId(value) { + this.db.set("webhookId", value); + }, + getWebhookId() { + return this.db.get("webhookId"); + }, + setAuthKey(value) { + this.db.set("authKey", value); + }, + getAuthKey() { + return this.db.get("authKey"); + }, + getEventName() { + return [ + "payment.created", + ]; + }, + generateMeta(resource) { + return { + id: resource.id, + summary: `New Payment: ${resource.id}`, + ts: Date.now(), + }; + }, + isSignatureValid(signature, data, skip = true) { + // skip signature validation for now. Due to the following issue: + // https://github.com/quaderno/quaderno-api/issues/54 + if (skip) { + return true; + } + const authKey = this.getAuthKey(); + const computedSignature = createHmac("sha1", authKey) + .update(data) + .digest("base64"); + + return computedSignature === signature; + }, + processEvent(event) { + this.$emit(event, this.generateMeta(event.data?.object || event)); + }, + }, + async run({ + method, url, body, headers, bodyRaw, + }) { + if (method === "HEAD") { + return this.http.respond({ + status: 200, + }); + } + + const signature = headers["x-quaderno-signature"]; + const data = `${url}${bodyRaw}`; + + if (!this.isSignatureValid(signature, data)) { + throw new Error("Invalid signature"); + } + + this.processEvent(body); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/shipcloud-new-shipment-status.mjs b/packages/component_code_gen/tests/webhooks/reference/shipcloud-new-shipment-status.mjs new file mode 100644 index 0000000000000..af86ca58680c0 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/shipcloud-new-shipment-status.mjs @@ -0,0 +1,89 @@ +export default { + key: "shipcloud-new-shipment-status", + name: "New Shipment Status", + description: "Emit new event for shipment status changes [See docs here](https://developers.shipcloud.io/reference/#webhooks)", + version: "0.0.1", + type: "source", + props: { + shipcloud: { + type: "app", + app: "shipcloud", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: true, + }, + eventTypes: { + label: "Event types", + description: "The shipment update(s) that should trigger an event", + type: "string[]", + options: [ + "shipment.*", + "shipment.status.*", + "shipment.status.deleted", + "shipment.tracking.*", + "shipment.tracking.awaits_pickup_by_receiver", + "shipment.tracking.canceled", + "shipment.tracking.delayed", + "shipment.tracking.delivered", + "shipment.tracking.destroyed", + "shipment.tracking.exception", + "shipment.tracking.label_created", + "shipment.tracking.not_delivered", + "shipment.tracking.notification", + "shipment.tracking.out_for_delivery", + "shipment.tracking.picked_up", + "shipment.tracking.transit", + "shipment.tracking.unknown", + ], + }, + }, + hooks: { + async activate() { + const data = { + event_types: this.eventTypes, + url: this.http.endpoint, + }; + + const { id } = await this.shipcloud.createHook(data); + + this.db.set("hookId", id); + }, + async deactivate() { + const id = this.db.get("hookId"); + + await this.shipcloud.deleteHook({ + id, + }); + }, + }, + async run(data) { + this.http.respond({ + status: 200, + }); + + const { body } = data; + + let { id } = body; + if (typeof id !== "string") { + id = Date.now(); + } + + let summary = body.type; + if (typeof summary !== "string") { + summary = "Unknown event type"; + } + + const date = body.occured_at; + const ts = typeof date === "string" + ? new Date(date).valueOf() + : Date.now(); + + this.$emit(body, { + id, + summary, + ts, + }); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/stripe-new-payment.mjs b/packages/component_code_gen/tests/webhooks/reference/stripe-new-payment.mjs new file mode 100644 index 0000000000000..8d351449260e2 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/stripe-new-payment.mjs @@ -0,0 +1,88 @@ +export default { + key: "stripe-new-payment", + name: "New Payment", + description: "Emit new event for each new payment", + version: "0.0.1", + type: "source", + props: { + stripe: { + type: "app", + app: "stripe", + }, + http: { + type: "$.interface.http", + customResponse: true, + }, + db: "$.service.db", + }, + hooks: { + async activate() { + let enabledEvents = this.getEvents(); + + if (enabledEvents.includes("*")) enabledEvents = [ + "*", + ]; + + const endpoint = await this.stripe.sdk().webhookEndpoints.create({ + url: this.http.endpoint, + enabled_events: enabledEvents, + }); + this.db.set("endpoint", JSON.stringify(endpoint)); + + for (const eventType of enabledEvents) { + const events = await this.stripe.getEvents({ + eventType, + }); + + for (const event of events) { + this.emitEvent(event); + } + } + }, + async deactivate() { + const endpoint = this.getEndpoint(); + this.db.set("endpoint", null); + if (!endpoint) return; + const confirmation = await this.stripe.sdk().webhookEndpoints.del(endpoint.id); + if ("deleted" in confirmation && !confirmation.deleted) { + throw new Error("Webhook endpoint not deleted"); + } + }, + }, + methods: { + getEvents() { + return [ + "payment_intent.created", + ]; + }, + }, + run(event) { + const endpoint = this.db.get("endpoint"); + if (!endpoint) { + this.http.respond({ + status: 500, + }); + throw new Error("Webhook endpoint config missing from db"); + } + const sig = event.headers["stripe-signature"]; + try { + event = this.stripe.sdk().webhooks.constructEvent(event.bodyRaw, sig, endpoint.secret); + } catch (err) { + this.http.respond({ + status: 400, + body: err.message, + }); + console.log(err.message); + return; + } + this.http.respond({ + status: 200, + }); + + this.$emit(event, { + id: event.id, + summary: `New event ${event.type} with ID ${event.data.id}`, + ts: Date.parse(event.created), + }); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/tally-new-response.mjs b/packages/component_code_gen/tests/webhooks/reference/tally-new-response.mjs new file mode 100644 index 0000000000000..55c324468b873 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/tally-new-response.mjs @@ -0,0 +1,69 @@ +export default { + key: "tally-new-response", + name: "New Response", + description: "Emit new event on each form message. [See docs here](https://tallyso.notion.site/Tally-OAuth-2-reference-d0442c679a464664823628f675f43454)", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + tally: { + type: "app", + app: "tally", + }, + formId: { + label: "Form", + description: "Select a form", + type: "string", + async options() { + const forms = await this.getForms(); + + return forms.map((form) => ({ + label: form.name, + value: form.id, + })); + }, + }, + db: "$.service.db", + http: "$.interface.http", + }, + hooks: { + async activate() { + const response = await this.tally.createWebhook({ + formId: this.formId, + url: this.http.endpoint, + eventTypes: this.getWebhookEventTypes(), + }); + + this._setWebhookId(response.id); + }, + async deactivate() { + const webhookId = this._getWebhookId(); + await this.tally.removeWebhook(webhookId); + }, + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId"); + }, + _setWebhookId(webhookId) { + this.db.set("webhookId", webhookId); + }, + getWebhookEventTypes() { + return [ + "FORM_RESPONSE", + ]; + }, + emitEvent(event) { + const { data: response } = event; + + this.$emit(response, { + id: response.responseId, + summary: `New response for ${response.formName} form`, + ts: response.createdAt, + }); + }, + }, + async run(event) { + this.emitEvent(event.body); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/twilio-new-call.mjs b/packages/component_code_gen/tests/webhooks/reference/twilio-new-call.mjs new file mode 100644 index 0000000000000..17d6438462070 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/twilio-new-call.mjs @@ -0,0 +1,125 @@ +export default { + key: "twilio-new-call", + name: "New Call (Instant)", + description: "Emit new event each time a call to the phone number is completed. Configures a webhook in Twilio, tied to a phone number.", + version: "0.1.3", + type: "source", + dedupe: "unique", + props: { + twilio: { + type: "app", + app: "twilio", + }, + incomingPhoneNumber: { + type: "string", + label: "Incoming Phone Number", + description: "The Twilio phone number where you'll receive messages. This source creates a webhook tied to this incoming phone number, **overwriting any existing webhook URL**.", + async options() { + const numbers = await this.listIncomingPhoneNumbers(); + return numbers.map((number) => { + return { + label: number.friendlyName, + value: number.sid, + }; + }); + }, + }, + authToken: { + type: "string", + secret: true, + label: "Twilio Auth Token", + description: "Your Twilio auth token, found [in your Twilio console](https://www.twilio.com/console). Required for validating Twilio events.", + }, + http: { + label: "HTTP Responder", + description: "Exposes a `respond()` method that lets the source issue HTTP responses", + type: "$.interface.http", + customResponse: true, + }, + }, + hooks: { + async activate() { + const createWebhookResp = await this.twilio.setWebhookURL({ + serviceType: this.getServiceType(), + phoneNumberSid: this.incomingPhoneNumber, + url: this.http.endpoint, + }); + console.log(createWebhookResp); + }, + async deactivate() { + // remove the webhook URL if url prop is not set + const deleteWebhookResp = await this.twilio.setWebhookURL({ + serviceType: this.getServiceType(), + phoneNumberSid: this.incomingPhoneNumber, + url: "", + }); + console.log(deleteWebhookResp); + }, + }, + methods: { + getServiceType() { + return "voice"; + }, + getResponseBody() { + return null; + }, + isRelevant(body) { + return body.CallStatus == "completed"; + }, + emitEvent(body, headers) { + this.$emit(body, { + /** if Twilio retries a message, but we've already emitted, dedupe */ + id: headers["i-twilio-idempotency-token"], + summary: `New call from ${this.getMaskedNumber(body.From)}`, + ts: Date.now(), + }); + }, + }, + async run(event) { + let { + body, + headers, + } = event; + + const responseBody = this.getResponseBody(); + if (responseBody) { + this.http.respond({ + status: 200, + headers: { + "Content-Type": "text/xml", + }, + body: responseBody, + }); + } + + if (typeof body !== "object") { + body = Object.fromEntries(new URLSearchParams(body)); + } + + if (!this.isRelevant(body)) { + console.log("Event not relevant. Skipping..."); + return; + } + + const signature = headers["x-twilio-signature"]; + if (!signature) { + console.log("No x-twilio-signature header in request. Exiting."); + return; + } + + // The url must match the incoming URL exactly, which contains a `/` at the end + const isRequestValid = this.twilio.validateRequest({ + signature, + url: `${this.http.endpoint}/`, + params: body, + authToken: this.authToken, + }); + + if (!isRequestValid) { + console.log("Event could not be validated. Skipping..."); + return; + } + + this.emitEvent(body, headers); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/woocommerce-new-order-event.mjs b/packages/component_code_gen/tests/webhooks/reference/woocommerce-new-order-event.mjs new file mode 100644 index 0000000000000..b4b93d47468e0 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/woocommerce-new-order-event.mjs @@ -0,0 +1,116 @@ +import crypto from "crypto"; + +export default { + key: "woocommerce-new-order-event", + name: "New Order Event (Instant)", + description: "Emit new event each time the specified order event(s) occur", + version: "0.0.3", + type: "source", + dedupe: "unique", + props: { + woocommerce: { + type: "app", + app: "woocommerce", + }, + db: "$.service.db", + http: "$.interface.http", + topics: { + type: "string[]", + label: "Event topics", + description: "Types of events to watch for", + options: [ + "created", + "updated", + "deleted", + ], + }, + }, + hooks: { + async deploy() { + const events = await this.getSampleEvents({ + perPage: 25, + }); + for (const event of events) { + const meta = this.generateMeta("", event); + this.$emit(event, meta); + } + }, + async activate() { + const hookIds = []; + for (const topicType of this.topics) { + const topic = this.getTopic(topicType); + const data = { + topic, + delivery_url: this.http.endpoint, + secret: this.woocommerce.$auth.secret, + }; + const { id } = await this.woocommerce.createWebhook(data); + hookIds.push(id); + } + this._setHookIds(hookIds); + }, + async deactivate() { + const hookIds = this._getHookIds(); + await Promise.all(hookIds.map(async (id) => await this.woocommerce.deleteWebhook(id))); + this._setHookIds(null); + }, + }, + methods: { + _getHookIds() { + return this.db.get("hookIds"); + }, + _setHookIds(hookIds) { + this.db.set("hookIds", hookIds); + }, + verifyWebhookRequest(bodyRaw, signature) { + const signatureComputed = crypto.createHmac("SHA256", this.woocommerce.$auth.secret) + .update(bodyRaw) + .digest("base64"); + return signatureComputed === signature; + }, + async getSampleEvents({ perPage }) { + return this.woocommerce.listOrders({ + per_page: perPage, + orderby: "date", + }); + }, + getTopic(topicType) { + return `order.${topicType}`; + }, + generateMeta(eventType, { + id, date_modified: dateModified, + }) { + const ts = Date.parse(dateModified); + return { + id: `${id}${ts}`, + summary: `Order ID: ${id} ${eventType}`, + ts, + }; + }, + }, + async run(event) { + const { + body, + bodyRaw, + headers, + } = event; + + // WooCommerce sends a request verifying the webhook that contains only the webhook_id. + // We can skip these requests. + if (body.webhook_id) { + return; + } + + // verify that the incoming webhook is valid + if (!this.verifyWebhookRequest(bodyRaw, headers["x-wc-webhook-signature"])) { + console.log("Could not verify incoming webhook signature"); + return; + } + + const eventType = headers["x-wc-webhook-event"]; + if (eventType) { + const meta = this.generateMeta(eventType, body); + this.$emit(body, meta); + } + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/reference/zenkit-new-notification.mjs b/packages/component_code_gen/tests/webhooks/reference/zenkit-new-notification.mjs new file mode 100644 index 0000000000000..ebc39d6c26bca --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/zenkit-new-notification.mjs @@ -0,0 +1,76 @@ +export default { + key: "zenkit-new-notification", + name: "New Notification (Instant)", + description: "Emit new event when there is a new notification in Zenkit", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + zenkit: { + type: "app", + app: "zenkit", + }, + http: "$.interface.http", + db: "$.service.db", + }, + hooks: { + async deploy() { + const events = await this.getHistoricalEvents({ + limit: 25, + }); + if (!events) { + return; + } + for (const event of events) { + this.emitEvent(event); + } + }, + async activate() { + const { id } = await this.zenkit.createWebhook({ + data: { + triggerType: 2, // notification + url: this.http.endpoint, + ...this.getWebhookParams(), + }, + }); + this._setHookId(id); + }, + async deactivate() { + const hookId = this._getHookId(); + await this.zenkit.deleteWebhook(hookId); + }, + }, + methods: { + async getHistoricalEvents(params) { + return this.zenkit.listNotifications({ + params, + }); + }, + _getHookId() { + return this.db.get("hookId"); + }, + _setHookId(hookId) { + this.db.set("hookId", hookId); + }, + getTriggerType() { + return "notification"; + }, + generateMeta(notification) { + return { + id: notification.id, + summary: `New Notification ${notification.id}`, + ts: Date.parse(notification.updated_at), + }; + }, + emitEvent(event) { + const meta = this.generateMeta(event); + this.$emit(event, meta); + }, + }, + async run(event) { + const { body } = event; + for (const item of body) { + this.emitEvent(item); + } + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/test.sh b/packages/component_code_gen/tests/webhooks/test.sh new file mode 100755 index 0000000000000..727c87e95866c --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/test.sh @@ -0,0 +1,31 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BASE_PATH=$SCRIPT_DIR/output/ + +echo "running github..." +poetry run python generate_webhook_source.py --app github "how to get webhooks for every new commit" > "$BASE_PATH"/github-new-commit.mjs +echo "running stripe..." +poetry run python generate_webhook_source.py --app stripe "how to get webhooks for every new payment" > "$BASE_PATH"/stripe-new-payment.mjs +echo "running twilio..." +poetry run python generate_webhook_source.py --app twilio "how to get webhooks for every new call" > "$BASE_PATH"/twilio-new-call.mjs +echo "running woocommerce..." +poetry run python generate_webhook_source.py --app woocommerce "how to get webhooks for every new order event" > "$BASE_PATH"/woocommerce-new-order-event.mjs +echo "running postmark..." +poetry run python generate_webhook_source.py --app postmark "how to get webhooks for every new inbound email" > "$BASE_PATH"/postmark-new-inbound-email-received.mjs + +echo "running process_street..." +poetry run python generate_webhook_source.py --app process_street "how to get webhooks for every new completed workflow run" > "$BASE_PATH"/process_street-workflow-run-completed.mjs +echo "running zenkit..." +poetry run python generate_webhook_source.py --app zenkit "how to get webhooks for every new notification" > "$BASE_PATH"/zenkit-new-notification.mjs +echo "running fibery..." +poetry run python generate_webhook_source.py --app fibery "how to get webhooks for every new created entity" > "$BASE_PATH"/fibery-entity-created.mjs +echo "running tally..." +poetry run python generate_webhook_source.py --app tally "how to get webhooks for every new response" > "$BASE_PATH"/tally-new-response.mjs + +echo "running asana..." +poetry run python generate_webhook_source.py --app asana "how to get webhooks for every new project" > "$BASE_PATH"/asana-new-project.mjs +echo "running accelo..." +poetry run python generate_webhook_source.py --app accelo "how to get webhooks for every new assigned task" > "$BASE_PATH"/accelo-new-task-assigned.mjs +echo "running shipcloud..." +poetry run python generate_webhook_source.py --app shipcloud "how to get webhooks for every new shipment status" > "$BASE_PATH"/shipcloud-new-shipment-status.mjs +echo "running quaderno..." +poetry run python generate_webhook_source.py --app quaderno "how to get webhooks for every new received payment" > "$BASE_PATH"/quaderno-payment-received.mjs From 858e16863afb11d5e69b75146699ce073c5e7eb6 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 1 Aug 2023 17:44:55 -0300 Subject: [PATCH 05/44] add component metadata rules (sources) --- .../templates/generate_webhook_sources.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index 143c0620c5a50..2ad5a1075e002 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -223,6 +223,27 @@ }, ``` +## Component Metadata + +Registry components require a unique key and version, and a friendly name and description. E.g. + +``` +export default { + key: "google_drive-new-shared-drive", + name: "New Shared Drive", + description: "Emits a new event any time a shared drive is created.", + version: "0.0.1", + type: "source", + dedupe: "unique", +}; +``` + +Component keys are in the format app_name_slug-slugified-component-name. +You should come up with a name and a description for the component you are generating. +In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). +Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). +Always add version "0.0.1", type "source", and dedupe "unique". + ## TypeScript Definitinos export interface Methods { From 53ed46e90227b9beefdf69286245faccc3440bb1 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 1 Aug 2023 17:45:40 -0300 Subject: [PATCH 06/44] add component metadata rules (actions) --- .../templates/transform_to_action.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/component_code_gen/templates/transform_to_action.py b/packages/component_code_gen/templates/transform_to_action.py index 8cd8bf9617051..12a659d09181e 100644 --- a/packages/component_code_gen/templates/transform_to_action.py +++ b/packages/component_code_gen/templates/transform_to_action.py @@ -196,6 +196,26 @@ }, ``` +## Component Metadata + +Registry components require a unique key and version, and a friendly name and description. E.g. + +``` +export default { + key: "google_drive-list-all-drives", + name: "List All Drives", + description: "Lists all drives in an account.", + version: "0.0.1", + type: "action", +}; +``` + +Component keys are in the format app_name_slug-slugified-component-name. +You should come up with a name and a description for the component you are generating. +In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). +Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). +Always add version "0.0.1" and type "action". + ## TypeScript Definitinos export interface Methods { From 81e89fa38f2c34c1a7e144c50a54c9ce9980202a Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 1 Aug 2023 18:06:51 -0300 Subject: [PATCH 07/44] make only one request to create actions --- .../code_gen/transform_code.py | 19 - .../component_code_gen/generate_action.py | 5 +- .../templates/generate_actions.py | 411 +++++++++++++++++- .../templates/transform_to_action.py | 400 ----------------- 4 files changed, 408 insertions(+), 427 deletions(-) delete mode 100644 packages/component_code_gen/code_gen/transform_code.py delete mode 100644 packages/component_code_gen/templates/transform_to_action.py diff --git a/packages/component_code_gen/code_gen/transform_code.py b/packages/component_code_gen/code_gen/transform_code.py deleted file mode 100644 index 16d7bf06ea60b..0000000000000 --- a/packages/component_code_gen/code_gen/transform_code.py +++ /dev/null @@ -1,19 +0,0 @@ -from dotenv import load_dotenv -load_dotenv() - -import openai -from config.config import config - - -def transform(code, templates): - openai.api_key = config['openai']['api_key'] - response = openai.ChatCompletion.create( - model="gpt-4", - messages=[ - {"role": "system", "content": templates.system_instructions}, - {"role": "user", "content": f"This is the code: {code}"}, - ], - temperature=0, - ) - - return response.choices[0].message.content.strip() diff --git a/packages/component_code_gen/generate_action.py b/packages/component_code_gen/generate_action.py index 760523a97ea0c..17311655130cc 100644 --- a/packages/component_code_gen/generate_action.py +++ b/packages/component_code_gen/generate_action.py @@ -1,7 +1,6 @@ import os import argparse import templates.generate_actions -import templates.transform_to_action def main(app, prompt, verbose=False): @@ -12,10 +11,8 @@ def main(app, prompt, verbose=False): # this is here so that the DEBUG environment variable is set before the imports from code_gen.generate_component_code import main - from code_gen.transform_code import transform - code = main(app, prompt, templates.generate_actions) - result = transform(code, templates.transform_to_action) + result = main(app, prompt, templates.generate_actions) return result diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index 7e0fd9edaa39a..fc09c2e9c492d 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -1,9 +1,412 @@ no_docs_user_prompt = """%s. The app is %s.""" -no_docs_system_instructions = """Your goal is to return code that answers the question you are given. -The code you should generate should a a Node.js HTTP request using axios. -The code you provide should use ESM. It should use async/await instead of promises. -You should not return any text other than the code. Do not return helpful messages, only code.""" +no_docs_system_instructions = """You are an agent that creates Pipedream Action Component Code. Your code should answer the question you are given. + +output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! + +## Pipedream components + +All Pipedream components are Node.js modules that have a default export: `defineComponent`. `defineComponent` is provided to the environment as a global — you do not need to import `defineComponent`. `defineComponent` is a function that takes an object — a Pipedream component — as its single argument. + +Here's an example component: + +```javascript +import { axios } from "@pipedream/platform" +export default defineComponent({ + key: "openai-list-models", + name: "List Models", + description: "Lists all models available to the user.", + version: "0.0.1", + type: "action", + props: { + openai: { + type: "app", + app: "openai", + } + }, + async run({steps, $}) { + return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, + }) + }, +}) +``` + +This object contains a `props` property, which defines a single prop of type "app": + +```javascript +export default defineComponent({ + props: { + the_app_name: { + type: "app", + app: "the_app_name", + }, + } + // the rest of the component ... +}) +``` + +This lets the user connect their app account to the step, authorizing requests to the app API. + +Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. + +The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. + +Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. + +## Pipedream Platform Axios + +If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: + +import { axios } from "@pipedream/platform"; + +You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. + +The `axios` constructor takes two arguments: + +1. `$` - the `$` context passed by the run method of the component. + +2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. + +For example: + +async run({steps, $}) { + return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, + }) +}, + +`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +Here's an example Pipedream component that makes a test request against the Slack API: + +export default defineComponent({ + props: { + slack: { + type: "app", + app: "slack", + }, + channel: { + type: "string", + label: "Channel", + description: "The channel to post the message to", + }, + text: { + type: "string", + label: "Message Text", + description: "The text of the message to post", + }, + }, + async run({ steps, $ }) { + return await axios($, { + method: "POST", + url: `https://slack.com/api/chat.postMessage`, + headers: { + Authorization: `Bearer ${this.slack.$auth.oauth_access_token}`, + }, + data: { + channel: this.channel, + text: this.text, + }, + }); + }, +}); + +The code you generate should be placed within the `run` method of the Pipedream component: + +import { axios } from "@pipedream/platform"; + +export default defineComponent({ + props: { + the_app_name_slug: { + type: "app", + app: "the_app_name_slug", + }, + }, + async run({ steps, $ }) { + return await axios($, { + // Add the axios configuration object to make the HTTP request here + }); + }, +}); + +## Async options props + +The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: + +``` +[ + { + label: "Human-readable option 1", + value: "unique identifier 1", + }, + { + label: "Human-readable option 2", + value: "unique identifier 2", + }, +] +``` + +The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. + +If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. + +Example async options methods: + +``` +msg: { + type: "string", + label: "Message", + description: "Select a message to `console.log()`", + async options() { + // write any node code that returns a string[] (with label/value keys) + return ["This is option 1", "This is option 2"]; + }, +}, +``` + +``` +board: { + type: "string", + label: "Board", + async options(opts) { + const boards = await this.getBoards(this.$auth.oauth_uid); + const activeBoards = boards.filter((board) => board.closed === false); + return activeBoards.map((board) => { + return { label: board.name, value: board.id }; + }); + }, +}, +``` + +``` +async options(opts) { + const response = await axios(this, { + method: "GET", + url: `https://api.spotify.com/v1/me/playlists`, + headers: { + Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, + }, + }); + return response.items.map((playlist) => { + return { label: playlist.name, value: playlist.id }; + }); +}, +``` + +## Component Metadata + +Registry components require a unique key and version, and a friendly name and description. E.g. + +``` +export default { + key: "google_drive-list-all-drives", + name: "List All Drives", + description: "Lists all drives in an account.", + version: "0.0.1", + type: "action", +}; +``` + +Component keys are in the format app_name_slug-slugified-component-name. +You should come up with a name and a description for the component you are generating. +In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). +Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). +Always add version "0.0.1" and type "action". +You MUST add metadata to the component code you generate. + +## TypeScript Definitinos + +export interface Methods { + [key: string]: (...args: any) => unknown; +} + +// $.flow.exit() and $.flow.delay() +export interface FlowFunctions { + exit: (reason: string) => void; + delay: (ms: number) => { + resume_url: string; + cancel_url: string; + }; +} + +export interface Pipedream { + export: (key: string, value: JSONValue) => void; + send: SendFunctionsWrapper; + /** + * Respond to an HTTP interface. + * @param response Define the status and body of the request. + * @returns A promise that is fulfilled when the body is read or an immediate response is issued + */ + respond: (response: HTTPResponse) => Promise | void; + flow: FlowFunctions; +} + +// Arguments to the options method for props +export interface OptionsMethodArgs { + page?: number; + prevContext?: any; + [key: string]: any; +} + +// You can reference the values of previously-configured props! +export interface OptionalOptsFn { + (configuredProps: { [key: string]: any; }): object; +} + +export type PropDefinition = + [App, string] | + [App, string, OptionalOptsFn]; + +// You can reference props defined in app methods, referencing the propDefintion directly in props +export interface PropDefinitionReference { + propDefinition: PropDefinition; +} + +export interface App< + Methods, + AppPropDefinitions +> { + type: "app"; + app: string; + propDefinitions?: AppPropDefinitions; + methods?: Methods & ThisType; +} + +export function defineApp< + Methods, + AppPropDefinitions, +> +(app: App): App { + return app; +} + +// Props + +export interface DefaultConfig { + intervalSeconds?: number; + cron?: string; +} + +export interface Field { + name: string; + value: string; +} + +export interface BasePropInterface { + label?: string; + description?: string; +} + +export type PropOptions = any[] | Array<{ [key: string]: string; }>; + +export interface UserProp extends BasePropInterface { + type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; + options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); + optional?: boolean; + default?: JSONValue; + secret?: boolean; + min?: number; + max?: number; +} + +export interface InterfaceProp extends BasePropInterface { + type: "$.interface.http" | "$.interface.timer"; + default?: string | DefaultConfig; +} + +// When users ask about data stores, remember to include a prop of type "data_store" in the props object +export interface DataStoreProp extends BasePropInterface { + type: "data_store"; +} + +export interface HttpRequestProp extends BasePropInterface { + type: "http_request"; + default?: DefaultHttpRequestPropConfig; +} + +export interface ActionPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; +} + +export interface AppPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp; +} + +export interface ActionRunOptions { + $: Pipedream; + steps: JSONValue; +} + +type PropThis = { + [Prop in keyof Props]: Props[Prop] extends App ? any : any +}; + +export interface Action< + Methods, + ActionPropDefinitions +> { + key: string; + name?: string; + description?: string; + version: string; + type: "action"; + methods?: Methods & ThisType & Methods>; + props?: ActionPropDefinitions; + additionalProps?: ( + previousPropDefs: ActionPropDefinitions + ) => Promise; + run: (this: PropThis & Methods, options?: ActionRunOptions) => any; +} + +export function defineAction< + Methods, + ActionPropDefinitions, +> +(component: Action): Action { + return component; +} + +## Additional rules + +1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. + +2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. + +3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. + +4. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. + +5. Double-check the code against known Node.js examples, from GitHub and any other real code you find. + +6. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it. + +7. Always use this signature for the run method: + +async run({steps, $}) { + // your code here +} + +Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. + +## Remember, return ONLY code + +Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. + +--- + +Your code: +""" with_docs_system_instructions = f"""{no_docs_system_instructions} You are an agent designed to interact with an OpenAPI JSON specification. diff --git a/packages/component_code_gen/templates/transform_to_action.py b/packages/component_code_gen/templates/transform_to_action.py deleted file mode 100644 index 12a659d09181e..0000000000000 --- a/packages/component_code_gen/templates/transform_to_action.py +++ /dev/null @@ -1,400 +0,0 @@ -system_instructions = """You are an agent designed to create Pipedream Component Code. You will receive a code snippet in Node.js using axios for a HTTP request. Your goal is to transform the code input you receive into a Pipedream Component. You should not return any text other than the code. -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! - -## Pipedream components - -All Pipedream components are Node.js modules that have a default export: `defineComponent`. `defineComponent` is provided to the environment as a global — you do not need to import `defineComponent`. `defineComponent` is a function that takes an object — a Pipedream component — as its single argument. - -Here's an example component: - -```javascript -import { axios } from "@pipedream/platform" -export default defineComponent({ - props: { - openai: { - type: "app", - app: "openai", - } - }, - async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) - }, -}) -``` - -This object contains a `props` property, which defines a single prop of type "app": - -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` - -This lets the user connect their app account to the step, authorizing requests to the app API. - -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. - -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. - -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `$` - the `$` context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -Here's an example Pipedream component that makes a test request against the Slack API: - -export default defineComponent({ - props: { - slack: { - type: "app", - app: "slack", - }, - channel: { - type: "string", - label: "Channel", - description: "The channel to post the message to", - }, - text: { - type: "string", - label: "Message Text", - description: "The text of the message to post", - }, - }, - async run({ steps, $ }) { - return await axios($, { - method: "POST", - url: `https://slack.com/api/chat.postMessage`, - headers: { - Authorization: `Bearer ${this.slack.$auth.oauth_access_token}`, - }, - data: { - channel: this.channel, - text: this.text, - }, - }); - }, -}); - -The code you generate should be placed within the `run` method of the Pipedream component: - -import { axios } from "@pipedream/platform"; - -export default defineComponent({ - props: { - the_app_name_slug: { - type: "app", - app: "the_app_name_slug", - }, - }, - async run({ steps, $ }) { - return await axios($, { - // Add the axios configuration object to make the HTTP request here - }); - }, -}); - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-list-all-drives", - name: "List All Drives", - description: "Lists all drives in an account.", - version: "0.0.1", - type: "action", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). -Always add version "0.0.1" and type "action". - -## TypeScript Definitinos - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. - -4. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -5. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -6. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it. - -7. Always use this signature for the run method: - -async run({steps, $}) { - // your code here -} - -Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - ---- - -Your code: -""" From 2e230e3bb3440a5064dd3d1737e919330dbb9894 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 2 Aug 2023 11:18:23 -0300 Subject: [PATCH 08/44] make gpt model a config --- packages/component_code_gen/README.md | 2 +- packages/component_code_gen/config/config.py | 1 + packages/component_code_gen/generate_action.py | 2 +- packages/component_code_gen/generate_component_code.py | 2 +- packages/component_code_gen/generate_webhook_sample.py | 2 +- packages/component_code_gen/generate_webhook_source.py | 2 +- packages/component_code_gen/helpers/langchain_helpers.py | 4 ++-- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 8be13fab0a748..4cc6ce3858ddf 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -1,6 +1,6 @@ # AI Code Gen for Pipedream Components -Generate components using OpenAI GPT-4. +Generate components using OpenAI GPT. ### Run diff --git a/packages/component_code_gen/config/config.py b/packages/component_code_gen/config/config.py index ac152928cafb8..1e7c089d0e29b 100644 --- a/packages/component_code_gen/config/config.py +++ b/packages/component_code_gen/config/config.py @@ -18,4 +18,5 @@ "logging": { "level": "DEBUG" if os.environ.get('DEBUG') == "1" else "WARN", }, + "model": "gpt-4", } diff --git a/packages/component_code_gen/generate_action.py b/packages/component_code_gen/generate_action.py index 17311655130cc..37405b3407014 100644 --- a/packages/component_code_gen/generate_action.py +++ b/packages/component_code_gen/generate_action.py @@ -27,7 +27,7 @@ def validate_inputs(app, prompt): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') args = parser.parse_args() result = main(args.app, args.prompt, args.verbose) diff --git a/packages/component_code_gen/generate_component_code.py b/packages/component_code_gen/generate_component_code.py index f3da238ee6b81..6a66d69f3b3dd 100644 --- a/packages/component_code_gen/generate_component_code.py +++ b/packages/component_code_gen/generate_component_code.py @@ -29,7 +29,7 @@ def validate_inputs(app, prompt): parser = argparse.ArgumentParser() parser.add_argument('--action', help='which kind of code you want to generate?', choices=actions.keys(), required=True) parser.add_argument('--app', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') args = parser.parse_args() result = main(args.action, args.app, args.prompt, args.verbose) diff --git a/packages/component_code_gen/generate_webhook_sample.py b/packages/component_code_gen/generate_webhook_sample.py index 33df4dd3ee12e..5b9c630f391d4 100644 --- a/packages/component_code_gen/generate_webhook_sample.py +++ b/packages/component_code_gen/generate_webhook_sample.py @@ -27,7 +27,7 @@ def validate_inputs(app, prompt): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') args = parser.parse_args() result = main(args.app, args.prompt, args.verbose) diff --git a/packages/component_code_gen/generate_webhook_source.py b/packages/component_code_gen/generate_webhook_source.py index 38fa5f6f71358..6932deab84fab 100644 --- a/packages/component_code_gen/generate_webhook_source.py +++ b/packages/component_code_gen/generate_webhook_source.py @@ -27,7 +27,7 @@ def validate_inputs(app, prompt): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt-4, in between quotes') + parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') args = parser.parse_args() result = main(args.app, args.prompt, args.verbose) diff --git a/packages/component_code_gen/helpers/langchain_helpers.py b/packages/component_code_gen/helpers/langchain_helpers.py index 424fa8fc6caea..32f85b3074066 100644 --- a/packages/component_code_gen/helpers/langchain_helpers.py +++ b/packages/component_code_gen/helpers/langchain_helpers.py @@ -30,7 +30,7 @@ def __init__(self, docs, templates): format_instructions=templates.format_instructions, input_variables=['input', 'agent_scratchpad'] ) - llm = ChatOpenAI(model_name='gpt-4', temperature=0, request_timeout=300) + llm = ChatOpenAI(model_name=config["model"], temperature=0, request_timeout=300) llm_chain = LLMChain(llm=llm, prompt=prompt_template) agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names) verbose = True if config['logging']['level'] == 'DEBUG' else False @@ -69,7 +69,7 @@ def ask_agent(user_prompt, docs, templates): def no_docs(app, prompt, templates): openai.api_key = config['openai']['api_key'] result = openai.ChatCompletion.create( - model="gpt-4", + model=config["model"], messages=[ {"role": "system", "content": format_template(templates.no_docs_system_instructions)}, {"role": "user", "content": templates.no_docs_user_prompt % (prompt, app)}, From 057775875f28114708f99657fef45e65e75434ca Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 2 Aug 2023 11:47:42 -0300 Subject: [PATCH 09/44] try and go to next if failed --- .../code_gen/generate_component_code.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index efbf79f8d227f..607a7b866a6d8 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -13,17 +13,25 @@ def main(app, prompt, templates): docs_meta = db.get_app_docs_meta(app) - if 'docs_url' in docs_meta: - contents = db.get_docs_contents(app) - if contents: - docs = { row['url']: row['content'] for row in contents } - return with_docs(app, prompt, docs, 'api reference', templates) - - if 'openapi_url' in docs_meta: - contents = db.get_openapi_contents(app) - if contents: - docs = { row['path']: row['content'] for row in contents } - return with_docs(app, prompt, docs, 'openapi', templates) + try: + if 'docs_url' in docs_meta: + contents = db.get_docs_contents(app) + if contents: + docs = { row['url']: row['content'] for row in contents } + return with_docs(app, prompt, docs, 'api reference', templates) + except Exception as e: + logger.error(e) + logger.error("failed with docs") + + try: + if 'openapi_url' in docs_meta: + contents = db.get_openapi_contents(app) + if contents: + docs = { row['path']: row['content'] for row in contents } + return with_docs(app, prompt, docs, 'openapi', templates) + except Exception as e: + logger.error(e) + logger.error("failed with openapi") return no_docs(app, prompt, templates) From 08d3eb330b9dbc6cd5f01bf7062611a76c4bfe77 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 11:07:18 -0300 Subject: [PATCH 10/44] add component metadata to examples --- packages/component_code_gen/templates/generate_actions.py | 5 +++++ .../templates/generate_webhook_sources.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index fc09c2e9c492d..c9f86dfa617b5 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -91,6 +91,11 @@ Here's an example Pipedream component that makes a test request against the Slack API: export default defineComponent({ + key: "slack-send-message", + name: "Send Message", + version: "0.0.1", + description: "Sends a message to a channel. [See docs here]()", + type: "action", props: { slack: { type: "app", diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index 2ad5a1075e002..55d8c902845a3 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -14,8 +14,14 @@ Here's an example component: ```javascript +import { axios } from "@pipedream/platform" export default { + key: "github-new-notification-received", + name: "New Notification Received", + description: "Emit new event when a notification is received.", + version: "0.0.1", type: "source", + dedupe: "unique", props: { github: { type: "app", From 0dbd8cda9aca28e0cc08b9a892866c547644cf42 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 11:07:43 -0300 Subject: [PATCH 11/44] modify source key/name to past tense --- .../component_code_gen/templates/generate_webhook_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index 55d8c902845a3..5853c4c18cee4 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -235,8 +235,8 @@ ``` export default { - key: "google_drive-new-shared-drive", - name: "New Shared Drive", + key: "google_drive-new-shared-drive-created", + name: "New Shared Drive Created", description: "Emits a new event any time a shared drive is created.", version: "0.0.1", type: "source", From bd0d2795f57e2b7a4001989de9adf4708646dd88 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 11:24:41 -0300 Subject: [PATCH 12/44] remove generate webhook sample --- .../generate_component_code.py | 2 - .../generate_webhook_sample.py | 34 -------------- .../templates/webhook_samples.py | 46 ------------------- 3 files changed, 82 deletions(-) delete mode 100644 packages/component_code_gen/generate_webhook_sample.py delete mode 100644 packages/component_code_gen/templates/webhook_samples.py diff --git a/packages/component_code_gen/generate_component_code.py b/packages/component_code_gen/generate_component_code.py index 6a66d69f3b3dd..7ceeabd938fb6 100644 --- a/packages/component_code_gen/generate_component_code.py +++ b/packages/component_code_gen/generate_component_code.py @@ -1,6 +1,5 @@ import argparse import generate_action -import generate_webhook_sample import generate_webhook_source @@ -20,7 +19,6 @@ def validate_inputs(app, prompt): actions = { 'generate_action': generate_action.main, - 'generate_webhook_sample': generate_webhook_sample.main, 'generate_webhook_source': generate_webhook_source.main, } diff --git a/packages/component_code_gen/generate_webhook_sample.py b/packages/component_code_gen/generate_webhook_sample.py deleted file mode 100644 index 5b9c630f391d4..0000000000000 --- a/packages/component_code_gen/generate_webhook_sample.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import argparse -import templates.webhook_samples - - -def main(app, prompt, verbose=False): - validate_inputs(app, prompt) - - if verbose: - os.environ['DEBUG'] = '1' - - # this is here so that the DEBUG environment variable is set before the imports - from code_gen.generate_component_code import main - - result = main(app, prompt, templates.webhook_samples) - return result - - -def validate_inputs(app, prompt): - if not (bool(app) and bool(prompt)): - raise Exception('app and prompt are required') - - if type(app) != str and type(prompt) != str: - raise Exception('app and prompt should be strings') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() - result = main(args.app, args.prompt, args.verbose) - print(result) diff --git a/packages/component_code_gen/templates/webhook_samples.py b/packages/component_code_gen/templates/webhook_samples.py deleted file mode 100644 index 159139e784213..0000000000000 --- a/packages/component_code_gen/templates/webhook_samples.py +++ /dev/null @@ -1,46 +0,0 @@ -no_docs_user_prompt = """I want a webhook example. %s. The app is %s.""" - -no_docs_system_instructions = """You are an agent that generates correct JSON data for a webhook event for a specific app. -The JSON object should have fictitious data, but should have the correct structure. -You should not return any text other than the JSON object.""" - -with_docs_system_instructions = """You are an agent designed to interact with an OpenAPI JSON specification. -Your goal is to return a JSON webhook object that is fired when a specific event happens. -You should not return any text other than the JSON object. - -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """Begin! -Remember, DO NOT include any other text in your response other than the Pipedream Component code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always espace curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" From b8405181a4ce131c3034c97d969801bff7391365 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 11:49:34 -0300 Subject: [PATCH 13/44] add webhook signature validation --- .../templates/generate_webhook_sources.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index 5853c4c18cee4..9992bc7e3f72e 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -14,6 +14,7 @@ Here's an example component: ```javascript +import crypto from "crypto" import { axios } from "@pipedream/platform" export default { key: "github-new-notification-received", @@ -34,6 +35,11 @@ db: "$.service.db", }, async run(event) { + const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") + if (computedSignature !== webhookSignature) { + this.http.respond({ status: 401, body: "Unauthorized" }) + return + } console.log(`Emitting event...`); this.$emit(event, { id: event.id, @@ -76,6 +82,14 @@ The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". +Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. + +Here's an example: +``` +const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") +if (computedSignature !== webhookSignature) this.http.respond({ status: 401, body: "Unauthorized" }) +``` + The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. ## Pipedream Platform Axios From 56972fabe16c3004b725d76f85ac87adfcb9b1b2 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 12:03:39 -0300 Subject: [PATCH 14/44] add webhook activate and deactivate hooks --- .../templates/generate_webhook_sources.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index 9992bc7e3f72e..ca523cd96189b 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -34,18 +34,36 @@ }, db: "$.service.db", }, + methods: { + _getWebhookId() { + return this.db.get("webhookId") + }, + _setWebhookId(id) { + this.db.set("webhookId", id) + }, + }, + hooks: { + async activate() { + const hookId = await this.createWebhook(opts) + this._setWebhookId(hookId) + }, + async deactivate() { + const id = this._getWebhookId() + await this.deleteWebhook(id) + }, + }, async run(event) { const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") if (computedSignature !== webhookSignature) { this.http.respond({ status: 401, body: "Unauthorized" }) return } - console.log(`Emitting event...`); + console.log(`Emitting event...`) this.$emit(event, { id: event.id, summary: `New event: ${event.name}`, ts: Date.parse(event.ts), - }); + }) }, }; ``` @@ -84,12 +102,6 @@ Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. -Here's an example: -``` -const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") -if (computedSignature !== webhookSignature) this.http.respond({ status: 401, body: "Unauthorized" }) -``` - The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. ## Pipedream Platform Axios From 5580553370ada047a9bc2ae1929bc259ea60d23b Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 12:04:09 -0300 Subject: [PATCH 15/44] add brex test webhook source --- .../reference/brex-new-transfer-event.mjs | 167 ++++++++++++++++++ .../component_code_gen/tests/webhooks/test.sh | 2 + 2 files changed, 169 insertions(+) create mode 100644 packages/component_code_gen/tests/webhooks/reference/brex-new-transfer-event.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/brex-new-transfer-event.mjs b/packages/component_code_gen/tests/webhooks/reference/brex-new-transfer-event.mjs new file mode 100644 index 0000000000000..4e4a2b4124654 --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/reference/brex-new-transfer-event.mjs @@ -0,0 +1,167 @@ +import brex from "../../brex.app.mjs"; +import crypto from "crypto"; +import { axios } from "@pipedream/platform"; +import { v4 as uuidv4 } from "uuid"; + +export default { + key: "brex-new-transfer-event", + name: "New Transfer Event (Instant)", + description: "Emit new event for new failed or processed events", + version: "0.1.0", + type: "source", + dedupe: "unique", + props: { + brex, + http: { + type: "$.interface.http", + customResponse: true, + }, + db: "$.service.db", + events: { + type: "string[]", + label: "Events", + description: "Please specify the events you want to watch with this source.", + options: [ + "TRANSFER_PROCESSED", + "TRANSFER_FAILED", + ], + }, + }, + hooks: { + async activate() { + await this.unregisterHook(); + const res = await axios(this, this._getAxiosParams({ + method: "POST", + path: "/v1/webhooks", + data: { + url: this.http.endpoint, + event_types: this.events, + }, + returnFullResponse: true, + })); + + if (!res.data?.id) { + throw new Error("It was not possible to register the webhook. Please try again or contact the support"); + } + + this._setHookId(res.data.id); + + console.log(`Hook successfully registered with id ${res.data.id}`); + }, + async deactivate() { + await this.unregisterHook(); + }, + }, + methods: { + _getBaseUrl() { + return "https://platform.brexapis.com"; + }, + _getHeaders() { + return { + "Content-Type": "application/json", + "Idempotency-Key": uuidv4(), + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, + }; + }, + _getAxiosParams(opts = {}) { + const res = { + ...opts, + url: this._getBaseUrl() + opts.path, + headers: this._getHeaders(), + }; + return res; + }, + async unregisterHook() { + const hookId = this._getHookId(); + if (!hookId) { + return; + } + await axios(this, this._getAxiosParams({ + method: "DELETE", + path: `/v1/webhooks/${this._getHookId()}`, + })); + this._setHookId(null); + console.log("Hook successfully unregistered"); + }, + async getSecretKeys() { + // Get secrets from Brex + const res = await axios(this, this._getAxiosParams({ + method: "GET", + path: "/v1/webhooks/secrets", + returnFullResponse: true, + })); + + if (res.data?.length === 0) { + throw new Error("It was not possible to verify the veracity of this request."); + } + + return res.data.map((key) => key.secret); + }, + checkVeracity(webhookSignature, webhookId, webhookTimestamp, webhookBody, secrets) { + for (let i = 0; i < secrets.length; i++) { + const signedContent = `${webhookId}.${webhookTimestamp}.${webhookBody}`; + const base64DecodedSecret = Buffer.from(secrets[i], "base64"); + const hmac = crypto.createHmac("sha256", base64DecodedSecret); + const computedSignature = hmac.update(signedContent).digest("base64"); + if (computedSignature === webhookSignature) { + return; + } + } + + throw new Error("The received request is not trustable. The computed signature does not match with the hook signature. THe request was aborted."); + }, + async getTransactionDetails(transactionId) { + return axios(this, this._getAxiosParams({ + method: "GET", + path: `/v1/transfers/${transactionId}`, + })); + }, + _emit(data) { + this.$emit(data, { + id: data.details.id, + summary: data.details.id, + ts: new Date(), + }); + }, + _setHookId(id) { + this.db.set("hookId", id); + }, + _getHookId() { + return this.db.get("hookId"); + }, + }, + async run(event) { + if ( + !event.headers || + !event.headers["webhook-signature"] || + !event.headers["webhook-id"] || + !event.headers["webhook-timestamp"] + ) { + throw new Error("The received request is not trustable, some header(s) is missing. The request was aborted."); + } + const keys = await this.getSecretKeys(); + const signatures = event.headers["webhook-signature"].split(" "); + for (let i = 0; i < signatures.length; i++) { + this.checkVeracity( + signatures[i].split(",")[1], + event.headers["webhook-id"], + event.headers["webhook-timestamp"], + event.bodyRaw, + keys, + ); + } + + const transaction = await this.getTransactionDetails(event.body.transfer_id); + this._emit({ + ...event.body, + details: transaction, + }); + + this.http.respond({ + status: 200, + headers: { + "Content-Type": "application/json", + }, + }); + }, +}; diff --git a/packages/component_code_gen/tests/webhooks/test.sh b/packages/component_code_gen/tests/webhooks/test.sh index 727c87e95866c..4f3d3a3ac35bd 100755 --- a/packages/component_code_gen/tests/webhooks/test.sh +++ b/packages/component_code_gen/tests/webhooks/test.sh @@ -29,3 +29,5 @@ echo "running shipcloud..." poetry run python generate_webhook_source.py --app shipcloud "how to get webhooks for every new shipment status" > "$BASE_PATH"/shipcloud-new-shipment-status.mjs echo "running quaderno..." poetry run python generate_webhook_source.py --app quaderno "how to get webhooks for every new received payment" > "$BASE_PATH"/quaderno-payment-received.mjs +echo "running brex..." +poetry run python generate_webhook_source.py --app brex "how to get webhooks for every new transfer event" > "$BASE_PATH"/brex-new-transfer-event.mjs From a4cbf92285e5c41e0681670b1d9eb2e64683a971 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 12:31:24 -0300 Subject: [PATCH 16/44] redirect all test output to file --- .../component_code_gen/tests/webhooks/test.sh | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/component_code_gen/tests/webhooks/test.sh b/packages/component_code_gen/tests/webhooks/test.sh index 4f3d3a3ac35bd..26a5330fd98d9 100755 --- a/packages/component_code_gen/tests/webhooks/test.sh +++ b/packages/component_code_gen/tests/webhooks/test.sh @@ -2,32 +2,32 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) BASE_PATH=$SCRIPT_DIR/output/ echo "running github..." -poetry run python generate_webhook_source.py --app github "how to get webhooks for every new commit" > "$BASE_PATH"/github-new-commit.mjs +poetry run python generate_webhook_source.py --app github "how to get webhooks for every new commit" > "$BASE_PATH"/github-new-commit.mjs 2>&1 echo "running stripe..." -poetry run python generate_webhook_source.py --app stripe "how to get webhooks for every new payment" > "$BASE_PATH"/stripe-new-payment.mjs +poetry run python generate_webhook_source.py --app stripe "how to get webhooks for every new payment" > "$BASE_PATH"/stripe-new-payment.mjs 2>&1 echo "running twilio..." -poetry run python generate_webhook_source.py --app twilio "how to get webhooks for every new call" > "$BASE_PATH"/twilio-new-call.mjs +poetry run python generate_webhook_source.py --app twilio "how to get webhooks for every new call" > "$BASE_PATH"/twilio-new-call.mjs 2>&1 echo "running woocommerce..." -poetry run python generate_webhook_source.py --app woocommerce "how to get webhooks for every new order event" > "$BASE_PATH"/woocommerce-new-order-event.mjs +poetry run python generate_webhook_source.py --app woocommerce "how to get webhooks for every new order event" > "$BASE_PATH"/woocommerce-new-order-event.mjs 2>&1 echo "running postmark..." -poetry run python generate_webhook_source.py --app postmark "how to get webhooks for every new inbound email" > "$BASE_PATH"/postmark-new-inbound-email-received.mjs +poetry run python generate_webhook_source.py --app postmark "how to get webhooks for every new inbound email" > "$BASE_PATH"/postmark-new-inbound-email-received.mjs 2>&1 echo "running process_street..." -poetry run python generate_webhook_source.py --app process_street "how to get webhooks for every new completed workflow run" > "$BASE_PATH"/process_street-workflow-run-completed.mjs +poetry run python generate_webhook_source.py --app process_street "how to get webhooks for every new completed workflow run" > "$BASE_PATH"/process_street-workflow-run-completed.mjs 2>&1 echo "running zenkit..." -poetry run python generate_webhook_source.py --app zenkit "how to get webhooks for every new notification" > "$BASE_PATH"/zenkit-new-notification.mjs +poetry run python generate_webhook_source.py --app zenkit "how to get webhooks for every new notification" > "$BASE_PATH"/zenkit-new-notification.mjs 2>&1 echo "running fibery..." -poetry run python generate_webhook_source.py --app fibery "how to get webhooks for every new created entity" > "$BASE_PATH"/fibery-entity-created.mjs +poetry run python generate_webhook_source.py --app fibery "how to get webhooks for every new created entity" > "$BASE_PATH"/fibery-entity-created.mjs 2>&1 echo "running tally..." -poetry run python generate_webhook_source.py --app tally "how to get webhooks for every new response" > "$BASE_PATH"/tally-new-response.mjs +poetry run python generate_webhook_source.py --app tally "how to get webhooks for every new response" > "$BASE_PATH"/tally-new-response.mjs 2>&1 echo "running asana..." -poetry run python generate_webhook_source.py --app asana "how to get webhooks for every new project" > "$BASE_PATH"/asana-new-project.mjs +poetry run python generate_webhook_source.py --app asana "how to get webhooks for every new project" > "$BASE_PATH"/asana-new-project.mjs 2>&1 echo "running accelo..." -poetry run python generate_webhook_source.py --app accelo "how to get webhooks for every new assigned task" > "$BASE_PATH"/accelo-new-task-assigned.mjs +poetry run python generate_webhook_source.py --app accelo "how to get webhooks for every new assigned task" > "$BASE_PATH"/accelo-new-task-assigned.mjs 2>&1 echo "running shipcloud..." -poetry run python generate_webhook_source.py --app shipcloud "how to get webhooks for every new shipment status" > "$BASE_PATH"/shipcloud-new-shipment-status.mjs +poetry run python generate_webhook_source.py --app shipcloud "how to get webhooks for every new shipment status" > "$BASE_PATH"/shipcloud-new-shipment-status.mjs 2>&1 echo "running quaderno..." -poetry run python generate_webhook_source.py --app quaderno "how to get webhooks for every new received payment" > "$BASE_PATH"/quaderno-payment-received.mjs +poetry run python generate_webhook_source.py --app quaderno "how to get webhooks for every new received payment" > "$BASE_PATH"/quaderno-payment-received.mjs 2>&1 echo "running brex..." -poetry run python generate_webhook_source.py --app brex "how to get webhooks for every new transfer event" > "$BASE_PATH"/brex-new-transfer-event.mjs +poetry run python generate_webhook_source.py --app brex "how to get webhooks for every new transfer event" > "$BASE_PATH"/brex-new-transfer-event.mjs 2>&1 From 4b6355a72cc7143164242155d0007d7e0b092f0d Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 19:48:49 -0300 Subject: [PATCH 17/44] add deploy hook --- .../templates/generate_webhook_sources.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index ca523cd96189b..e6ed3103eb9ed 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -43,6 +43,16 @@ }, }, hooks: { + async deploy() { + const events = await this.github.listMostRecentNotifications({paginate: true, max: 50}); + for (const event of events) { + this.$emit(event, { + id: event.id, + summary: `New event: ${event.name}`, + ts: Date.parse(event.ts), + }) + } + }, async activate() { const hookId = await this.createWebhook(opts) this._setWebhookId(hookId) @@ -104,6 +114,10 @@ The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. +## Source Hooks + +Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks. + ## Pipedream Platform Axios If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: From 12e7adcbfb9da686292613310c2924b428f54a17 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 20:05:20 -0300 Subject: [PATCH 18/44] fix typo --- packages/component_code_gen/templates/generate_actions.py | 2 +- .../component_code_gen/templates/generate_webhook_sources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index c9f86dfa617b5..b7e63d38868ef 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -230,7 +230,7 @@ Always add version "0.0.1" and type "action". You MUST add metadata to the component code you generate. -## TypeScript Definitinos +## TypeScript Definitions export interface Methods { [key: string]: (...args: any) => unknown; diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index e6ed3103eb9ed..c69c8ffd31405 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -290,7 +290,7 @@ Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). Always add version "0.0.1", type "source", and dedupe "unique". -## TypeScript Definitinos +## TypeScript Definitions export interface Methods { [key: string]: (...args: any) => unknown; From 1456fc37a82e0285d07acf6af481d4e79ff73ff6 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 20:06:55 -0300 Subject: [PATCH 19/44] add env example --- packages/component_code_gen/.env.example | 4 ++++ packages/component_code_gen/.gitignore | 1 + 2 files changed, 5 insertions(+) create mode 100644 packages/component_code_gen/.env.example diff --git a/packages/component_code_gen/.env.example b/packages/component_code_gen/.env.example new file mode 100644 index 0000000000000..d7a3e02d0e643 --- /dev/null +++ b/packages/component_code_gen/.env.example @@ -0,0 +1,4 @@ +BROWSERLESS_API_KEY=your-browserless-api-key +OPENAI_API_KEY=your-openai-api-key +SUPABASE_URL=https://your-supabase-url.supabase.co +SUPABASE_API_KEY=your-supabase-service-role-key diff --git a/packages/component_code_gen/.gitignore b/packages/component_code_gen/.gitignore index 6065c74698e1a..acc6a7b9e1612 100644 --- a/packages/component_code_gen/.gitignore +++ b/packages/component_code_gen/.gitignore @@ -1,4 +1,5 @@ .env +.env.example .vscode ve/ node_modules/ From 17a1f8df7eaeba383146c75c1a1ca7b59d98be78 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 20:10:33 -0300 Subject: [PATCH 20/44] change order in readme --- packages/component_code_gen/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 4cc6ce3858ddf..5a6cf79c124df 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -3,6 +3,16 @@ Generate components using OpenAI GPT. +### Installation + +1. Install poetry: follow instructions at https://python-poetry.org/docs/#installation + +2. Run install: +``` +poetry install +``` + + ### Run ``` @@ -14,16 +24,6 @@ poetry run python3 "$SCRIPT" --app "$APP" "$PROMPT" --verbose # print debug logs ``` -### Installation - -1. Install poetry: follow instructions at https://python-poetry.org/docs/#installation - -2. Run install: -``` -poetry install -``` - - ### Setup Create a `.env` file From 52d6c9b021bd5f19bd11630dda1f674a3d0c9d17 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 7 Aug 2023 20:13:57 -0300 Subject: [PATCH 21/44] change main function name --- .../component_code_gen/code_gen/generate_component_code.py | 2 +- packages/component_code_gen/generate_action.py | 4 ++-- packages/component_code_gen/generate_webhook_source.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index 607a7b866a6d8..6df4fed846851 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -8,7 +8,7 @@ logger = logging_config.getLogger(__name__) -def main(app, prompt, templates): +def generate_code(app, prompt, templates): db = supabase_helpers.SupabaseConnector() docs_meta = db.get_app_docs_meta(app) diff --git a/packages/component_code_gen/generate_action.py b/packages/component_code_gen/generate_action.py index 37405b3407014..1b43b00ba580f 100644 --- a/packages/component_code_gen/generate_action.py +++ b/packages/component_code_gen/generate_action.py @@ -10,9 +10,9 @@ def main(app, prompt, verbose=False): os.environ['DEBUG'] = '1' # this is here so that the DEBUG environment variable is set before the imports - from code_gen.generate_component_code import main + from code_gen.generate_component_code import generate_code - result = main(app, prompt, templates.generate_actions) + result = generate_code(app, prompt, templates.generate_actions) return result diff --git a/packages/component_code_gen/generate_webhook_source.py b/packages/component_code_gen/generate_webhook_source.py index 6932deab84fab..1ef4d79d3e72f 100644 --- a/packages/component_code_gen/generate_webhook_source.py +++ b/packages/component_code_gen/generate_webhook_source.py @@ -10,9 +10,9 @@ def main(app, prompt, verbose=False): os.environ['DEBUG'] = '1' # this is here so that the DEBUG environment variable is set before the imports - from code_gen.generate_component_code import main + from code_gen.generate_component_code import generate_code - result = main(app, prompt, templates.generate_webhook_sources) + result = generate_code(app, prompt, templates.generate_webhook_sources) return result From f7a668b2fb4e89e8e89e86969dbf2df0002a9262 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 09:00:02 -0300 Subject: [PATCH 22/44] add code example with auth to templates --- .../code_gen/generate_component_code.py | 7 +++++++ packages/component_code_gen/helpers/supabase_helpers.py | 8 ++++++++ packages/component_code_gen/templates/generate_actions.py | 7 +++---- .../templates/generate_webhook_sources.py | 6 +++--- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index 6df4fed846851..4ed420e243c5f 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -11,8 +11,11 @@ def generate_code(app, prompt, templates): db = supabase_helpers.SupabaseConnector() + auth_meta = db.get_app_auth_meta(app) docs_meta = db.get_app_docs_meta(app) + add_code_example(templates, auth_meta['component_code_scaffold_raw']) # TODO: is this needed only for actions? + try: if 'docs_url' in docs_meta: contents = db.get_docs_contents(app) @@ -52,3 +55,7 @@ def with_docs(app, prompt, docs, docs_type, templates): logger.debug("trying again without docs") return no_docs(app, prompt, templates) + + +def add_code_example(templates, example): + templates.no_docs_system_instructions %= example diff --git a/packages/component_code_gen/helpers/supabase_helpers.py b/packages/component_code_gen/helpers/supabase_helpers.py index 8ebfc540ad82e..c76bef5d2c31a 100644 --- a/packages/component_code_gen/helpers/supabase_helpers.py +++ b/packages/component_code_gen/helpers/supabase_helpers.py @@ -12,6 +12,14 @@ def __init__(self): config['supabase']['api_key'] ) + def get_app_auth_meta(self, app): + row = self.client \ + .table('apps') \ + .select('auth_type,component_code_scaffold_raw') \ + .match({'name_slug': app}) \ + .execute() + return row.data[0] if len(row.data) else {} + def get_app_docs_meta(self, app): row = self.client \ .table('components') \ diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index b7e63d38868ef..e082c75f002c7 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -408,9 +408,7 @@ Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. ---- - -Your code: +Consider all the instructions and rules above, and use the following code as a template for your code: %s """ with_docs_system_instructions = f"""{no_docs_system_instructions} @@ -434,7 +432,8 @@ In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" -suffix = """Begin! +suffix = """--- +Begin! Remember, DO NOT include any other text in your response other than the code. DO NOT return ``` or any other code formatting characters in your response. diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index c69c8ffd31405..fdb4d16d44417 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -463,9 +463,8 @@ ## Remember, return ONLY code Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. ---- -Your code: +Consider all the instructions and rules above, and use the following code as a template for your code: %s """ with_docs_system_instructions = f"""{no_docs_system_instructions} @@ -489,7 +488,8 @@ In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" -suffix = """Begin! +suffix = """--- +Begin! Remember, DO NOT include any other text in your response other than the code. DO NOT return ``` or any other code formatting characters in your response. From b65e6ac14ab3ce433f3d6db3de8b0841adfae019 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 10:07:15 -0300 Subject: [PATCH 23/44] change instructions for running --- packages/component_code_gen/.gitignore | 2 +- packages/component_code_gen/README.md | 35 ++++++++++------ .../code_gen/generate_component_code.py | 12 ++++++ .../component_code_gen/generate_action.py | 34 ---------------- .../generate_component_code.py | 34 ---------------- .../generate_webhook_source.py | 34 ---------------- .../instructions.md.example | 33 +++++++++++++++ packages/component_code_gen/main.py | 40 +++++++++++++++++++ 8 files changed, 108 insertions(+), 116 deletions(-) delete mode 100644 packages/component_code_gen/generate_action.py delete mode 100644 packages/component_code_gen/generate_component_code.py delete mode 100644 packages/component_code_gen/generate_webhook_source.py create mode 100644 packages/component_code_gen/instructions.md.example create mode 100644 packages/component_code_gen/main.py diff --git a/packages/component_code_gen/.gitignore b/packages/component_code_gen/.gitignore index acc6a7b9e1612..3fba59fa7f17d 100644 --- a/packages/component_code_gen/.gitignore +++ b/packages/component_code_gen/.gitignore @@ -1,5 +1,5 @@ .env -.env.example +instructions.md .vscode ve/ node_modules/ diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 5a6cf79c124df..ab9788ffda321 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -8,29 +8,38 @@ Generate components using OpenAI GPT. 1. Install poetry: follow instructions at https://python-poetry.org/docs/#installation 2. Run install: -``` -poetry install -``` - - -### Run ``` -SCRIPT=generate_action.py -APP=slack -PROMPT="how to send myself a direct message?" -poetry run python3 "$SCRIPT" --app "$APP" "$PROMPT" -poetry run python3 "$SCRIPT" --app "$APP" "$PROMPT" --verbose # print debug logs +poetry install ``` ### Setup -Create a `.env` file +1. Create a `.env` file -Add these API Keys: +2. Add these API Keys: - BROWSERLESS_API_KEY=api_key - OPENAI_API_KEY=API_KEY - SUPABASE_URL=https://url.supabase.co - SUPABASE_API_KEY=service_role_key + +3. Create a `instructions.md` file with a similar structure as the `instructions.md.example` file: + +``` +## Prompt + +... your prompt here ... + +## API docs + +... copy and paste relevant parts of the api docs here ... +``` + + +### Run + +``` +poetry run python main.py --component_type action --app slack --instructions instructions.md --verbose +``` diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index 4ed420e243c5f..3a7de271c7d6b 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -9,6 +9,8 @@ def generate_code(app, prompt, templates): + validate_inputs(app, prompt, templates) + db = supabase_helpers.SupabaseConnector() auth_meta = db.get_app_auth_meta(app) @@ -59,3 +61,13 @@ def with_docs(app, prompt, docs, docs_type, templates): def add_code_example(templates, example): templates.no_docs_system_instructions %= example + + +def validate_inputs(app, prompt, templates): + assert app and type(app) == str + assert prompt and type(prompt) == str + assert templates.no_docs_user_prompt + assert templates.no_docs_system_instructions + assert templates.with_docs_system_instructions + assert templates.suffix + assert templates.format_instructions diff --git a/packages/component_code_gen/generate_action.py b/packages/component_code_gen/generate_action.py deleted file mode 100644 index 1b43b00ba580f..0000000000000 --- a/packages/component_code_gen/generate_action.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import argparse -import templates.generate_actions - - -def main(app, prompt, verbose=False): - validate_inputs(app, prompt) - - if verbose: - os.environ['DEBUG'] = '1' - - # this is here so that the DEBUG environment variable is set before the imports - from code_gen.generate_component_code import generate_code - - result = generate_code(app, prompt, templates.generate_actions) - return result - - -def validate_inputs(app, prompt): - if not (bool(app) and bool(prompt)): - raise Exception('app and prompt are required') - - if type(app) != str and type(prompt) != str: - raise Exception('app and prompt should be strings') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() - result = main(args.app, args.prompt, args.verbose) - print(result) diff --git a/packages/component_code_gen/generate_component_code.py b/packages/component_code_gen/generate_component_code.py deleted file mode 100644 index 7ceeabd938fb6..0000000000000 --- a/packages/component_code_gen/generate_component_code.py +++ /dev/null @@ -1,34 +0,0 @@ -import argparse -import generate_action -import generate_webhook_source - - -def main(action, app, prompt, verbose=False): - validate_inputs(app, prompt) - result = actions[action](app, prompt, verbose) - return result - - -def validate_inputs(app, prompt): - if not (bool(app) and bool(prompt)): - raise Exception('app and prompt are required') - - if type(app) != str and type(prompt) != str: - raise Exception('app and prompt should be strings') - - -actions = { - 'generate_action': generate_action.main, - 'generate_webhook_source': generate_webhook_source.main, -} - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--action', help='which kind of code you want to generate?', choices=actions.keys(), required=True) - parser.add_argument('--app', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() - result = main(args.action, args.app, args.prompt, args.verbose) - print(result) diff --git a/packages/component_code_gen/generate_webhook_source.py b/packages/component_code_gen/generate_webhook_source.py deleted file mode 100644 index 1ef4d79d3e72f..0000000000000 --- a/packages/component_code_gen/generate_webhook_source.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import argparse -import templates.generate_webhook_sources - - -def main(app, prompt, verbose=False): - validate_inputs(app, prompt) - - if verbose: - os.environ['DEBUG'] = '1' - - # this is here so that the DEBUG environment variable is set before the imports - from code_gen.generate_component_code import generate_code - - result = generate_code(app, prompt, templates.generate_webhook_sources) - return result - - -def validate_inputs(app, prompt): - if not (bool(app) and bool(prompt)): - raise Exception('app and prompt are required') - - if type(app) != str and type(prompt) != str: - raise Exception('app and prompt should be strings') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--app', '-a', help='the app_name_slug', required=True) - parser.add_argument('prompt', help='the prompt to send to gpt, in between quotes') - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') - args = parser.parse_args() - result = main(args.app, args.prompt, args.verbose) - print(result) diff --git a/packages/component_code_gen/instructions.md.example b/packages/component_code_gen/instructions.md.example new file mode 100644 index 0000000000000..50409714d4d4f --- /dev/null +++ b/packages/component_code_gen/instructions.md.example @@ -0,0 +1,33 @@ +## Prompt + +How to send myself a direct message? + +## API docs + +https://api.slack.com/messaging/sending +https://api.slack.com/methods/chat.postMessage + +### Picking the right conversation + +Now we need to find somewhere within your workspace that we'll send a message. This could be any Slack conversation, but we'll use a public channel in this guide. + +We'll remind you again - it's not a good idea to attempt the instructions in this guide with a real, living workspace. If you really have to, then at least create a new, empty public channel within the workspace, for testing purposes. + +In order to find a valid Slack conversation ID, we'll use the conversations.list API method. This API will return a list of all public channels in the workspace your app is installed to. You'll need the channels:read permission granted to your app. + +Within that list, we'll be able to find a specific id of the conversation that we want to access. Here's an example API call: + +GET https://slack.com/api/conversations.list +Authorization: Bearer xoxb-your-token + +### Publishing your message + +We're nearly there, we just need to make one more API call, this time to chat.postMessage. Again substitute in the values of the token and conversation ID that you noted earlier: + +POST https://slack.com/api/chat.postMessage +Content-type: application/json +Authorization: Bearer xoxb-your-token +{ + "channel": "YOUR_CHANNEL_ID", + "text": "Hello world :tada:" +} diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py new file mode 100644 index 0000000000000..a9b472309b289 --- /dev/null +++ b/packages/component_code_gen/main.py @@ -0,0 +1,40 @@ +import os +import argparse +import templates.generate_actions +import templates.generate_webhook_sources + + +available_templates = { + 'action': templates.generate_actions, + 'webhook_source': templates.generate_webhook_sources, +} + + +def main(component_type, app, instructions_file, verbose=False): + if verbose: + os.environ['DEBUG'] = '1' + + with open(instructions_file, 'r') as f: + prompt = f.read() + + try: + templates = available_templates[component_type] + except: + raise ValueError(f'Templates for {component_type}s are not available. Please choose one of {available_templates.keys()}') + + # this is here so that the DEBUG environment variable is set before the import + from code_gen.generate_component_code import generate_code + result = generate_code(app, prompt, templates) + return result + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--component_type', help='which kind of code you want to generate?', choices=available_templates.keys(), required=True) + parser.add_argument('--app', help='the app_name_slug', required=True) + parser.add_argument('--instructions', help='markdown file with instructions: prompt + api docs', required=True) + parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + args = parser.parse_args() + + result = main(args.component_type, args.app, args.instructions, args.verbose) + print(result) From 2ab355f051a84ca9fcca0f1446d96f0c83c4a018 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 10:17:27 -0300 Subject: [PATCH 24/44] disable docs by default --- .../code_gen/generate_component_code.py | 9 +++++++-- packages/component_code_gen/config/config.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index 3a7de271c7d6b..b23893c5449b4 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -4,6 +4,7 @@ import helpers.langchain_helpers as langchain_helpers import helpers.supabase_helpers as supabase_helpers +from config.config import config import config.logging_config as logging_config logger = logging_config.getLogger(__name__) @@ -14,10 +15,14 @@ def generate_code(app, prompt, templates): db = supabase_helpers.SupabaseConnector() auth_meta = db.get_app_auth_meta(app) - docs_meta = db.get_app_docs_meta(app) - add_code_example(templates, auth_meta['component_code_scaffold_raw']) # TODO: is this needed only for actions? + if config['enable_docs'] == False: + logger.warn("docs are disabled") + return no_docs(app, prompt, templates) + + docs_meta = db.get_app_docs_meta(app) + try: if 'docs_url' in docs_meta: contents = db.get_docs_contents(app) diff --git a/packages/component_code_gen/config/config.py b/packages/component_code_gen/config/config.py index 1e7c089d0e29b..a3f14bcbe4267 100644 --- a/packages/component_code_gen/config/config.py +++ b/packages/component_code_gen/config/config.py @@ -19,4 +19,5 @@ "level": "DEBUG" if os.environ.get('DEBUG') == "1" else "WARN", }, "model": "gpt-4", + "enable_docs": False, } From 8ec3e2d83367023e924472f2f9a5e53e1f6bd62b Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 12:03:59 -0300 Subject: [PATCH 25/44] add tests to readme --- packages/component_code_gen/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index ab9788ffda321..08023330e0d36 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -43,3 +43,15 @@ poetry install ``` poetry run python main.py --component_type action --app slack --instructions instructions.md --verbose ``` + + +### Tests + +To run a suite of tests (e.g. webhooks): + +``` +./tests/webhooks/test.sh +``` + +This script will generate code for some selected apps/components for comparison with registry components +Compare `./tests/webhooks/output/*` with `./tests/webhooks/output/reference/*` From 7518ec9f154ab872f136a635e52b8618dd1be5d8 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 14:20:33 -0300 Subject: [PATCH 26/44] add tests for actions --- .../reference/accelo-create-contact.mjs | 130 ++++++++ .../actions/reference/asana-update-task.mjs | 217 +++++++++++++ .../reference/brex-set-limit-for-user.mjs | 97 ++++++ .../reference/fibery-get-entity-or-create.mjs | 243 +++++++++++++++ .../reference/github-get-repository.mjs | 68 ++++ .../reference/postmark-send-single-email.mjs | 208 +++++++++++++ .../process_street-start-workflow-run.mjs | 93 ++++++ .../reference/quaderno-create-invoice.mjs | 292 ++++++++++++++++++ .../reference/shipcloud-get-shipment-info.mjs | 98 ++++++ .../stripe-cancel-payment-intent.mjs | 96 ++++++ .../actions/reference/tally-get-responses.mjs | 80 +++++ .../actions/reference/twilio-get-message.mjs | 67 ++++ .../woocommerce-search-customers.mjs | 103 ++++++ .../reference/zenkit-add-entry-comment.mjs | 147 +++++++++ .../component_code_gen/tests/actions/test.sh | 33 ++ 15 files changed, 1972 insertions(+) create mode 100644 packages/component_code_gen/tests/actions/reference/accelo-create-contact.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/asana-update-task.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/brex-set-limit-for-user.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/fibery-get-entity-or-create.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/github-get-repository.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/postmark-send-single-email.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/process_street-start-workflow-run.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/quaderno-create-invoice.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/shipcloud-get-shipment-info.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/stripe-cancel-payment-intent.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/tally-get-responses.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/twilio-get-message.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/woocommerce-search-customers.mjs create mode 100644 packages/component_code_gen/tests/actions/reference/zenkit-add-entry-comment.mjs create mode 100644 packages/component_code_gen/tests/actions/test.sh diff --git a/packages/component_code_gen/tests/actions/reference/accelo-create-contact.mjs b/packages/component_code_gen/tests/actions/reference/accelo-create-contact.mjs new file mode 100644 index 0000000000000..e8eb2dec43dcf --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/accelo-create-contact.mjs @@ -0,0 +1,130 @@ +import { axios } from "@pipedream/platform"; + +export default { + name: "Create Contact", + version: "0.0.1", + key: "accelo-create-contact", + description: "Creates a contact. [See docs here](https://api.accelo.com/docs/?_ga=2.136158329.97118171.1674049767-1568937371.1674049767#create-a-contact)", + type: "action", + props: { + accelo: { + type: "app", + app: "accelo", + }, + companyId: { + label: "Company ID", + description: "The company ID", + type: "string", + async options() { + const { response: companies } = await this.getCompanies(); + return companies.map((company) => ({ + value: company.id, + label: company.name, + })); + }, + }, + firstname: { + label: "First Name", + description: "The contact's first name", + type: "string", + }, + middlename: { + label: "Middle Name", + description: "The contact's middle name", + type: "string", + optional: true, + }, + surname: { + label: "Surname", + description: "The contact's surname", + type: "string", + }, + username: { + label: "Username", + description: "The contact's new username, this must be a unique username", + type: "string", + optional: true, + }, + password: { + label: "Password", + description: "The contact's new password for the Accelo deployment", + type: "string", + optional: true, + }, + title: { + label: "Title", + description: "The contact's title", + type: "string", + optional: true, + }, + phone: { + label: "Phone", + description: "The contact's phone number in their role in the associated company.", + type: "string", + optional: true, + }, + email: { + label: "Email", + description: "The contact's email in their role in the associated company.", + type: "string", + optional: true, + }, + }, + methods: { + _hostname() { + return this.$auth.hostname; + }, + _accessToken() { + return this.$auth.oauth_access_token; + }, + _apiUrl() { + return `https://${this._hostname()}.api.accelo.com/api/v0`; + }, + async _makeRequest({ + $ = this, path, ...args + }) { + return axios($, { + url: `${this._apiUrl()}${path}`, + headers: { + Authorization: `Bearer ${this._accessToken()}`, + }, + ...args, + }); + }, + async getCompanies(args = {}) { + return this._makeRequest({ + path: "/companies", + ...args, + }); + }, + async createContact(args = {}) { + return this._makeRequest({ + path: "/contacts", + method: "post", + ...args, + }); + }, + }, + async run({ $ }) { + const { response } = await this.createContact({ + $, + data: { + company_id: this.companyId, + firstname: this.firstname, + middlename: this.middlename, + surname: this.surname, + username: this.username, + password: this.password, + title: this.title, + phone: this.phone, + email: this.email, + }, + }); + + if (response) { + $.export("$summary", `Successfully created contact with id ${response.id}`); + } + + return response; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/asana-update-task.mjs b/packages/component_code_gen/tests/actions/reference/asana-update-task.mjs new file mode 100644 index 0000000000000..010253044c895 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/asana-update-task.mjs @@ -0,0 +1,217 @@ +import { axios } from "@pipedream/platform"; + +export default { + key: "asana-update-task", + name: "Update Task", + description: "Updates a specific and existing task. [See the docs here](https://developers.asana.com/docs/update-a-task)", + version: "0.3.3", + type: "action", + props: { + asana: { + type: "app", + app: "asana", + }, + workspace: { + type: "string", + label: "Workspace", + description: "Gid of a workspace.", + optional: true, + async options() { + const workspaces = await this.getWorkspaces(); + return workspaces.map((workspace) => ({ + label: workspace.name, + value: workspace.gid, + })); + }, + }, + project: { + type: "string", + label: "Project", + description: "List of projects. This field use the project GID.", + optional: true, + async options({ workspace }) { + const projects = await this.getProjects(workspace); + return projects.map((tag) => ({ + label: tag.name, + value: tag.gid, + })); + }, + }, + task_gid: { + type: "string", + label: "Task GID", + description: "The ID of the task to update", + async options({ project }) { + const tasks = await this.getTasks({ + params: { + project, + }, + }); + return tasks.map(({ + name: label, gid: value, + }) => ({ + label, + value, + })); + }, + }, + name: { + type: "string", + label: "Name", + description: "Name of the task. This is generally a short sentence fragment that fits on a line in the UI for maximum readability. However, it can be longer.", + }, + assignee: { + type: "string", + label: "Assignee", + description: "Gid of a user.", + optional: true, + async options() { + const users = await this.getUsers(); + return users.map((user) => ({ + label: user.name, + value: user.gid, + })); + }, + }, + assignee_section: { + type: "string", + label: "Assignee Section", + description: "The assignee section is a subdivision of a project that groups tasks together in the assignee's \"My Tasks\" list.", + optional: true, + async options({ project }) { + const sections = await this.getSections(project); + return sections.map((section) => { + return { + label: section.name, + value: section.gid, + }; + }); + }, + }, + completed: { + label: "Completed", + description: "True if the task is currently marked complete, false if not.", + type: "boolean", + optional: true, + }, + due_at: { + label: "Due At", + description: "The UTC date and time on which this task is due, or null if the task has no due time. This takes an ISO 8601 date string in UTC and should not be used together with due_on.", + type: "string", + optional: true, + }, + due_on: { + label: "Due On", + description: "The localized date on which this task is due, or null if the task has no due date. This takes a date with YYYY-MM-DD format and should not be used together with due_at.", + type: "string", + optional: true, + }, + html_notes: { + label: "HTML Notes", + description: "The notes of the text with formatting as HTML.", + type: "string", + optional: true, + }, + notes: { + label: "Notes", + description: "Free-form textual information associated with the task (i.e. its description).", + type: "string", + optional: true, + }, + start_on: { + label: "Start On", + description: "The day on which work begins for the task , or null if the task has no start date. This takes a date with YYYY-MM-DD format.", + type: "string", + optional: true, + }, + custom_fields: { + label: "Custom Fields", + description: `An object where each key is a Custom Field gid and each value is an enum gid, string, or number: E.g. { + "4578152156": "Not Started", + "5678904321": "On Hold" + }`, + type: "string", + optional: true, + }, + }, + methods: { + _accessToken() { + return this.$auth.oauth_access_token; + }, + _apiUrl() { + return "https://app.asana.com/api/1.0"; + }, + _headers() { + return { + Accept: "application/json", + Authorization: `Bearer ${this._accessToken()}`, + }; + }, + async _makeRequest(path, options = {}, $ = this) { + const config = { + url: `${this._apiUrl()}/${path}`, + headers: this._headers(), + ...options, + }; + return axios($, config); + }, + async getWorkspaces() { + return (await this._makeRequest("workspaces")).data; + }, + async getProjects(workspaceId, params = {}, $) { + return (await this._makeRequest("projects", { + params: { + workspace: workspaceId, + ...params, + }, + }, $)).data; + }, + async getTasks(params, $) { + const response = await this._makeRequest("tasks", params, $); + return response.data; + }, + async getUsers(params = {}) { + const { + workspace, + team, + } = params; + return (await this._makeRequest("users", { + params: { + workspace, + team, + }, + })).data; + }, + async getSections(project, $) { + const response = await this._makeRequest(`projects/${project}/sections`, {}, $); + return response.data ?? []; + }, + }, + async run({ $ }) { + let customFields; + if (this.custom_fields) customFields = JSON.parse(this.custom_fields); + + const response = await this._makeRequest(`tasks/${this.task_gid}`, { + method: "put", + data: { + data: { + name: this.name, + assignee: this.assignee, + assignee_section: this.assignee_section, + completed: this.completed, + due_at: this.due_at, + due_on: this.due_on, + html_notes: this.html_notes, + notes: this.notes, + start_on: this.start_on, + workspace: this.workspace, + custom_fields: customFields, + }, + }, + }, $); + + $.export("$summary", "Successfully updated task"); + + return response.data; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/brex-set-limit-for-user.mjs b/packages/component_code_gen/tests/actions/reference/brex-set-limit-for-user.mjs new file mode 100644 index 0000000000000..d04b25af1262d --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/brex-set-limit-for-user.mjs @@ -0,0 +1,97 @@ +import { axios } from "@pipedream/platform"; +import { v4 as uuidv4 } from "uuid"; + +export default { + name: "Set Limit for User", + description: "Sets the monthly limit for a user. [See the docs here](https://developer.brex.com/openapi/team_api/#operation/setUserLimit).", + key: "brex-set-limit-for-user", + version: "0.1.0", + type: "action", + props: { + brex: { + type: "app", + app: "brex", + }, + user: { + label: "User", + description: "User to set the new limit", + withLabel: true, + async options({ prevContext }) { + const LIMIT = 100; + const res = await this.getUsers(prevContext.cursor, LIMIT); + return { + options: res.data.items?.map((item) => ({ + label: `${item.first_name} ${item.last_name} <${item.email}>`, + value: item.id, + })), + context: { + cursor: res.data.next_cursor, + }, + }; + }, + }, + amount: { + type: "integer", + label: "Monthly Limit", + description: "The amount of money, in the smallest denomination of the currency indicated by currency. For example, when currency is USD, amount is in cents (`1000.00`).", + }, + currency: { + type: "string", + label: "Currency", + description: "The type of currency, in [ISO 4217](https://en.wikipedia.org/wiki/ISO_4217) format. Default to `USD` if not specified", + optional: true, + }, + }, + methods: { + _getBaseUrl() { + return "https://platform.brexapis.com"; + }, + _getHeaders() { + return { + "Content-Type": "application/json", + "Idempotency-Key": uuidv4(), + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, + }; + }, + _getAxiosParams(opts = {}) { + const res = { + ...opts, + url: this._getBaseUrl() + opts.path, + headers: this._getHeaders(), + }; + return res; + }, + async getUsers(cursor, limit) { + return axios(this, this._getAxiosParams({ + method: "GET", + path: "/v2/users", + params: { + cursor, + limit, + }, + returnFullResponse: true, + })); + }, + }, + async run ({ $ }) { + const { + user, + amount, + currency, + } = this; + + const res = await axios($, this._getAxiosParams({ + method: "POST", + path: `/v2/users/${user.value || user}/limit`, + data: { + monthly_limit: { + amount, + currency, + }, + }, + })); + + $.export("$summary", `Monthly limit for ${user.label || user} successfully updated`); + return res; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/fibery-get-entity-or-create.mjs b/packages/component_code_gen/tests/actions/reference/fibery-get-entity-or-create.mjs new file mode 100644 index 0000000000000..1753affd96480 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/fibery-get-entity-or-create.mjs @@ -0,0 +1,243 @@ +import { axios } from "@pipedream/platform"; + +export default { + key: "fibery-get-entity-or-create", + name: "Get or Create Entity", + description: "Get an entity or create one if it doesn't exist. [See the docs here](https://api.fibery.io/graphql.html#create)", + version: "0.0.1", + type: "action", + props: { + fibery: { + type: "app", + app: "fibery", + }, + type: { + type: "string", + label: "Type", + description: "A custom type in your Fibery account", + async options() { + const types = await this.listTypes(); + return types.map((t) => (t["fibery/name"])); + }, + }, + where: { + type: "string", + label: "Where", + description: `A list of expressions to filter the results. [See docs here](https://api.fibery.io/#filter-entities). + E.g. \`[ "=", [ "Development/name" ], "$pipedream" ]\``, + optional: true, + }, + params: { + type: "object", + label: "Params", + description: "The params to pass with the `where` query. E.g. `{ \"$pipedream\": \"pipedream\" }`", + optional: true, + }, + fields: { + type: "string[]", + label: "Fields", + description: `The select fields for an entity type. This prop is an array of strings. + Each string should have the same structure as [in the docs](https://api.fibery.io/#select-fields) + E.g. \`["fibery/id",{"Development/Team":["fibery/id"]}]\``, + optional: true, + async options({ type }) { + const fields = await this.listFieldsForType({ + type, + }); + return fields.map((field) => field["fibery/name"]); + }, + }, + attributes: { + type: "object", + label: "Attributes", + description: `The attributes of the entity to create. + This prop is a JSON object, where each key is the name of the attribute, and each value is the value to set for the field. + You can use the **List Fields for Entity Type** action to get the list of available fields`, + }, + }, + methods: { + _baseUrl() { + return `https://${this.$auth.account_name}.fibery.io/api`; + }, + _auth() { + return this.$auth.api_key; + }, + async _makeRequest({ + $ = this, path, method = "post", ...opts + }) { + return axios($, { + ...opts, + url: this._baseUrl() + path, + method, + headers: { + ...opts.headers, + "Authorization": `Token ${this._auth()}`, + "Content-Type": "application/json", + }, + }); + }, + _createEntityCommand({ + type, id, attributes, + }) { + const command = id + ? "fibery.entity/update" + : "fibery.entity/create"; + return { + command, + args: { + type, + entity: { + ...attributes, + "fibery/id": id, + }, + }, + }; + }, + _isCustomType(type) { + const firstLetterIsUpperCase = (string) => string[0] === string[0].toUpperCase(); + return firstLetterIsUpperCase(type["fibery/name"]); + }, + singularOrPluralEntity(array) { + return array.length === 1 + ? "entity" + : "entities"; + }, + async getFieldName(type) { + const fields = await this.listFieldsForType({ + type, + }); + const field = fields.find((field) => field["fibery/name"].toLowerCase().endsWith("/name")); + return field["fibery/name"]; + }, + async makeCommand({ + command, args = {}, ...opts + }) { + const [ + response, + ] = await this._makeRequest({ + ...opts, + path: "/commands", + data: [ + { + command, + args, + }, + ], + }); + + if (response.success === false) { + throw new Error(JSON.stringify(response.result, null, 2)); + } + + return response; + }, + async makeBatchCommands(commands) { + const response = await this._makeRequest({ + path: "/commands", + data: commands, + }); + return response; + }, + async listEntities({ + type, fields = [], where, params, orderBy, limit = 50, ...opts + }) { + return this.makeCommand({ + ...opts, + command: "fibery.entity/query", + args: { + query: { + "q/from": type, + "q/select": [ + "fibery/id", + "fibery/creation-date", + await this.getFieldName(type), + ...fields, + ], + "q/order-by": orderBy, + "q/where": where, + "q/limit": limit, + }, + params, + }, + }); + }, + async listTypes(opts = {}) { + const response = await this.makeCommand({ + command: "fibery.schema/query", + ...opts, + }); + return response["result"]["fibery/types"] + .filter((type) => this._isCustomType(type)); + }, + async listFieldsForType({ + type, ...opts + }) { + const response = await this.makeCommand({ + command: "fibery.schema/query", + ...opts, + }); + return response["result"]["fibery/types"] + .find((t) => t["fibery/name"] === type)["fibery/fields"]; + }, + parseProps(fields, where, params) { + return { + fields: typeof (fields) === "string" + ? JSON.parse(fields) + : fields, + where: typeof (where) === "string" + ? JSON.parse(where) + : where, + params: typeof (params) === "string" + ? JSON.parse(params) + : params, + }; + }, + async findEntities($) { + const { + fields, + where, + params, + } = this.parseProps(this.fields, this.where, this.params); + const { result: entities } = await this.listEntities({ + $, + type: this.type, + where, + fields, + params, + }); + $.export("$summary", `Found ${entities.length} existing ${this.singularOrPluralEntity(entities)}`); + return entities; + }, + async createEntity($) { + const config = this._createEntityCommand({ + $, + type: this.type, + attributes: this.attributes, + }); + const response = await this.makeCommand(config); + $.export("$summary", "Succesfully created a new entity"); + return response; + }, + async updateEntities($, ids) { + const configs = []; + for (const id of ids) { + const config = this._createEntityCommand({ + $, + id, + type: this.type, + attributes: this.attributes, + }); + configs.push(config); + } + const response = await this.makeBatchCommands(configs); + $.export("$summary", `Succesfully updated ${this.singularOrPluralEntity(ids)}`); + return response; + }, + }, + async run({ $ }) { + const entities = await this.findEntities($); + return entities.length + ? entities + : this.createEntity($); + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/github-get-repository.mjs b/packages/component_code_gen/tests/actions/reference/github-get-repository.mjs new file mode 100644 index 0000000000000..8d2c8d7c8b331 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/github-get-repository.mjs @@ -0,0 +1,68 @@ +import { Octokit } from "@octokit/core"; +import { paginateRest } from "@octokit/plugin-paginate-rest"; +import { ConfigurationError } from "@pipedream/platform"; +const CustomOctokit = Octokit.plugin(paginateRest); + +export default { + key: "github-get-repository", + name: "Get Repository", + description: "Get specific repository. [See docs here](https://docs.github.com/en/rest/repos/repos#get-a-repository)", + version: "0.0.9", + type: "action", + props: { + github: { + type: "app", + app: "github", + }, + repoFullname: { + type: "string", + label: "Repository", + description: "The name of the repository. The name is not case sensitive", + async options({ org }) { + const repositories = await this.getRepos({ + org, + }); + return repositories.map((repository) => repository.full_name); + }, + }, + }, + methods: { + _baseApiUrl() { + return "https://api.github.com"; + }, + _accessToken() { + return this.$auth.oauth_access_token; + }, + _client() { + const client = new CustomOctokit({ + auth: this._accessToken(), + }); + client.hook.error("request", this.handleRequestException); + return client; + }, + handleRequestException(exception) { + console.error(exception); + const status = exception?.status; + if (status && (status === 404 || status === 403)) { + throw new ConfigurationError(`The request failed with status "${status}". It is likely that your token doesn't have sufficient permissions to execute that request. [see mor information here](https://docs.github.com/en/rest/overview/authenticating-to-the-rest-api?apiVersion=2022-11-28#about-authentication).`); + } + throw exception; + }, + async getRepos() { + return this._client().paginate("GET /user/repos", {}); + }, + async getRepo({ repoFullname }) { + const response = await this._client().request(`GET /repos/${repoFullname}`, {}); + return response.data; + }, + }, + async run({ $ }) { + const response = await this.getRepo({ + repoFullname: this.repoFullname, + }); + + $.export("$summary", "Successfully retrieved repository."); + + return response; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/postmark-send-single-email.mjs b/packages/component_code_gen/tests/actions/reference/postmark-send-single-email.mjs new file mode 100644 index 0000000000000..9e01d08dea2a3 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/postmark-send-single-email.mjs @@ -0,0 +1,208 @@ +import { axios } from "@pipedream/platform"; + +export default { + key: "postmark-send-single-email", + name: "Send Single Email", + description: "Send a single email with Postmark [(See docs here)](https://postmarkapp.com/developer/api/email-api#send-a-single-email)", + version: "0.2.0", + type: "action", + props: { + postmark: { + type: "app", + app: "postmark", + }, + subject: { + type: "string", + label: "Subject", + description: "Email subject.", + }, + htmlBody: { + type: "string", + label: "HTML Body", + description: + `HTML email message. + \\ + **Required** if no \`Text Body\` is specified. + \\ + **Required** to enable \`Open Tracking\`.`, + optional: true, + }, + textBody: { + type: "string", + label: "Text Body", + description: + `Plain text email message. + \\ + **Required** if no \`HTML Body\` is specified.`, + optional: true, + }, + toEmail: { + type: "string", + label: "Recipient email address(es)", + description: + "Recipient email address. Multiple addresses are comma separated. Max 50.", + }, + ccEmail: { + type: "string", + label: "CC email address(es)", + description: + "Cc recipient email address. Multiple addresses are comma separated. Max 50.", + optional: true, + }, + bccEmail: { + type: "string", + label: "BCC email address(es)", + description: + "Bcc recipient email address. Multiple addresses are comma separated. Max 50.", + optional: true, + }, + tag: { + type: "string", + label: "Tag", + description: + "Email tag that allows you to categorize outgoing emails and get detailed statistics.", + optional: true, + }, + replyTo: { + type: "string", + label: "\"Reply To\" email address", + description: + "Reply To override email address. Defaults to the Reply To set in the sender signature.", + optional: true, + }, + customHeaders: { + type: "string[]", + label: "Custom Headers", + description: "List of custom headers to include.", + optional: true, + }, + trackOpens: { + type: "boolean", + label: "Track Opens", + description: `Activate open tracking for this email. + \\ + **Note:** the email must have \`HTML Body\` to enable open tracking.`, + optional: true, + }, + trackLinks: { + type: "string", + label: "Track Links", + description: + "Activate link tracking for links in the HTML or Text bodies of this email.", + optional: true, + options: [ + "None", + "HtmlAndText", + "HtmlOnly", + "TextOnly", + ], + }, + attachments: { + type: "string[]", + label: "Attachments", + description: `Each attachment should be a string with the parameters separated by a pipe character \`|\`, in the format: \`Name|Content|ContentType\`. Alternatively, you can pass a string representing an object. All three parameters are required: + \\ + \\ + \`Name\` - the filename with extension, i.e. \`readme.txt\` + \\ + \`Content\` - the base64-encoded string with the binary data for the file, i.e. \`dGVzdCBjb250ZW50\` + \\ + \`ContentType\` - the MIME content type, i.e. \`text/plain\` + \\ + \\ + Example with pipe-separated parameters: \`readme.txt|dGVzdCBjb250ZW50|text/plain\` + \\ + Example with JSON-stringified object: \`{"Name":"readme.txt","Content":"dGVzdCBjb250ZW50","ContentType":"text/plain"}\` + `, + optional: true, + }, + metadata: { + type: "object", + label: "Metadata", + description: "Custom metadata key/value pairs.", + optional: true, + }, + messageStream: { + type: "string", + label: "Message stream", + description: + "Set message stream ID that's used for sending. If not provided, message will default to the outbound transactional stream.", + optional: true, + }, + }, + methods: { + _apikey() { + return this.$auth.api_key; + }, + getHeaders() { + return { + "X-Postmark-Server-Token": this._apikey(), + "Content-Type": "application/json", + "Accept": "application/json", + }; + }, + async sharedRequest($, params) { + const { + endpoint, + method, + data, + } = params; + + return axios($, { + url: `https://api.postmarkapp.com/${endpoint}`, + method, + headers: this.getHeaders(), + data, + }); + }, + async sharedActionRequest($, endpoint, data) { + return this.sharedRequest($, { + endpoint, + method: "POST", + data, + }); + }, + async sendSingleEmail($, data) { + return this.sharedActionRequest($, "email", data); + }, + getActionRequestCommonData() { + return { + From: this.fromEmail, + To: this.toEmail, + Cc: this.ccEmail, + Bcc: this.bccEmail, + Tag: this.tag, + ReplyTo: this.replyTo, + Headers: this.customHeaders, + TrackOpens: this.trackOpens, + TrackLinks: this.trackLinks, + Attachments: this.getAttachmentData(this.attachments), + Metadata: this.metadata, + MessageStream: this.messageStream, + }; + }, + getAttachmentData(attachments) { + return attachments?.map((str) => { + let params = str.split("|"); + return params.length === 3 + ? { + Name: params[0], + Content: params[1], + ContentType: params[2], + } + : JSON.parse(str); + }); + }, + }, + async run({ $ }) { + const data = { + ...this.getActionRequestCommonData(), + Subject: this.subject, + HtmlBody: this.htmlBody, + TextBody: this.textBody, + }; + const response = await this.sendSingleEmail($, data); + $.export("$summary", "Sent email successfully"); + return response; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/process_street-start-workflow-run.mjs b/packages/component_code_gen/tests/actions/reference/process_street-start-workflow-run.mjs new file mode 100644 index 0000000000000..5a84cc1c11428 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/process_street-start-workflow-run.mjs @@ -0,0 +1,93 @@ +import { axios } from "@pipedream/platform"; +import _ from "lodash"; + +export default { + key: "process_street-start-workflow-run", + name: "Start Workflow Run", + description: "Starts a workflow run. [See the docs here](https://public-api.process.st/api/v1.1/docs/index.html#operation/createWorkflowRun)", + version: "0.0.1", + type: "action", + props: { + processStreet: { + type: "app", + app: "process_street", + }, + workflowId: { + type: "string", + label: "Workflow ID", + description: "The ID of the Workflow", + async options() { + const { workflows } = await this.listWorkflows(); + return workflows.map((workflow) => ({ + label: workflow.name, + value: workflow.id, + })); + }, + }, + name: { + type: "string", + label: "Name", + description: "The name of the workflow run", + optional: true, + }, + dueDate: { + type: "string", + label: "Due Date", + description: "The due date in the [ISO 8601 format](https://en.wikipedia.org/wiki/ISO_8601) of the workflow run", + optional: true, + }, + shared: { + type: "boolean", + label: "Shared", + description: "Whether the workflow run is shared with other users", + optional: true, + }, + }, + methods: { + _baseUrl() { + return "https://public-api.process.st/api/v1.1"; + }, + _auth() { + return this.$auth.api_key; + }, + async _makeRequest({ + $ = this, path, ...opts + }) { + return axios($, { + ...opts, + url: this._baseUrl() + path, + headers: { + ...opts.headers, + "X-API-KEY": this._auth(), + }, + }); + }, + async listWorkflows(opts = {}) { + return this._makeRequest({ + ...opts, + path: "/workflows", + }); + }, + async runWorkflow(opts) { + return this._makeRequest({ + ...opts, + path: "/workflow-runs", + method: "post", + }); + }, + }, + async run({ $ }) { + const data = _.pickBy(_.pick(this, [ + "workflowId", + "name", + "dueDate", + "shared", + ])); + const response = await this.runWorkflow({ + $, + data, + }); + $.export("$summary", `Succesfully started workflow run ${this.name || ""}`); + return response; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/quaderno-create-invoice.mjs b/packages/component_code_gen/tests/actions/reference/quaderno-create-invoice.mjs new file mode 100644 index 0000000000000..3581843b9b379 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/quaderno-create-invoice.mjs @@ -0,0 +1,292 @@ +import { axios } from "@pipedream/platform"; + +export default { + key: "quaderno-create-invoice", + name: "Create Invoice", + description: "Generate a new invoice in Quaderno. [See the Documentation](https://developers.quaderno.io/api/#tag/Invoices/operation/createInvoice).", + type: "action", + version: "0.0.1", + props: { + quaderno: { + type: "app", + app: "quaderno", + }, + firstName: { + type: "string", + label: "First Name", + description: "The customer's first name who will be billed.", + optional: true, + }, + lastName: { + type: "string", + label: "Last Name", + description: "The customer's last name who will be billed.", + optional: true, + }, + dueDate: { + type: "string", + label: "Due Date", + description: "The date on which payment for this invoice is due. Must be in `YYYY-MM-DD` format.", + optional: true, + }, + currency: { + type: "string", + label: "Currency", + description: "Three-letter [ISO currency code](https://en.wikipedia.org/wiki/ISO_4217), in uppercase.", + optional: true, + }, + recurringPeriod: { + type: "string", + label: "Recurring Period", + description: "The period of time between each invoice. Can be `days`, `weeks`, `months`, `years`.", + optional: true, + options: Object.values(constants.PERIOD), + }, + recurringFrequency: { + type: "integer", + label: "Recurring Frequency", + description: "The number of periods between each invoice.", + optional: true, + }, + country: { + type: "string", + label: "Country", + description: "2-letter [ISO country code](https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes).", + optional: true, + }, + postalCode: { + type: "string", + label: "Postal Code", + description: "ZIP or postal code.", + optional: true, + }, + region: { + type: "string", + label: "Region", + description: "State/Province/Region.", + optional: true, + }, + streetLine1: { + type: "string", + label: "Street Line 1", + description: "Address line 1 (Street address/PO Box).", + optional: true, + }, + subject: { + type: "string", + label: "Subject", + description: "The subject of the invoice.", + optional: true, + }, + howManyItems: { + type: "integer", + label: "How Many Items", + description: "The number of line items to add to the invoice.", + reloadProps: true, + default: 1, + }, + }, + additionalProps() { + return Array.from({ + length: this.howManyItems, + }).reduce((props, _, idx) => { + const counter = idx + 1; + const item = `item${counter}`; + const label = `Item ${counter}:`; + const description = `${item}${SEP}description`; + const discountRate = `${item}${SEP}discountRate`; + const productCode = `${item}${SEP}productCode`; + const quantity = `${item}${SEP}quantity`; + const totalAmount = `${item}${SEP}totalAmount`; + const unitPrice = `${item}${SEP}unitPrice`; + return { + ...props, + [description]: { + type: "string", + label: `${label} Description`, + description: "The description of the item.", + optional: true, + }, + [discountRate]: { + type: "string", + label: `${label} Discount Rate`, + description: "Discount percent out of 100, if applicable.", + optional: true, + }, + [productCode]: { + type: "string", + label: `${label} Product Code`, + description: "The SKU of the Quaderno **Product** being invoiced. Use this attribute if you want to track your sales per product.", + optional: true, + }, + [quantity]: { + type: "integer", + label: `${label} Quantity`, + description: "The quantity of the item.", + optional: true, + default: 1, + }, + [totalAmount]: { + type: "string", + label: `${label} Total Amount`, + description: "The total amount to be charged after discounts and taxes. Required if **Unit Price** is not passed.", + optional: true, + }, + [unitPrice]: { + type: "string", + label: `${label} Unit Price`, + description: "The unit price of the item before any discount or tax is applied. Required if **Total Amount** is not passed.", + optional: true, + }, + }; + }, {}); + }, + methods: { + getBaseUrl() { + const baseUrl = `${constants.BASE_URL}${constants.VERSION_PATH}`; + return baseUrl.replace(constants.ACCOUNT_PLACEHOLDER, this.$auth.account_name) + .replace(constants.DOMAIN_PLACEHOLDER, this.$auth.domain); + }, + getUrl(path, url) { + return url || `${this.getBaseUrl()}${path}`; + }, + getAuth() { + return { + username: this.$auth.api_key, + }; + }, + getHeaders(headers) { + return { + "Content-Type": "application/json", + "Accept": `application/json; api_version=${constants.API_VERSION}`, + ...headers, + }; + }, + makeRequest({ + step = this, path, headers, url, ...args + } = {}) { + + const config = { + auth: this.getAuth(), + headers: this.getHeaders(headers), + url: this.getUrl(path, url), + ...args, + }; + + return axios(step, config); + }, + post(args = {}) { + return this.makeRequest({ + method: "post", + ...args, + }); + }, + createInvoice(args = {}) { + return this.post({ + path: "/invoices", + ...args, + }); + }, + getItems(length) { + return Array.from({ + length, + }).map((_, idx) => { + const counter = idx + 1; + const item = `item${counter}`; + const description = this[`${item}${SEP}description`]; + const discountRate = this[`${item}${SEP}discountRate`]; + const productCode = this[`${item}${SEP}productCode`]; + const quantity = this[`${item}${SEP}quantity`]; + const totalAmount = this[`${item}${SEP}totalAmount`]; + const unitPrice = this[`${item}${SEP}unitPrice`]; + return { + description, + discount_rate: discountRate, + product_code: productCode, + quantity, + total_amount: totalAmount, + unit_price: unitPrice, + }; + }); + }, + }, + async run({ $: step }) { + const { + firstName, + lastName, + dueDate, + currency, + recurringPeriod, + recurringFrequency, + country, + postalCode, + region, + streetLine1, + howManyItems, + } = this; + + const response = await this.createInvoice({ + step, + data: { + contact: { + first_name: firstName, + last_name: lastName, + }, + due_date: dueDate, + currency, + recurring_period: recurringPeriod, + recurring_frequency: recurringFrequency, + country, + postal_code: postalCode, + region, + street_line_1: streetLine1, + items_attributes: this.getItems(howManyItems), + }, + }); + + step.export("$summary", `Successfully created invoice with ID ${response.id}`); + + return response; + }, +}; + +const DOMAIN_PLACEHOLDER = "{domain}"; +const ACCOUNT_PLACEHOLDER = "{account_name}"; +const BASE_URL = `https://${ACCOUNT_PLACEHOLDER}.${DOMAIN_PLACEHOLDER}`; +const VERSION_PATH = "/api"; +const LAST_CREATED_AT = "lastCreatedAt"; +const DEFAULT_MAX = 600; + +const API_VERSION = "20220325"; + +const CONTACT_TYPE = { + PERSON: "person", + COMPANY: "company", +}; + +const PERIOD = { + DAYS: "days", + WEEKS: "weeks", + MONTHS: "months", + YEARS: "years", +}; + +const SEP = "_"; + +const WEBHOOK_ID = "webhookId"; +const AUTH_KEY = "authKey"; + +const constants = { + BASE_URL, + VERSION_PATH, + DEFAULT_MAX, + LAST_CREATED_AT, + API_VERSION, + DOMAIN_PLACEHOLDER, + ACCOUNT_PLACEHOLDER, + CONTACT_TYPE, + PERIOD, + SEP, + WEBHOOK_ID, + AUTH_KEY, +}; diff --git a/packages/component_code_gen/tests/actions/reference/shipcloud-get-shipment-info.mjs b/packages/component_code_gen/tests/actions/reference/shipcloud-get-shipment-info.mjs new file mode 100644 index 0000000000000..09f4182467b01 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/shipcloud-get-shipment-info.mjs @@ -0,0 +1,98 @@ +import { axios } from "@pipedream/platform"; + +export default { + name: "Get Shipment Info", + description: "Retrieve details for a shipment [See docs here](https://developers.shipcloud.io/reference/#getting-information-about-a-shipment)", + key: "shipcloud-get-shipment-info", + version: "0.0.1", + type: "action", + props: { + shipcloud: { + type: "app", + app: "shipcloud", + }, + shipmentId: { + type: "string", + label: "Shipment", + description: `Select a **Shipment** from the list. + \\ + Alternatively, you can provide a custom *Shipment ID*.`, + async options() { + const shipments = await this.listShipments(); + + return shipments.map((shipment) => { + return { + label: this.getShipmentLabel(shipment), + value: shipment.id, + }; + }); + }, + }, + }, + methods: { + _baseUrl() { + return "https://api.shipcloud.io/v1"; + }, + async _httpRequest({ + $ = this, + endpoint, + ...args + }) { + return axios($, { + url: this._baseUrl() + endpoint, + auth: { + username: this.$auth.api_key, + password: "", + }, + ...args, + }); + }, + async listShipments() { + const response = await this._httpRequest({ + endpoint: "/shipments", + }); + + return response.shipments ?? []; + }, + async getShipment({ + id, ...params + }) { + return this._httpRequest({ + endpoint: `/shipments/${id}`, + ...params, + }); + }, + getShipmentLabel({ + packages, price, to, + }) { + return `${packages.length} packages ($${price}) to ${this.getAddressLabel( + to, + )}`; + }, + getAddressLabel({ + first_name, + last_name, + street, + street_no, + zip_code, + city, + country, + }) { + return `${first_name} ${last_name} - ${street_no} ${street}, ${city} ${zip_code} (${country})`; + }, + }, + async run({ $ }) { + const params = { + $, + id: this.shipmentId, + }; + const data = await this.getShipment(params); + + $.export( + "$summary", + "Retrieved shipment info successfully", + ); + + return data; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/stripe-cancel-payment-intent.mjs b/packages/component_code_gen/tests/actions/reference/stripe-cancel-payment-intent.mjs new file mode 100644 index 0000000000000..e785768b28ba7 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/stripe-cancel-payment-intent.mjs @@ -0,0 +1,96 @@ +import stripe from "stripe"; +import pick from "lodash.pick"; + +export default { + key: "stripe-cancel-payment-intent", + name: "Cancel a Payment Intent", + type: "action", + version: "0.0.2", + description: "Cancel a [payment intent](https://stripe.com/docs/payments/payment-intents). " + + "Once canceled, no additional charges will be made by the payment intent and any operations " + + "on the payment intent will fail with an error. For payment intents with status=" + + "`requires_capture`, the remaining amount_capturable will automatically be refunded. [See the" + + " docs](https://stripe.com/docs/api/payment_intents/cancel) for more information", + props: { + stripe: { + type: "app", + app: "stripe", + }, + id: { + type: "string", + label: "Payment Intent ID", + description: "Example: `pi_0FhyHzGHO3mdGsgAJNHu7VeJ`", + options: createOptionsMethod("paymentIntents", [ + "id", + "description", + ]), + }, + cancellation_reason: { + type: "string", + label: "Cancellation Reason", + description: "Indicate why the payment was cancelled", + options: [ + "duplicate", + "fraudulent", + "requested_by_customer", + "abandoned", + ], + optional: true, + }, + }, + methods: { + _apiKey() { + return this.$auth.api_key; + }, + sdk() { + return stripe(this._apiKey(), { + apiVersion: "2020-03-02", + maxNetworkRetries: 2, + }); + }, + }, + async run({ $ }) { + const params = pick(this, [ + "cancellation_reason", + ]); + const resp = await this.sdk().paymentIntents.cancel(this.id, params); + $.export("$summary", "Successfully cancelled payment intent"); + return resp; + }, +}; + +const createOptionsMethod = (collectionOrFn, keysOrFn) => async function ({ + prevContext, ...opts +}) { + let { startingAfter } = prevContext; + let result; + if (typeof collectionOrFn === "function") { + result = await collectionOrFn.call(this, { + prevContext, + ...opts, + }); + } else { + result = await this.sdk()[collectionOrFn].list({ + starting_after: startingAfter, + }); + } + + let options; + if (typeof keysOrFn === "function") { + options = result.data.map(keysOrFn.bind(this)); + } else { + options = result.data.map((obj) => ({ + value: obj[keysOrFn[0]], + label: obj[keysOrFn[1]], + })); + } + + startingAfter = options?.[options.length - 1]?.value; + + return { + options, + context: { + startingAfter, + }, + }; +}; diff --git a/packages/component_code_gen/tests/actions/reference/tally-get-responses.mjs b/packages/component_code_gen/tests/actions/reference/tally-get-responses.mjs new file mode 100644 index 0000000000000..abc503637f445 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/tally-get-responses.mjs @@ -0,0 +1,80 @@ +import { axios } from "@pipedream/platform"; + +export default { + name: "Get Responses", + version: "0.0.1", + key: "tally-get-responses", + description: "Get a list of responses. [See docs here](https://tallyso.notion.site/Tally-OAuth-2-reference-d0442c679a464664823628f675f43454)", + type: "action", + props: { + tally: { + type: "app", + app: "tally", + }, + formId: { + label: "Form", + description: "Select a form", + type: "string", + async options() { + const forms = await this.getForms(); + return forms.map((form) => ({ + label: form.name, + value: form.id, + })); + }, + }, + }, + methods: { + _accessToken() { + return this.$auth.oauth_access_token; + }, + _apiUrl() { + return "https://api.tally.so"; + }, + async _makeRequest(path, options = {}, $ = undefined) { + return axios($ ?? this, { + url: `${this._apiUrl()}/${path}`, + headers: { + Authorization: `Bearer ${this._accessToken()}`, + }, + ...options, + }); + }, + async getForms({ $ } = {}) { + return this._makeRequest("forms", {}, $); + }, + async getResponses({ + formId, $, + }) { + let allResponses = []; + let page = 1; + + while (page > 0) { + const responses = await this._makeRequest(`forms/${formId}/responses`, { + params: { + page, + }, + }, $); + + if (responses.length > 0) { + allResponses = allResponses.concat(responses); + page++; + } else { + page = 0; + } + } + + return allResponses; + }, + }, + async run({ $ }) { + const response = await this.getResponses({ + formId: this.formId, + $, + }); + + $.export("$summary", "Successfully retrieved responses"); + + return response; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/twilio-get-message.mjs b/packages/component_code_gen/tests/actions/reference/twilio-get-message.mjs new file mode 100644 index 0000000000000..bd443f3518d63 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/twilio-get-message.mjs @@ -0,0 +1,67 @@ +import twilio from "twilio"; + +export default { + key: "twilio-get-message", + name: "Get Message", + description: "Return details of a message. [See the docs](https://www.twilio.com/docs/sms/api/message-resource#fetch-a-message-resource) for more information", + version: "0.1.2", + type: "action", + props: { + twilio: { + type: "app", + app: "twilio", + }, + messageId: { + type: "string", + label: "Message ID", + description: "The SID of the Message", + optional: true, + async options() { + const messages = await this.listMessages(); + return messages.map((message) => { + return { + label: messageToString(message), + value: message.sid, + }; + }); + }, + }, + }, + methods: { + getClient() { + // Uncomment this line when users are ready to migrate + // return twilio(this.$auth.accountSid, this.$auth.authToken); + return twilio(this.$auth.Sid, this.$auth.Secret, { + accountSid: this.$auth.AccountSid, + }); + }, + listMessages(params) { + const client = this.getClient(); + return client.messages.list(params); + }, + getMessage(sid) { + const client = this.getClient(); + return client.messages(sid).fetch(); + }, + }, + async run({ $ }) { + const resp = await this.getMessage(this.messageId); + $.export("$summary", `Successfully fetched the message, "${messageToString(resp)}"`); + return resp; + }, +}; + +function formatDateString(date) { + const dateObj = new Date(date); + return dateObj.toISOString().split("T")[0]; +} + +function messageToString(message) { + const MAX_LENGTH = 30; + const messageText = message.body.length > MAX_LENGTH + ? `${message.body.slice(0, MAX_LENGTH)}...` + : message.body; // truncate long text + const messageDate = message.dateSent || message.dateCreated; + const dateString = formatDateString(messageDate); + return `${message.from} to ${message.to} on ${dateString}: ${messageText}`; +} diff --git a/packages/component_code_gen/tests/actions/reference/woocommerce-search-customers.mjs b/packages/component_code_gen/tests/actions/reference/woocommerce-search-customers.mjs new file mode 100644 index 0000000000000..43ca5b46b5a09 --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/woocommerce-search-customers.mjs @@ -0,0 +1,103 @@ +import WooCommerceRestApi from "@woocommerce/woocommerce-rest-api"; +import querystring from "query-string"; +import pick from "lodash.pick"; +import pickBy from "lodash.pickby"; + +export default { + key: "woocommerce-search-customers", + name: "Search Customers", + description: "Finds a customer by searching. [See the docs](https://woocommerce.github.io/woocommerce-rest-api-docs/#list-all-customers)", + version: "0.0.2", + type: "action", + props: { + woocommerce: { + type: "app", + app: "woocommerce", + }, + search: { + type: "string", + label: "Search", + description: "Limit results to those matching a string", + optional: true, + }, + email: { + type: "string", + label: "Email", + description: "Limit result set to resources with a specific email", + optional: true, + }, + role: { + type: "string", + label: "Role", + description: "Limit result set to resources with a specific role", + options: [ + "all", + "administrator", + "editor", + "author", + "contributor", + "subscriber", + "customer", + ], + optional: true, + default: "customer", + }, + maxResults: { + type: "integer", + label: "Max Results", + description: "Maximum number of results to return", + optional: true, + default: 20, + }, + }, + methods: { + async getClient() { + let url = this.$auth.url; + + if (!/^(http(s?):\/\/)/.test(url)) { + url = `https://${url}`; + } + + return new WooCommerceRestApi.default({ + url, + consumerKey: this.$auth.key, + consumerSecret: this.$auth.secret, + wpAPI: true, + version: "wc/v3", + }); + }, + async listResources(endpoint) { + const client = await this.getClient(); + return (await client.get(endpoint)).data; + }, + async listCustomers(params = null) { + const q = querystring.stringify(params); + return this.listResources(`customers?${q}`); + }, + }, + async run({ $ }) { + const { maxResults } = this; + const params = { + page: 1, + per_page: 10, + ...pickBy(pick(this, [ + "search", + "email", + "role", + ])), + }; + + const customers = []; + let results; + do { + results = await this.listCustomers(params); + customers.push(...results); + params.page += 1; + } while (results.length === params.per_page && customers.length < maxResults); + if (customers.length > maxResults) { + customers.length = maxResults; + } + $.export("$summary", `Found ${customers.length} results`); + return customers; + }, +}; diff --git a/packages/component_code_gen/tests/actions/reference/zenkit-add-entry-comment.mjs b/packages/component_code_gen/tests/actions/reference/zenkit-add-entry-comment.mjs new file mode 100644 index 0000000000000..a50aaf76b507f --- /dev/null +++ b/packages/component_code_gen/tests/actions/reference/zenkit-add-entry-comment.mjs @@ -0,0 +1,147 @@ +import { axios } from "@pipedream/platform"; + +export default { + key: "zenkit-add-entry-comment", + name: "Add Entry Comment", + description: "Add a comment to an entry/item within a list/collection on Zenkit. [See the docs](https://base.zenkit.com/docs/api/activity/post-api-v1-users-me-lists-listallid-entries-listentryallid-activities)", + version: "0.0.1", + type: "action", + props: { + zenkit: { + type: "app", + app: "zenkit", + }, + workspaceId: { + type: "string", + label: "Workspace", + description: "Filter by workspace", + async options() { + const workspaces = await this.listWorkspaces(); + if (!workspaces || workspaces?.length === 0) { + return []; + } + return workspaces.map((workspace) => ({ + value: workspace.id, + label: workspace.name, + })); + }, + }, + listId: { + type: "string", + label: "List", + description: "Filter by list", + async options({ workspaceId }) { + const lists = await this.listLists({ + workspaceId, + }); + if (!lists || lists?.length === 0) { + return []; + } + return lists.map((list) => ({ + value: list.shortId, + label: list.name, + })); + }, + }, + entryId: { + type: "string", + label: "Entry", + description: "Filter by entry", + async options({ + listId, page, + }) { + const limit = 10; + const entries = await this.listListEntries({ + listId, + data: { + limit, + skip: limit * page, + }, + }); + if (!entries || entries?.length === 0) { + return []; + } + return entries.map((entry) => ({ + value: entry.id, + label: entry.displayString, + })); + }, + }, + comment: { + type: "string", + label: "Comment", + description: "Comment to add to entry", + }, + }, + methods: { + _baseUrl() { + return "https://zenkit.com/api/v1/"; + }, + _getHeaders() { + return { + Authorization: `Bearer ${this.$auth.oauth_access_token}`, + }; + }, + async _makeRequest(args = {}) { + const { + $ = this, + method = "GET", + path, + ...otherArgs + } = args; + const config = { + method, + url: `${this._baseUrl()}${path}`, + headers: this._getHeaders(), + ...otherArgs, + }; + return axios($, config); + }, + async listWorkspaces(args = {}) { + return this._makeRequest({ + path: "users/me/workspacesWithLists", + ...args, + }); + }, + async listLists({ + workspaceId, ...args + }) { + const workspaces = await this.listWorkspaces({ + ...args, + }); + const workspace = workspaces.find((workspace) => workspace.id == workspaceId); + return workspace?.lists; + }, + async listListEntries({ + listId, ...args + }) { + const { listEntries } = await this._makeRequest({ + method: "POST", + path: `lists/${listId}/entries/filter/list`, + ...args, + }); + return listEntries; + }, + async addCommentToEntry({ + listId, entryId, ...args + }) { + return this._makeRequest({ + method: "POST", + path: `users/me/lists/${listId}/entries/${entryId}/activities`, + ...args, + }); + }, + }, + async run({ $ }) { + const response = await this.addCommentToEntry({ + listId: this.listId, + entryId: this.entryId, + data: { + message: this.comment, + }, + $, + }); + $.export("$summary", `Successfully added comment to entry '${response?.listEntryDisplayString}'`); + return response; + }, +}; diff --git a/packages/component_code_gen/tests/actions/test.sh b/packages/component_code_gen/tests/actions/test.sh new file mode 100644 index 0000000000000..1db55e078ea4d --- /dev/null +++ b/packages/component_code_gen/tests/actions/test.sh @@ -0,0 +1,33 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BASE_PATH=$SCRIPT_DIR/output/ + +echo "running github..." +poetry run python main.py --component_type action --app github "how to get a specific repository" > "$BASE_PATH"/github-get-repository.mjs 2>&1 +echo "running stripe..." +poetry run python main.py --component_type action --app stripe "how to cancel a payment intent" > "$BASE_PATH"/stripe-cancel-payment-intent.mjs 2>&1 +echo "running twilio..." +poetry run python main.py --component_type action --app twilio "how to get a message" > "$BASE_PATH"/twilio-get-message.mjs 2>&1 +echo "running woocommerce..." +poetry run python main.py --component_type action --app woocommerce "how to search for customers" > "$BASE_PATH"/woocommerce-search-customers.mjs 2>&1 +echo "running postmark..." +poetry run python main.py --component_type action --app postmark "how to send an email" > "$BASE_PATH"/postmark-send-single-email.mjs 2>&1 + +echo "running process_street..." +poetry run python main.py --component_type action --app process_street "how to start a workflow run" > "$BASE_PATH"/process_street-start-workflow-run.mjs 2>&1 +echo "running zenkit..." +poetry run python main.py --component_type action --app zenkit "how to add a comment to an entry/item within a list/collection" > "$BASE_PATH"/zenkit-add-entry-comment.mjs 2>&1 +echo "running fibery..." +poetry run python main.py --component_type action --app fibery "how to get an entity or create one if it doesn't exist" > "$BASE_PATH"/fibery-get-entity-or-create.mjs 2>&1 +echo "running tally..." +poetry run python main.py --component_type action --app tally "how to get a list of responses" > "$BASE_PATH"/tally-get-responses.mjs 2>&1 + +echo "running asana..." +poetry run python main.py --component_type action --app asana "how to update a task" > "$BASE_PATH"/asana-update-task.mjs 2>&1 +echo "running accelo..." +poetry run python main.py --component_type action --app accelo "how to create a contact" > "$BASE_PATH"/accelo-create-contact.mjs 2>&1 +echo "running shipcloud..." +poetry run python main.py --component_type action --app shipcloud "how to get information about a shipment" > "$BASE_PATH"/shipcloud-get-shipment-info.mjs 2>&1 +echo "running quaderno..." +poetry run python main.py --component_type action --app quaderno "how to create an invoice" > "$BASE_PATH"/quaderno-create-invoice.mjs 2>&1 +echo "running brex..." +poetry run python main.py --component_type action --app brex "how to set a limit for an user" > "$BASE_PATH"/brex-set-limit-for-user.mjs 2>&1 From 31ed78968dbc9825993850d720ebd6e198d191d0 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 15:34:31 -0300 Subject: [PATCH 27/44] read instructions file outside of main --- packages/component_code_gen/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py index a9b472309b289..51226fdf9a536 100644 --- a/packages/component_code_gen/main.py +++ b/packages/component_code_gen/main.py @@ -10,13 +10,10 @@ } -def main(component_type, app, instructions_file, verbose=False): +def main(component_type, app, instructions, verbose=False): if verbose: os.environ['DEBUG'] = '1' - with open(instructions_file, 'r') as f: - prompt = f.read() - try: templates = available_templates[component_type] except: @@ -24,7 +21,7 @@ def main(component_type, app, instructions_file, verbose=False): # this is here so that the DEBUG environment variable is set before the import from code_gen.generate_component_code import generate_code - result = generate_code(app, prompt, templates) + result = generate_code(app, instructions, templates) return result @@ -36,5 +33,8 @@ def main(component_type, app, instructions_file, verbose=False): parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') args = parser.parse_args() - result = main(args.component_type, args.app, args.instructions, args.verbose) + with open(args.instructions, 'r') as f: + instructions = f.read() + + result = main(args.component_type, args.app, instructions, args.verbose) print(result) From 85a895bd49f625ae1b7a670208b08496473a5375 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 15:42:10 -0300 Subject: [PATCH 28/44] change tests to python --- packages/component_code_gen/README.md | 4 +- .../component_code_gen/tests/actions/test.py | 89 +++++++++++++++++++ .../component_code_gen/tests/actions/test.sh | 33 ------- .../component_code_gen/tests/webhooks/test.py | 89 +++++++++++++++++++ .../component_code_gen/tests/webhooks/test.sh | 33 ------- 5 files changed, 180 insertions(+), 68 deletions(-) create mode 100644 packages/component_code_gen/tests/actions/test.py delete mode 100644 packages/component_code_gen/tests/actions/test.sh create mode 100644 packages/component_code_gen/tests/webhooks/test.py delete mode 100755 packages/component_code_gen/tests/webhooks/test.sh diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 08023330e0d36..4cd68b37cb93b 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -50,8 +50,8 @@ poetry run python main.py --component_type action --app slack --instructions ins To run a suite of tests (e.g. webhooks): ``` -./tests/webhooks/test.sh +poetry run python -m tests.webhooks.test ``` This script will generate code for some selected apps/components for comparison with registry components -Compare `./tests/webhooks/output/*` with `./tests/webhooks/output/reference/*` +Compare `./tests/webhooks/output/*` with `./tests/webhooks/reference/*` diff --git a/packages/component_code_gen/tests/actions/test.py b/packages/component_code_gen/tests/actions/test.py new file mode 100644 index 0000000000000..37412878b6ede --- /dev/null +++ b/packages/component_code_gen/tests/actions/test.py @@ -0,0 +1,89 @@ +import sys +sys.path.append("...") # go back to root - hack to allow importing main +from main import main + + +apps = [ + { + 'app': 'github', + 'instructions': 'how to get a specific repository', + 'key': 'github-get-repository' + }, + { + 'app': 'stripe', + 'instructions': 'how to cancel a payment intent', + 'key': 'stripe-cancel-payment-intent' + }, + { + 'app': 'twilio', + 'instructions': 'how to get a message', + 'key': 'twilio-get-message' + }, + { + 'app': 'woocommerce', + 'instructions': 'how to search for customers', + 'key': 'woocommerce-search-customers' + }, + { + 'app': 'postmark', + 'instructions': 'how to send an email', + 'key': 'postmark-send-single-email' + }, + { + 'app': 'process_street', + 'instructions': 'how to start a workflow run', + 'key': 'process_street-start-workflow-run' + }, + { + 'app': 'zenkit', + 'instructions': 'how to add a comment to an entry/item within a list/collection', + 'key': 'zenkit-add-entry-comment' + }, + { + 'app': 'fibery', + 'instructions': "how to get an entity or create one if it doesn't exist", + 'key': 'fibery-get-entity-or-create' + }, + { + 'app': 'tally', + 'instructions': 'how to get a list of responses', + 'key': 'tally-get-responses' + }, + { + 'app': 'asana', + 'instructions': 'how to update a task', + 'key': 'asana-update-task' + }, + { + 'app': 'accelo', + 'instructions': 'how to create a contact', + 'key': 'accelo-create-contact' + }, + { + 'app': 'shipcloud', + 'instructions': 'how to get information about a shipment', + 'key': 'shipcloud-get-shipment-info' + }, + { + 'app': 'quaderno', + 'instructions': 'how to create an invoice', + 'key': 'quaderno-create-invoice' + }, + { + 'app': 'brex', + 'instructions': 'how to set a limit for an user', + 'key': 'brex-set-limit-for-user' + }, +] + + +def run_tests(): + for app in apps: + print(f"testing {app['app']}...") + result = main('action', app['app'], app['instructions'], verbose=True) + with open(f'./tests/actions/output/{app["key"]}.mjs', 'w') as f: + f.write(result) + + +if __name__ == '__main__': + run_tests() diff --git a/packages/component_code_gen/tests/actions/test.sh b/packages/component_code_gen/tests/actions/test.sh deleted file mode 100644 index 1db55e078ea4d..0000000000000 --- a/packages/component_code_gen/tests/actions/test.sh +++ /dev/null @@ -1,33 +0,0 @@ -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -BASE_PATH=$SCRIPT_DIR/output/ - -echo "running github..." -poetry run python main.py --component_type action --app github "how to get a specific repository" > "$BASE_PATH"/github-get-repository.mjs 2>&1 -echo "running stripe..." -poetry run python main.py --component_type action --app stripe "how to cancel a payment intent" > "$BASE_PATH"/stripe-cancel-payment-intent.mjs 2>&1 -echo "running twilio..." -poetry run python main.py --component_type action --app twilio "how to get a message" > "$BASE_PATH"/twilio-get-message.mjs 2>&1 -echo "running woocommerce..." -poetry run python main.py --component_type action --app woocommerce "how to search for customers" > "$BASE_PATH"/woocommerce-search-customers.mjs 2>&1 -echo "running postmark..." -poetry run python main.py --component_type action --app postmark "how to send an email" > "$BASE_PATH"/postmark-send-single-email.mjs 2>&1 - -echo "running process_street..." -poetry run python main.py --component_type action --app process_street "how to start a workflow run" > "$BASE_PATH"/process_street-start-workflow-run.mjs 2>&1 -echo "running zenkit..." -poetry run python main.py --component_type action --app zenkit "how to add a comment to an entry/item within a list/collection" > "$BASE_PATH"/zenkit-add-entry-comment.mjs 2>&1 -echo "running fibery..." -poetry run python main.py --component_type action --app fibery "how to get an entity or create one if it doesn't exist" > "$BASE_PATH"/fibery-get-entity-or-create.mjs 2>&1 -echo "running tally..." -poetry run python main.py --component_type action --app tally "how to get a list of responses" > "$BASE_PATH"/tally-get-responses.mjs 2>&1 - -echo "running asana..." -poetry run python main.py --component_type action --app asana "how to update a task" > "$BASE_PATH"/asana-update-task.mjs 2>&1 -echo "running accelo..." -poetry run python main.py --component_type action --app accelo "how to create a contact" > "$BASE_PATH"/accelo-create-contact.mjs 2>&1 -echo "running shipcloud..." -poetry run python main.py --component_type action --app shipcloud "how to get information about a shipment" > "$BASE_PATH"/shipcloud-get-shipment-info.mjs 2>&1 -echo "running quaderno..." -poetry run python main.py --component_type action --app quaderno "how to create an invoice" > "$BASE_PATH"/quaderno-create-invoice.mjs 2>&1 -echo "running brex..." -poetry run python main.py --component_type action --app brex "how to set a limit for an user" > "$BASE_PATH"/brex-set-limit-for-user.mjs 2>&1 diff --git a/packages/component_code_gen/tests/webhooks/test.py b/packages/component_code_gen/tests/webhooks/test.py new file mode 100644 index 0000000000000..0a8013de5751b --- /dev/null +++ b/packages/component_code_gen/tests/webhooks/test.py @@ -0,0 +1,89 @@ +import sys +sys.path.append("...") # go back to root - hack to allow importing main +from main import main + + +apps = [ + { + 'app': 'github', + 'instructions': 'how to get webhooks for every new commit', + 'key': 'github-new-commit' + }, + { + 'app': 'stripe', + 'instructions': 'how to get webhooks for every new payment', + 'key': 'stripe-new-payment' + }, + { + 'app': 'twilio', + 'instructions': 'how to get webhooks for every new call', + 'key': 'twilio-new-call' + }, + { + 'app': 'woocommerce', + 'instructions': 'how to get webhooks for every new order event', + 'key': 'woocommerce-new-order-event' + }, + { + 'app': 'postmark', + 'instructions': 'how to get webhooks for every new inbound email', + 'key': 'postmark-new-inbound-email-received' + }, + { + 'app': 'process_street', + 'instructions': 'how to get webhooks for every new completed workflow run', + 'key': 'process_street-workflow-run-completed' + }, + { + 'app': 'zenkit', + 'instructions': 'how to get webhooks for every new notification', + 'key': 'zenkit-new-notification' + }, + { + 'app': 'fibery', + 'instructions': 'how to get webhooks for every new created entity', + 'key': 'fibery-entity-created' + }, + { + 'app': 'tally', + 'instructions': 'how to get webhooks for every new response', + 'key': 'tally-new-response' + }, + { + 'app': 'asana', + 'instructions': 'how to get webhooks for every new project', + 'key': 'asana-new-project' + }, + { + 'app': 'accelo', + 'instructions': 'how to get webhooks for every new assigned task', + 'key': 'accelo-new-task-assigned' + }, + { + 'app': 'shipcloud', + 'instructions': 'how to get webhooks for every new shipment status', + 'key': 'shipcloud-new-shipment-status' + }, + { + 'app': 'quaderno', + 'instructions': 'how to get webhooks for every new received payment', + 'key': 'quaderno-payment-received' + }, + { + 'app': 'brex', + 'instructions': 'how to get webhooks for every new transfer event', + 'key': 'brex-new-transfer-event' + }, +] + + +def run_tests(): + for app in apps: + print(f"testing {app['app']}...") + result = main('webhook_source', app['app'], app['instructions'], verbose=True) + with open(f'./tests/webhooks/output/{app["key"]}.mjs', 'w') as f: + f.write(result) + + +if __name__ == '__main__': + run_tests() diff --git a/packages/component_code_gen/tests/webhooks/test.sh b/packages/component_code_gen/tests/webhooks/test.sh deleted file mode 100755 index 26a5330fd98d9..0000000000000 --- a/packages/component_code_gen/tests/webhooks/test.sh +++ /dev/null @@ -1,33 +0,0 @@ -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -BASE_PATH=$SCRIPT_DIR/output/ - -echo "running github..." -poetry run python generate_webhook_source.py --app github "how to get webhooks for every new commit" > "$BASE_PATH"/github-new-commit.mjs 2>&1 -echo "running stripe..." -poetry run python generate_webhook_source.py --app stripe "how to get webhooks for every new payment" > "$BASE_PATH"/stripe-new-payment.mjs 2>&1 -echo "running twilio..." -poetry run python generate_webhook_source.py --app twilio "how to get webhooks for every new call" > "$BASE_PATH"/twilio-new-call.mjs 2>&1 -echo "running woocommerce..." -poetry run python generate_webhook_source.py --app woocommerce "how to get webhooks for every new order event" > "$BASE_PATH"/woocommerce-new-order-event.mjs 2>&1 -echo "running postmark..." -poetry run python generate_webhook_source.py --app postmark "how to get webhooks for every new inbound email" > "$BASE_PATH"/postmark-new-inbound-email-received.mjs 2>&1 - -echo "running process_street..." -poetry run python generate_webhook_source.py --app process_street "how to get webhooks for every new completed workflow run" > "$BASE_PATH"/process_street-workflow-run-completed.mjs 2>&1 -echo "running zenkit..." -poetry run python generate_webhook_source.py --app zenkit "how to get webhooks for every new notification" > "$BASE_PATH"/zenkit-new-notification.mjs 2>&1 -echo "running fibery..." -poetry run python generate_webhook_source.py --app fibery "how to get webhooks for every new created entity" > "$BASE_PATH"/fibery-entity-created.mjs 2>&1 -echo "running tally..." -poetry run python generate_webhook_source.py --app tally "how to get webhooks for every new response" > "$BASE_PATH"/tally-new-response.mjs 2>&1 - -echo "running asana..." -poetry run python generate_webhook_source.py --app asana "how to get webhooks for every new project" > "$BASE_PATH"/asana-new-project.mjs 2>&1 -echo "running accelo..." -poetry run python generate_webhook_source.py --app accelo "how to get webhooks for every new assigned task" > "$BASE_PATH"/accelo-new-task-assigned.mjs 2>&1 -echo "running shipcloud..." -poetry run python generate_webhook_source.py --app shipcloud "how to get webhooks for every new shipment status" > "$BASE_PATH"/shipcloud-new-shipment-status.mjs 2>&1 -echo "running quaderno..." -poetry run python generate_webhook_source.py --app quaderno "how to get webhooks for every new received payment" > "$BASE_PATH"/quaderno-payment-received.mjs 2>&1 -echo "running brex..." -poetry run python generate_webhook_source.py --app brex "how to get webhooks for every new transfer event" > "$BASE_PATH"/brex-new-transfer-event.mjs 2>&1 From ce6d5abfdcf628e68d7d5e0db4d8207876905bb6 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 17:31:07 -0300 Subject: [PATCH 29/44] return copy of template instead of overwriting --- packages/component_code_gen/code_gen/generate_component_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index b23893c5449b4..bf9ffeaa02b0d 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -65,7 +65,7 @@ def with_docs(app, prompt, docs, docs_type, templates): def add_code_example(templates, example): - templates.no_docs_system_instructions %= example + return templates.no_docs_system_instructions % example def validate_inputs(app, prompt, templates): From 57fabfb0369424c1207a7f25e5981dfcb8b5d872 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 18:03:52 -0300 Subject: [PATCH 30/44] use only one script for running tests --- packages/component_code_gen/README.md | 6 ++-- .../tests/actions/{test.py => apps.py} | 17 ----------- packages/component_code_gen/tests/test.py | 29 +++++++++++++++++++ .../test.py => webhook_sources/apps.py} | 17 ----------- .../reference/accelo-new-task-assigned.mjs | 0 .../reference/asana-new-project.mjs | 0 .../reference/brex-new-transfer-event.mjs | 0 .../reference/fibery-entity-created.mjs | 0 .../reference/github-new-commit.mjs | 0 .../postmark-new-inbound-email-received.mjs | 0 .../process_street-workflow-run.completed.mjs | 0 .../reference/quaderno-payment-received.mjs | 0 .../shipcloud-new-shipment-status.mjs | 0 .../reference/stripe-new-payment.mjs | 0 .../reference/tally-new-response.mjs | 0 .../reference/twilio-new-call.mjs | 0 .../reference/woocommerce-new-order-event.mjs | 0 .../reference/zenkit-new-notification.mjs | 0 18 files changed, 32 insertions(+), 37 deletions(-) rename packages/component_code_gen/tests/actions/{test.py => apps.py} (82%) create mode 100644 packages/component_code_gen/tests/test.py rename packages/component_code_gen/tests/{webhooks/test.py => webhook_sources/apps.py} (83%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/accelo-new-task-assigned.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/asana-new-project.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/brex-new-transfer-event.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/fibery-entity-created.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/github-new-commit.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/postmark-new-inbound-email-received.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/process_street-workflow-run.completed.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/quaderno-payment-received.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/shipcloud-new-shipment-status.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/stripe-new-payment.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/tally-new-response.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/twilio-new-call.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/woocommerce-new-order-event.mjs (100%) rename packages/component_code_gen/tests/{webhooks => webhook_sources}/reference/zenkit-new-notification.mjs (100%) diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 4cd68b37cb93b..8e94560067da4 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -47,11 +47,11 @@ poetry run python main.py --component_type action --app slack --instructions ins ### Tests -To run a suite of tests (e.g. webhooks): +To run a suite of tests (e.g. webhook_sources): ``` -poetry run python -m tests.webhooks.test +poetry run python -m tests.test --type webhook_sources ``` This script will generate code for some selected apps/components for comparison with registry components -Compare `./tests/webhooks/output/*` with `./tests/webhooks/reference/*` +Compare `./tests/webhook_sources/output/*` with `./tests/webhook_sources/reference/*` diff --git a/packages/component_code_gen/tests/actions/test.py b/packages/component_code_gen/tests/actions/apps.py similarity index 82% rename from packages/component_code_gen/tests/actions/test.py rename to packages/component_code_gen/tests/actions/apps.py index 37412878b6ede..ade1f18961ffc 100644 --- a/packages/component_code_gen/tests/actions/test.py +++ b/packages/component_code_gen/tests/actions/apps.py @@ -1,8 +1,3 @@ -import sys -sys.path.append("...") # go back to root - hack to allow importing main -from main import main - - apps = [ { 'app': 'github', @@ -75,15 +70,3 @@ 'key': 'brex-set-limit-for-user' }, ] - - -def run_tests(): - for app in apps: - print(f"testing {app['app']}...") - result = main('action', app['app'], app['instructions'], verbose=True) - with open(f'./tests/actions/output/{app["key"]}.mjs', 'w') as f: - f.write(result) - - -if __name__ == '__main__': - run_tests() diff --git a/packages/component_code_gen/tests/test.py b/packages/component_code_gen/tests/test.py new file mode 100644 index 0000000000000..9c46c111302c0 --- /dev/null +++ b/packages/component_code_gen/tests/test.py @@ -0,0 +1,29 @@ +import argparse +import importlib +import os +import sys +sys.path.append("..") # go back to root - hack to allow importing main +from main import main + + +def run(): + cwd = os.listdir('./tests') + available_tests = list(filter(lambda x: x != '__pycache__' and x != 'test.py', cwd)) + + parser = argparse.ArgumentParser() + parser.add_argument('--type', type=str, choices=available_tests, required=True) + args = parser.parse_args() + + test_type = args.type + + apps = importlib.import_module(f'tests.{test_type}.apps') + + for app in apps.apps: + print(f"testing {app['app']}...") + result = main(test_type[:-1], app['app'], app['instructions'], verbose=True) + with open(f'./tests/{test_type}/output/{app["key"]}.mjs', 'w') as f: + f.write(result) + + +if __name__ == '__main__': + run() diff --git a/packages/component_code_gen/tests/webhooks/test.py b/packages/component_code_gen/tests/webhook_sources/apps.py similarity index 83% rename from packages/component_code_gen/tests/webhooks/test.py rename to packages/component_code_gen/tests/webhook_sources/apps.py index 0a8013de5751b..bf9ee9abf6a6a 100644 --- a/packages/component_code_gen/tests/webhooks/test.py +++ b/packages/component_code_gen/tests/webhook_sources/apps.py @@ -1,8 +1,3 @@ -import sys -sys.path.append("...") # go back to root - hack to allow importing main -from main import main - - apps = [ { 'app': 'github', @@ -75,15 +70,3 @@ 'key': 'brex-new-transfer-event' }, ] - - -def run_tests(): - for app in apps: - print(f"testing {app['app']}...") - result = main('webhook_source', app['app'], app['instructions'], verbose=True) - with open(f'./tests/webhooks/output/{app["key"]}.mjs', 'w') as f: - f.write(result) - - -if __name__ == '__main__': - run_tests() diff --git a/packages/component_code_gen/tests/webhooks/reference/accelo-new-task-assigned.mjs b/packages/component_code_gen/tests/webhook_sources/reference/accelo-new-task-assigned.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/accelo-new-task-assigned.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/accelo-new-task-assigned.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/asana-new-project.mjs b/packages/component_code_gen/tests/webhook_sources/reference/asana-new-project.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/asana-new-project.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/asana-new-project.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/brex-new-transfer-event.mjs b/packages/component_code_gen/tests/webhook_sources/reference/brex-new-transfer-event.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/brex-new-transfer-event.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/brex-new-transfer-event.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/fibery-entity-created.mjs b/packages/component_code_gen/tests/webhook_sources/reference/fibery-entity-created.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/fibery-entity-created.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/fibery-entity-created.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/github-new-commit.mjs b/packages/component_code_gen/tests/webhook_sources/reference/github-new-commit.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/github-new-commit.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/github-new-commit.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/postmark-new-inbound-email-received.mjs b/packages/component_code_gen/tests/webhook_sources/reference/postmark-new-inbound-email-received.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/postmark-new-inbound-email-received.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/postmark-new-inbound-email-received.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/process_street-workflow-run.completed.mjs b/packages/component_code_gen/tests/webhook_sources/reference/process_street-workflow-run.completed.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/process_street-workflow-run.completed.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/process_street-workflow-run.completed.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/quaderno-payment-received.mjs b/packages/component_code_gen/tests/webhook_sources/reference/quaderno-payment-received.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/quaderno-payment-received.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/quaderno-payment-received.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/shipcloud-new-shipment-status.mjs b/packages/component_code_gen/tests/webhook_sources/reference/shipcloud-new-shipment-status.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/shipcloud-new-shipment-status.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/shipcloud-new-shipment-status.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/stripe-new-payment.mjs b/packages/component_code_gen/tests/webhook_sources/reference/stripe-new-payment.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/stripe-new-payment.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/stripe-new-payment.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/tally-new-response.mjs b/packages/component_code_gen/tests/webhook_sources/reference/tally-new-response.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/tally-new-response.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/tally-new-response.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/twilio-new-call.mjs b/packages/component_code_gen/tests/webhook_sources/reference/twilio-new-call.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/twilio-new-call.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/twilio-new-call.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/woocommerce-new-order-event.mjs b/packages/component_code_gen/tests/webhook_sources/reference/woocommerce-new-order-event.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/woocommerce-new-order-event.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/woocommerce-new-order-event.mjs diff --git a/packages/component_code_gen/tests/webhooks/reference/zenkit-new-notification.mjs b/packages/component_code_gen/tests/webhook_sources/reference/zenkit-new-notification.mjs similarity index 100% rename from packages/component_code_gen/tests/webhooks/reference/zenkit-new-notification.mjs rename to packages/component_code_gen/tests/webhook_sources/reference/zenkit-new-notification.mjs From 98e4439f2d50002cefaf6ecab509457b1fff409f Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 8 Aug 2023 18:05:03 -0300 Subject: [PATCH 31/44] change component_type arg to type --- packages/component_code_gen/README.md | 2 +- packages/component_code_gen/main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 8e94560067da4..9336ea032cf15 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -41,7 +41,7 @@ poetry install ### Run ``` -poetry run python main.py --component_type action --app slack --instructions instructions.md --verbose +poetry run python main.py --type action --app slack --instructions instructions.md --verbose ``` diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py index 51226fdf9a536..37bd15baa60cb 100644 --- a/packages/component_code_gen/main.py +++ b/packages/component_code_gen/main.py @@ -27,7 +27,7 @@ def main(component_type, app, instructions, verbose=False): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--component_type', help='which kind of code you want to generate?', choices=available_templates.keys(), required=True) + parser.add_argument('--type', help='which kind of code you want to generate?', choices=available_templates.keys(), required=True) parser.add_argument('--app', help='the app_name_slug', required=True) parser.add_argument('--instructions', help='markdown file with instructions: prompt + api docs', required=True) parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') @@ -36,5 +36,5 @@ def main(component_type, app, instructions, verbose=False): with open(args.instructions, 'r') as f: instructions = f.read() - result = main(args.component_type, args.app, instructions, args.verbose) + result = main(args.type, args.app, instructions, args.verbose) print(result) From 7cfb73cf0d60e7b686acf7b449805b827b13a1ab Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Thu, 10 Aug 2023 14:24:14 -0300 Subject: [PATCH 32/44] add $summary export --- .../templates/generate_actions.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index e082c75f002c7..70ce25b07bbd6 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -25,12 +25,14 @@ } }, async run({steps, $}) { - return await axios($, { + const response = await axios($, { url: `https://api.openai.com/v1/models`, headers: { Authorization: `Bearer ${this.openai.$auth.api_key}`, }, }) + $.export("$summary", "Successfully listed models") + return response }, }) ``` @@ -61,6 +63,8 @@ Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. +A short summary should be exported before the end so that the user can quickly read what has happened. This is done by calling `$.export("$summary", "Your summary here")`. This is not optional. + ## Pipedream Platform Axios If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: @@ -113,7 +117,7 @@ }, }, async run({ steps, $ }) { - return await axios($, { + const response = await axios($, { method: "POST", url: `https://slack.com/api/chat.postMessage`, headers: { @@ -123,7 +127,9 @@ channel: this.channel, text: this.text, }, - }); + }) + $.export("$summary", "Sent message successfully") + return response }, }); @@ -139,9 +145,11 @@ }, }, async run({ steps, $ }) { - return await axios($, { + const response = await axios($, { // Add the axios configuration object to make the HTTP request here - }); + }) + $.export("$summary", "Your summary here") + return response }, }); From 789b200395f64e56186f4cd76da9db70410ac82c Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 14 Aug 2023 19:06:16 -0300 Subject: [PATCH 33/44] add azure openai option and adjust configs --- packages/component_code_gen/.env.example | 14 ++++++- packages/component_code_gen/README.md | 24 ++++++++--- packages/component_code_gen/config/config.py | 41 ++++++++++++++---- .../helpers/langchain_helpers.py | 42 +++++++++++-------- packages/component_code_gen/main.py | 2 +- 5 files changed, 91 insertions(+), 32 deletions(-) diff --git a/packages/component_code_gen/.env.example b/packages/component_code_gen/.env.example index d7a3e02d0e643..82df9f272f86e 100644 --- a/packages/component_code_gen/.env.example +++ b/packages/component_code_gen/.env.example @@ -1,4 +1,16 @@ BROWSERLESS_API_KEY=your-browserless-api-key -OPENAI_API_KEY=your-openai-api-key + SUPABASE_URL=https://your-supabase-url.supabase.co SUPABASE_API_KEY=your-supabase-service-role-key + +OPENAI_API_TYPE=openai +OPENAI_API_KEY=your-openai-api-key +OPENAI_MODEL=gpt-4 + +# comment or remove these below if using openai api +OPENAI_API_TYPE=azure +OPENAI_DEPLOYMENT_NAME=deployment-name +OPENAI_API_VERSION=2023-05-15 +OPENAI_API_BASE=https://resource-name.openai.azure.com +OPENAI_API_KEY=azure-api-key +OPENAI_MODEL=gpt-4-32k diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 9336ea032cf15..74f452de1ae7b 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -20,12 +20,26 @@ poetry install 2. Add these API Keys: - - BROWSERLESS_API_KEY=api_key - - OPENAI_API_KEY=API_KEY - - SUPABASE_URL=https://url.supabase.co - - SUPABASE_API_KEY=service_role_key + - BROWSERLESS_API_KEY=api_key # not required + - SUPABASE_URL=https://your-project-url.supabase.co # get this from Supabase Project Settings -> API + - SUPABASE_API_KEY=service_role_key # get this from Supabase Project Settings -> API -3. Create a `instructions.md` file with a similar structure as the `instructions.md.example` file: +3. Add OpenAI keys + + - OPENAI_API_TYPE=openai + - OPENAI_API_KEY=your-openai-api-key + - OPENAI_MODEL=gpt-4 + +4. Or use a Azure OpenAI deployment (gpt-4-32k) + + - OPENAI_API_TYPE=azure + - OPENAI_DEPLOYMENT_NAME=deployment-name + - OPENAI_API_VERSION=2023-05-15 + - OPENAI_API_BASE=https://resource-name.openai.azure.com + - OPENAI_API_KEY=azure-api-key + - OPENAI_MODEL=gpt-4-32k + +5. Create a file named `instructions.md` with the same structure as the `instructions.md.exaple` file: ``` ## Prompt diff --git a/packages/component_code_gen/config/config.py b/packages/component_code_gen/config/config.py index a3f14bcbe4267..6395aab1dfb67 100644 --- a/packages/component_code_gen/config/config.py +++ b/packages/component_code_gen/config/config.py @@ -1,23 +1,48 @@ +import os from dotenv import load_dotenv load_dotenv() -import os + +DEFAULTS = { + "OPENAI_API_TYPE": "openai", + "OPENAI_MODEL": "gpt-4", + "OPENAI_API_VERSION": "2023-05-15", + "LOGGING_LEVEL": "WARN", + "ENABLE_DOCS": False +} + + +def get_env_var(var_name, required=False): + if os.environ.get(var_name): + return os.environ.get(var_name) + if required and var_name not in DEFAULTS: + raise Exception(f"Environment variable {var_name} is required") + if var_name in DEFAULTS: + return DEFAULTS[var_name] config = { + "openai_api_type": get_env_var("OPENAI_API_TYPE"), "openai": { - "api_key": os.environ.get('OPENAI_API_KEY'), + "api_key": get_env_var("OPENAI_API_KEY", required=True), + "model": get_env_var("OPENAI_MODEL"), + }, + "azure": { + "deployment_name": get_env_var("OPENAI_DEPLOYMENT_NAME"), + "api_version": get_env_var("OPENAI_API_VERSION"), + "api_base": get_env_var("OPENAI_API_BASE"), + "api_key": get_env_var("OPENAI_API_KEY", required=True), + "model": get_env_var("OPENAI_MODEL"), }, "browserless": { - "api_key": os.environ.get('BROWSERLESS_API_KEY'), + "api_key": get_env_var("BROWSERLESS_API_KEY"), }, "supabase": { - "url": os.environ.get('SUPABASE_URL'), - "api_key": os.environ.get('SUPABASE_API_KEY'), + "url": get_env_var("SUPABASE_URL", required=True), + "api_key": get_env_var("SUPABASE_API_KEY", required=True), }, "logging": { - "level": "DEBUG" if os.environ.get('DEBUG') == "1" else "WARN", + "level": get_env_var("LOGGING_LEVEL"), }, - "model": "gpt-4", - "enable_docs": False, + "enable_docs": get_env_var("ENABLE_DOCS"), } diff --git a/packages/component_code_gen/helpers/langchain_helpers.py b/packages/component_code_gen/helpers/langchain_helpers.py index 32f85b3074066..d5490f80c5afa 100644 --- a/packages/component_code_gen/helpers/langchain_helpers.py +++ b/packages/component_code_gen/helpers/langchain_helpers.py @@ -1,13 +1,18 @@ from dotenv import load_dotenv load_dotenv() -import openai +import openai # required from config.config import config from langchain import LLMChain from langchain.agents import ZeroShotAgent, AgentExecutor -from langchain.chat_models import ChatOpenAI +from langchain.chat_models import ChatOpenAI, AzureChatOpenAI from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit from langchain.tools.json.tool import JsonSpec +from langchain.schema import ( + # AIMessage, + HumanMessage, + SystemMessage +) class OpenAPIExplorerTool: @@ -30,8 +35,7 @@ def __init__(self, docs, templates): format_instructions=templates.format_instructions, input_variables=['input', 'agent_scratchpad'] ) - llm = ChatOpenAI(model_name=config["model"], temperature=0, request_timeout=300) - llm_chain = LLMChain(llm=llm, prompt=prompt_template) + llm_chain = LLMChain(llm=get_llm(), prompt=prompt_template) agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names) verbose = True if config['logging']['level'] == 'DEBUG' else False self.agent_executor = AgentExecutor.from_agent_and_tools( @@ -60,6 +64,17 @@ def format_result(result): return result +def get_llm(): + if config['openai_api_type'] == "azure": + azure_config = config["azure"] + llm = AzureChatOpenAI(deployment_name=azure_config['deployment_name'], + model_name=azure_config["model"], temperature=0, request_timeout=300) + else: + openai_config = config["openai"] + llm = ChatOpenAI(model_name=openai_config["model"], temperature=0, request_timeout=300) + return llm + + def ask_agent(user_prompt, docs, templates): agent = PipedreamOpenAPIAgent(docs, templates) result = agent.run(user_prompt) @@ -67,16 +82,9 @@ def ask_agent(user_prompt, docs, templates): def no_docs(app, prompt, templates): - openai.api_key = config['openai']['api_key'] - result = openai.ChatCompletion.create( - model=config["model"], - messages=[ - {"role": "system", "content": format_template(templates.no_docs_system_instructions)}, - {"role": "user", "content": templates.no_docs_user_prompt % (prompt, app)}, - - ], - temperature=0, - ) - - result = result.choices[0].message.content.strip() - return format_result(result) + result = get_llm()(messages=[ + SystemMessage(content=format_template(templates.no_docs_system_instructions)), + HumanMessage(content=templates.no_docs_user_prompt % (prompt, app)), + ]) + + return format_result(result.content) diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py index 37bd15baa60cb..45094021c0b02 100644 --- a/packages/component_code_gen/main.py +++ b/packages/component_code_gen/main.py @@ -12,7 +12,7 @@ def main(component_type, app, instructions, verbose=False): if verbose: - os.environ['DEBUG'] = '1' + os.environ['LOGGING_LEVEL'] = 'DEBUG' try: templates = available_templates[component_type] From 7adb7bffbc4f9d0f717bd345729b59c78f7a92fa Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 15 Aug 2023 13:12:56 -0300 Subject: [PATCH 34/44] add polling source template --- packages/component_code_gen/main.py | 2 + .../templates/generate_polling_sources.py | 482 ++++++++++++++++++ 2 files changed, 484 insertions(+) create mode 100644 packages/component_code_gen/templates/generate_polling_sources.py diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py index 45094021c0b02..3c44023cadc8d 100644 --- a/packages/component_code_gen/main.py +++ b/packages/component_code_gen/main.py @@ -2,11 +2,13 @@ import argparse import templates.generate_actions import templates.generate_webhook_sources +import templates.generate_polling_sources available_templates = { 'action': templates.generate_actions, 'webhook_source': templates.generate_webhook_sources, + 'polling_source': templates.generate_polling_sources, } diff --git a/packages/component_code_gen/templates/generate_polling_sources.py b/packages/component_code_gen/templates/generate_polling_sources.py new file mode 100644 index 0000000000000..7f368c31a44c0 --- /dev/null +++ b/packages/component_code_gen/templates/generate_polling_sources.py @@ -0,0 +1,482 @@ +no_docs_user_prompt = """%s. The app is %s.""" + +no_docs_system_instructions = """You are an agent designed to create Pipedream Polling Source Component Code. + +You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Polling Source Component Code. +You should not return any text other than the code. + +output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! + +## Pipedream Source Components + +All Pipedream polling source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument. + +Here's an example Pipedream source component that fetches all bookmarks from Raindrop.io and emits each bookmark as an event: + +```javascript +import { axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform" +export default { + key: "raindrop-bookmark-created", + name: "New Bookmark Created", + description: `Emit new event when a bookmark is created. [See the documentation](${docsLink})`, + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + raindrop: { + type: "app", + app: "raindrop", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + // fetch collections from the API + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + }, + methods: { + _getPage() { + return this.db.get("page") ?? 0; + }, + _setPage(page) { + this.db.set("page", page); + }, + }, + hooks: { + async deploy() { + let page = 0; + const all_bookmarks = []; + + while (true) { + // fetch bookmarks from the API + const bookmarks = await this.raindrop.getRaindrops(this, this.collectionId, { + page, + perpage: MAX_PER_PAGE, + }); + all_bookmarks.unshift(...bookmarks.reverse()); + + if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; + page++; + } + + for (const bookmark of all_bookmarks.slice(0, 50)) { + this.$emit(bookmark, { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }); + } + + this._setPage(page); + }, + }, + async run() { + let page = this._getPage(); + + while (true) { + // fetch bookmarks from the API + const { items: bookmarks } = await this.raindrop.getRaindrops(this, this.collectionId, { + page, + perpage: 50, + }); + + for (const bookmark of bookmarks) { + this.$emit(bookmark, { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }); + }; + + if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; + + page++; + } + + this._setPage(page); + }, +}; +``` + +This object contains a `props` property, which defines a single prop of type "app": + +```javascript +export default defineComponent({ + props: { + the_app_name: { + type: "app", + app: "the_app_name", + }, + } + // the rest of the component ... +}) +``` + +This lets the user connect their app account to the step, authorizing requests to the app API. + +Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. + +The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. + +Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. + +The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event. + +There are also two other props in sources: `http` and `db`. `http` is an interface that lets you receive and respond to HTTP requests. `db` is a data store that lets you store data between runs of the component. You should always include both. + +The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". + +Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. + +The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. + +## Source Hooks + +Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks. + +## Pipedream Platform Axios + +If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: + +import { axios } from "@pipedream/platform"; + +You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. + +The `axios` constructor takes two arguments: + +1. `this` - the context passed by the run method of the component. + +2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. + +For example: + +async run({steps, $}) { + return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, + }) +}, + +`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +## Async options props + +The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: + +``` +[ + { + label: "Human-readable option 1", + value: "unique identifier 1", + }, + { + label: "Human-readable option 2", + value: "unique identifier 2", + }, +] +``` + +The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. + +If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. + +Example async options methods: + +``` +msg: { + type: "string", + label: "Message", + description: "Select a message to `console.log()`", + async options() { + // write any node code that returns a string[] (with label/value keys) + return ["This is option 1", "This is option 2"]; + }, +}, +``` + +``` +board: { + type: "string", + label: "Board", + async options(opts) { + const boards = await this.getBoards(this.$auth.oauth_uid); + const activeBoards = boards.filter((board) => board.closed === false); + return activeBoards.map((board) => { + return { label: board.name, value: board.id }; + }); + }, +}, +``` + +``` +async options(opts) { + const response = await axios(this, { + method: "GET", + url: `https://api.spotify.com/v1/me/playlists`, + headers: { + Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, + }, + }); + return response.items.map((playlist) => { + return { label: playlist.name, value: playlist.id }; + }); +}, +``` + +## Component Metadata + +Registry components require a unique key and version, and a friendly name and description. E.g. + +``` +export default { + key: "google_drive-new-shared-drive-created", + name: "New Shared Drive Created", + description: "Emits a new event any time a shared drive is created.", + version: "0.0.1", + type: "source", + dedupe: "unique", +}; +``` + +Component keys are in the format app_name_slug-slugified-component-name. +You should come up with a name and a description for the component you are generating. +In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). +Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). +Always add version "0.0.1", type "source", and dedupe "unique". + +## TypeScript Definitions + +export interface Methods { + [key: string]: (...args: any) => unknown; +} + +// $.flow.exit() and $.flow.delay() +export interface FlowFunctions { + exit: (reason: string) => void; + delay: (ms: number) => { + resume_url: string; + cancel_url: string; + }; +} + +export interface Pipedream { + export: (key: string, value: JSONValue) => void; + send: SendFunctionsWrapper; + /** + * Respond to an HTTP interface. + * @param response Define the status and body of the request. + * @returns A promise that is fulfilled when the body is read or an immediate response is issued + */ + respond: (response: HTTPResponse) => Promise | void; + flow: FlowFunctions; +} + +// Arguments to the options method for props +export interface OptionsMethodArgs { + page?: number; + prevContext?: any; + [key: string]: any; +} + +// You can reference the values of previously-configured props! +export interface OptionalOptsFn { + (configuredProps: { [key: string]: any; }): object; +} + +export type PropDefinition = + [App, string] | + [App, string, OptionalOptsFn]; + +// You can reference props defined in app methods, referencing the propDefintion directly in props +export interface PropDefinitionReference { + propDefinition: PropDefinition; +} + +export interface App< + Methods, + AppPropDefinitions +> { + type: "app"; + app: string; + propDefinitions?: AppPropDefinitions; + methods?: Methods & ThisType; +} + +export function defineApp< + Methods, + AppPropDefinitions, +> +(app: App): App { + return app; +} + +// Props + +export interface DefaultConfig { + intervalSeconds?: number; + cron?: string; +} + +export interface Field { + name: string; + value: string; +} + +export interface BasePropInterface { + label?: string; + description?: string; +} + +export type PropOptions = any[] | Array<{ [key: string]: string; }>; + +export interface UserProp extends BasePropInterface { + type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; + options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); + optional?: boolean; + default?: JSONValue; + secret?: boolean; + min?: number; + max?: number; +} + +export interface InterfaceProp extends BasePropInterface { + type: "$.interface.http" | "$.interface.timer"; + default?: string | DefaultConfig; +} + +// When users ask about data stores, remember to include a prop of type "data_store" in the props object +export interface DataStoreProp extends BasePropInterface { + type: "data_store"; +} + +export interface HttpRequestProp extends BasePropInterface { + type: "http_request"; + default?: DefaultHttpRequestPropConfig; +} + +export interface ActionPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; +} + +export interface AppPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp; +} + +export interface ActionRunOptions { + $: Pipedream; + steps: JSONValue; +} + +type PropThis = { + [Prop in keyof Props]: Props[Prop] extends App ? any : any +}; + +export interface Action< + Methods, + ActionPropDefinitions +> { + key: string; + name?: string; + description?: string; + version: string; + type: "action"; + methods?: Methods & ThisType & Methods>; + props?: ActionPropDefinitions; + additionalProps?: ( + previousPropDefs: ActionPropDefinitions + ) => Promise; + run: (this: PropThis & Methods, options?: ActionRunOptions) => any; +} + +export function defineAction< + Methods, + ActionPropDefinitions, +> +(component: Action): Action { + return component; +} + +## Additional rules + +1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. + +2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. + +3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. + +4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. + +5. Always emit relevant data. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. + +6. Always use this signature for the run method: + +async run() { + // your code here +} + +## Remember, return ONLY code + +Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. + +Consider all the instructions and rules above, and use the following code as a template for your code: %s +""" + +with_docs_system_instructions = f"""{no_docs_system_instructions} +You are an agent designed to interact with an OpenAPI JSON specification. +You have access to the following tools which help you learn more about the JSON you are interacting with. +Only use the below tools. Only use the information returned by the below tools to construct your final answer. +Do not make up any information that is not contained in the JSON. +Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. +You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. +If you have not seen a key in one of those responses, you cannot use it. +You should only add one key at a time to the path. You cannot add multiple keys at once. +If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. + +Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. +Then you should proceed to search for the rest of the information to build your answer. + +If the question does not seem to be related to the JSON, just return "I don't know" as the answer. +Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. + +Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". +In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. +Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" + +suffix = """--- +Begin! +Remember, DO NOT include any other text in your response other than the code. +DO NOT return ``` or any other code formatting characters in your response. + +Question: {input} +{agent_scratchpad}""" + +format_instructions = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do. always escape curly brackets +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question. do not include any other text than the code itself""" From a963fe078994d34224f35f73eb6b9ddda1dd98f7 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 16 Aug 2023 15:25:05 -0300 Subject: [PATCH 35/44] add tests for polling components --- .../tests/polling_sources/apps.py | 72 +++++ ...123formbuilder-form-response-submitted.mjs | 175 +++++++++++++ .../reference/bigml-new-model-created.mjs | 152 +++++++++++ .../reference/coda-new-row-created.mjs | 178 +++++++++++++ .../docusign-envelope-sent-or-complete.mjs | 148 +++++++++++ .../reference/drata-failed-monitor.mjs | 168 ++++++++++++ .../faunadb-changes-to-collection.mjs | 138 ++++++++++ .../reference/here-weather-for-zip.mjs | 47 ++++ .../reference/hubspot-new-deal.mjs | 205 +++++++++++++++ .../reference/intercom-new-user-reply.mjs | 136 ++++++++++ .../reference/mailchimp-link-clicked.mjs | 246 ++++++++++++++++++ .../reference/monday-new-board.mjs | 150 +++++++++++ .../notion-updated-page-in-database.mjs | 143 ++++++++++ .../reference/raindrop-new-bookmark.mjs | 103 ++++++++ .../reference/supabase-new-row-added.mjs | 71 +++++ 15 files changed, 2132 insertions(+) create mode 100644 packages/component_code_gen/tests/polling_sources/apps.py create mode 100644 packages/component_code_gen/tests/polling_sources/reference/a123formbuilder-form-response-submitted.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/bigml-new-model-created.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/coda-new-row-created.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/docusign-envelope-sent-or-complete.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/drata-failed-monitor.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/faunadb-changes-to-collection.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/here-weather-for-zip.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/hubspot-new-deal.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/intercom-new-user-reply.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/mailchimp-link-clicked.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/monday-new-board.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/notion-updated-page-in-database.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/raindrop-new-bookmark.mjs create mode 100644 packages/component_code_gen/tests/polling_sources/reference/supabase-new-row-added.mjs diff --git a/packages/component_code_gen/tests/polling_sources/apps.py b/packages/component_code_gen/tests/polling_sources/apps.py new file mode 100644 index 0000000000000..5d829e53c11f7 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/apps.py @@ -0,0 +1,72 @@ +apps = [ + { + 'app': 'a123formbuilder', + 'instructions': 'how to emit events for every new form submission', + 'key': 'a123formbuilder-form-response-submitted' + }, + { + 'app': 'bigml', + 'instructions': 'how to emit events for every new model created', + 'key': 'bigml-new-model-created' + }, + { + 'app': 'coda', + 'instructions': 'how to emit events for every created or updated row', + 'key': 'coda-new-row-created' + }, + { + 'app': 'docusign', + 'instructions': 'how to emit events when an envelope status is set to sent or complete', + 'key': 'docusign-envelope-sent-or-complete' + }, + { + 'app': 'drata', + 'instructions': 'how to emit events whenever a monitor fails', + 'key': 'drata-failed-monitor' + }, + { + 'app': 'faunadb', + 'instructions': 'how to emit events each time you add or remove a document from a specific collection', + 'key': 'faunadb-changes-to-collection' + }, + { + 'app': 'here', + 'instructions': 'how to emit weather reports for a specific zip code on a schedule', + 'key': 'here-weather-for-zip' + }, + { + 'app': 'hubspot', + 'instructions': 'how to emit events for each new deal created', + 'key': 'hubspot-new-deal' + }, + { + 'app': 'intercom', + 'instructions': 'how to emit events each time a user replies to a conversation', + 'key': 'intercom-new-user-reply' + }, + { + 'app': 'mailchimp', + 'instructions': 'how to emit events when a recipient clicks a pre-specified link in an specific campaign', + 'key': 'mailchimp-link-clicked' + }, + { + 'app': 'monday', + 'instructions': 'how to emit events when a new board is created in Monday', + 'key': 'monday-new-board' + }, + { + 'app': 'notion', + 'instructions': 'how to emit events when a page in a database is updated', + 'key': 'notion-updated-page-in-database' + }, + { + 'app': 'raindrop', + 'instructions': 'how to emit events when a bookmark is added', + 'key': 'raindrop-new-bookmark' + }, + { + 'app': 'supabase', + 'instructions': 'how to emit events for every new row added in a table', + 'key': 'supabase-new-row-added' + }, +] diff --git a/packages/component_code_gen/tests/polling_sources/reference/a123formbuilder-form-response-submitted.mjs b/packages/component_code_gen/tests/polling_sources/reference/a123formbuilder-form-response-submitted.mjs new file mode 100644 index 0000000000000..b034ebeef22b0 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/a123formbuilder-form-response-submitted.mjs @@ -0,0 +1,175 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "a123formbuilder-form-response-submitted", + name: "Form Response Submitted", + description: "Emit new event for every submitted form response", + type: "source", + version: "0.0.2", + dedupe: "unique", + props: { + a123formbuilder: { + type: "app", + app: "a123formbuilder", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + form: { + type: "integer", + label: "Form", + description: "The id of a form", + async options({ prevContext }) { + const response = await this.getForms({ + params: { + limit: 10, + page: prevContext?.nextPage, + }, + }); + return { + options: response.data.map((form) => ({ + label: form.name, + value: form.id, + })), + context: { + nextPage: this.getCurrentPage(response) + 1, + }, + }; + }, + }, + }, + methods: { + _baseUrl() { + return `https://${this._region()}.123formbuilder.com/v2`; + }, + _region() { + return this.a123formbuilder.$auth.region; + }, + _auth() { + return this.a123formbuilder.$auth.oauth_access_token; + }, + getCurrentPage(response) { + return response.meta.pagination.current_page; + }, + isLastPage(response) { + return this.getCurrentPage(response) === response.meta.pagination.total_pages; + }, + async _makeRequest({ + $ = this, path, ...opts + }) { + return axios($, { + ...opts, + url: this._baseUrl() + path, + params: { + ...opts.params, + JWT: this._auth(), + }, + }); + }, + async getForms({ + paginate = false, ...opts + }) { + if (paginate) { + return this.paginate({ + ...opts, + fn: this.getForms, + }); + } + return this._makeRequest({ + ...opts, + path: "/forms", + }); + }, + async getFormResponses({ + paginate = false, form, ...opts + }) { + if (paginate) { + return this.paginate({ + ...opts, + fn: this.getFormResponses, + form, + }); + } + return this._makeRequest({ + ...opts, + path: `/forms/${form}/submissions`, + }); + }, + async paginate({ + fn, ...opts + }) { + const data = []; + opts.params = { + ...opts.params, + per_page: 1000, + page: 1, + }; + + while (true) { + const response = await fn.call(this, opts); + data.push(...response.data); + opts.params.page++; + + if (this.isLastPage(response)) { + return { + data, + meta: response.meta, + }; + } + } + }, + getPage() { + return this.db.get("page") || 1; + }, + setPage(page) { + this.db.set("page", page); + }, + getEmittedIds() { + return new Set(this.db.get("emittedIds") || []); + }, + setEmittedIds(ids) { + this.db.set("emittedIds", Array.from(ids)); + }, + getMeta(formResponse) { + return { + id: formResponse.id, + summary: `New form response with id ${formResponse.id}`, + ts: new Date(formResponse.date), + }; + }, + listingFn() { + return this.getFormResponses; + }, + listingFnParams() { + return { + form: this.form, + }; + }, + }, + async run() { + const page = this.getPage(); + const emittedIds = this.getEmittedIds(); + const response = await this.listingFn()({ + ...this.listingFnParams(), + paginate: true, + params: { + page, + }, + }); + this.setPage(this.getCurrentPage(response)); + response.data.forEach((form) => { + if (!emittedIds.has(form.id)) { + this.$emit(form, this.getMeta(form)); + emittedIds.add(form.id); + } + }); + this.setEmittedIds(emittedIds); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/bigml-new-model-created.mjs b/packages/component_code_gen/tests/polling_sources/reference/bigml-new-model-created.mjs new file mode 100644 index 0000000000000..a7e4843782b48 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/bigml-new-model-created.mjs @@ -0,0 +1,152 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "bigml-new-model-created", + name: "New Model Created", + description: "Emit new event for every created model. [See docs here.](https://bigml.com/api/models?id=listing-models)", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + bigml: { + type: "app", + app: "bigml", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + hooks: { + async deploy() { + this._setLastDate(new Date()); + + console.log("Retrieving historical events..."); + const { objects } = await this.listingFunction().call(this, { + params: { + limit: 50, + }, + }); + + for (const object of objects.reverse()) { + this.emitEvent(object); + } + }, + }, + methods: { + _username() { + return this.bigml.$auth.username; + }, + _auth() { + return this.bigml.$auth.api_key; + }, + async _makeRequest({ + $ = this, path, ...opts + }) { + return axios($, { + ...opts, + url: "https://bigml.io/andromeda" + path, + params: { + ...opts.params, + username: this._username(), + api_key: this._auth(), + }, + }); + }, + async paginate({ + fn, ...opts + }) { + const results = []; + const limit = 200; + let offset = 0; + + while (true) { + const { + meta, + objects, + } = await fn.call(this, { + ...opts, + params: { + ...opts.params, + limit, + offset, + }, + }); + + results.push(...objects); + offset += limit; + + if (!meta.next) { + return { + meta, + objects: results, + }; + } + } + }, + async listModels({ + paginate = false, ...opts + } = {}) { + if (paginate) { + return this.paginate({ + fn: this.listModels, + ...opts, + }); + } + return this._makeRequest({ + ...opts, + path: "/model", + }); + }, + _getLastDate() { + return this.db.get("lastDate"); + }, + _setLastDate(lastDate) { + this.db.set("lastDate", lastDate.toISOString().slice(0, -1)); + }, + listingFunction() { + return this.listModels; + }, + emitEvent(model) { + this.$emit(model, { + id: model.resource, + summary: `New model created: ${model.name}`, + ts: model.created, + }); + }, + }, + async run() { + let offset = 0; + + while (true) { + const lastDate = this._getLastDate(); + const currentDate = new Date(); + + const { objects } = await this.listingFunction().call(this, { + paginate: true, + params: { + offset, + limit: 200, + created__gte: lastDate, + }, + }); + + this._setLastDate(currentDate); + offset += objects.length; + + if (objects.length === 0) { + return; + } + + for (const object of objects) { + this.emitEvent(object); + } + } + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/coda-new-row-created.mjs b/packages/component_code_gen/tests/polling_sources/reference/coda-new-row-created.mjs new file mode 100644 index 0000000000000..c4a9004e52960 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/coda-new-row-created.mjs @@ -0,0 +1,178 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "coda-new-row-created", + name: "New Row Created", + description: "Emit new event for every created / updated row in a table. [See the docs here.](https://coda.io/developers/apis/v1#tag/Rows/operation/listRows)", + type: "source", + version: "0.0.1", + props: { + coda: { + type: "app", + app: "coda", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + docId: { + prtype: "string", + label: "Doc ID", + description: "ID of the Doc", + async options({ prevContext }) { + const response = await this.listDocs(this, { + pageToken: prevContext.nextPageToken, + }); + return this._makeOptionsResponse(response); + }, + }, + tableId: { + type: "string", + label: "Table ID", + description: "ID of the table", + async options({ + docId, + prevContext, + }) { + const response = await this.listTables(this, docId, { + pageToken: prevContext.nextPageToken, + }); + return this._makeOptionsResponse(response); + }, + }, + includeUpdates: { + type: "boolean", + label: "Include Updated Rows", + description: "Emit events for updates on existing rows?", + optional: true, + }, + }, + methods: { + _throwFormattedError(err) { + err = err.response.data; + throw Error(`${err.statusCode} - ${err.statusMessage} - ${err.message}`); + }, + _makeOptionsResponse(response) { + return { + options: response.items + .map((e) => ({ + label: e.name, + value: e.id, + })), + context: { + nextPageToken: response.nextPageToken, + }, + }; + }, + async _makeRequest($, opts) { + if (!opts.headers) opts.headers = {}; + opts.headers.Authorization = `Bearer ${this.coda.$auth.api_token}`; + opts.headers["user-agent"] = "@PipedreamHQ/pipedream v0.1"; + if (!opts.method) opts.method = "get"; + const { path } = opts; + delete opts.path; + opts.url = `https://coda.io/apis/v1${path[0] === "/" + ? "" + : "/"}${path}`; + try { + return await axios($ ?? this, opts); + } catch (err) { + this._throwFormattedError(err); + } + }, + async listDocs($, params = {}) { + let opts = { + path: "/docs", + params, + }; + return await this._makeRequest($, opts); + }, + async listTables($, docId, params = {}) { + let opts = { + path: `/docs/${docId}/tables`, + params, + }; + return await this._makeRequest($, opts); + }, + async findRow($, docId, tableId, params = {}) { + let opts = { + path: `/docs/${docId}/tables/${tableId}/rows`, + params, + }; + return await this._makeRequest($, opts); + }, + _getEmittedRows() { + return this.db.get("emittedRows") || []; + }, + _setEmittedRows(rows) { + this.db.set("emittedRows", rows); + }, + _getNextPageToken() { + return this.db.get("nextPageToken"); + }, + _setNextPageToken(nextPageToken) { + nextPageToken && this.db.set("nextPageToken", nextPageToken); + }, + async fetchRows() { + const rows = []; + let nextPageToken = this._getNextPageToken(); + const params = { + pageToken: nextPageToken, + }; + + while (true) { + const { + items = [], + nextPageToken, + } = await this.findRow( + null, + this.docId, + this.tableId, + params, + ); + + rows.push(...items); + params.pageToken = nextPageToken; + this._setNextPageToken(nextPageToken); + + if (!nextPageToken) { + return rows; + } + } + }, + rowSummary(row) { + const name = row.name && ` - ${row.name}` || ""; + return `Row index: ${row.index}` + name; + }, + emitEvents(events) { + const emittedRows = this._getEmittedRows(); + + for (const row of events) { + const id = this.includeUpdates + ? `${row.id}-${row.updatedAt}` + : row.id; + + if (!emittedRows.includes(id)) { + emittedRows.unshift(id); + this.$emit(row, { + id, + summary: this.rowSummary(row), + ts: row.updatedAt, + }); + } + } + + this._setEmittedRows(emittedRows); + }, + }, + async run() { + const rows = await this.fetchRows(); + this.emitEvents(rows.reverse()); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/docusign-envelope-sent-or-complete.mjs b/packages/component_code_gen/tests/polling_sources/reference/docusign-envelope-sent-or-complete.mjs new file mode 100644 index 0000000000000..8fb8669bb93f7 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/docusign-envelope-sent-or-complete.mjs @@ -0,0 +1,148 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "docusign-envelope-sent-or-complete", + version: "0.0.4", + name: "Envelope Sent or Complete", + description: "Emit new event when an envelope status is set to sent or complete", + type: "source", + props: { + docusign: { + type: "app", + app: "docusign", + }, + db: "$.service.db", + timer: { + label: "Polling Interval", + description: "Pipedream will poll the Docusign API on this schedule", + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + account: { + type: "string", + label: "Account", + description: "Docusign Account", + async options() { + const { accounts } = await this.getUserInfo({}); + return accounts.map((account) => ({ + label: account.account_name, + value: account.account_id, + })); + }, + }, + status: { + type: "string[]", + label: "Status", + description: "The envelope status that you are checking for", + options: [ + "sent", + "completed", + ], + default: [ + "sent", + ], + }, + }, + methods: { + _getHeaders() { + return { + "Authorization": `Bearer ${this.docusign.$auth.oauth_access_token}`, + "Content-Type": "application/json", + }; + }, + async _makeRequest({ + $, config, + }) { + config.headers = this._getHeaders(); + try { + return await axios($ ?? this, config); + } catch (e) { + throw new Error(e.response.data.message); + } + }, + async getBaseUri({ + $, accountId, + }) { + const { accounts } = await this.getUserInfo({ + $, + }); + const account = accounts.find((a) => a.account_id === accountId); + const { base_uri: baseUri } = account; + return `${baseUri}/restapi/v2.1/accounts/${accountId}/`; + }, + async getUserInfo({ $ }) { + const config = { + method: "GET", + url: "https://account.docusign.com/oauth/userinfo", + }; + return this._makeRequest({ + $, + config, + }); + }, + async listEnvelopes(baseUri, params) { + const config = { + method: "GET", + url: `${baseUri}envelopes`, + params, + }; + return this._makeRequest({ + config, + }); + }, + _getLastEvent() { + return this.db.get("lastEvent"); + }, + _setLastEvent(lastEvent) { + this.db.set("lastEvent", lastEvent); + }, + monthAgo() { + const monthAgo = new Date(); + monthAgo.setMonth(monthAgo.getMonth() - 1); + return monthAgo; + }, + generateMeta({ + envelopeId: id, emailSubject: summary, status, + }, ts) { + return { + id: `${id}${status}`, + summary, + ts, + }; + }, + }, + async run(event) { + const { timestamp: ts } = event; + const lastEvent = this._getLastEvent() || this.monthAgo().toISOString(); + const baseUri = await this.getBaseUri({ + accountId: this.account, + }); + let done = false; + const params = { + from_date: lastEvent, + status: this.status.join(), + }; + do { + const { + envelopes = [], + nextUri, + endPosition, + } = await this.listEnvelopes(baseUri, params); + if (nextUri) { + params.start_position += endPosition + 1; + } + else done = true; + + for (const envelope of envelopes) { + const meta = this.generateMeta(envelope, ts); + this.$emit(envelope, meta); + } + } while (!done); + this._setLastEvent(new Date(ts * 1000).toISOString()); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/drata-failed-monitor.mjs b/packages/component_code_gen/tests/polling_sources/reference/drata-failed-monitor.mjs new file mode 100644 index 0000000000000..bc677d548c1e0 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/drata-failed-monitor.mjs @@ -0,0 +1,168 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +const docsLink = "https://developers.drata.com/docs/openapi/reference/operation/MonitorsPublicController_listMonitors/"; + +export default { + key: "drata-failed-monitor", + name: "Failed Monitor", + description: `Emit a new event whenever a monitor fails. [See the documentation](${docsLink}).`, + type: "source", + version: "0.0.2", + dedupe: "unique", + props: { + drata: { + type: "app", + app: "drata", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + checkType: { + type: "string", + label: "Check Type", + description: "Filter monitors by check type (associated monitor instances)", + optional: true, + options: [ + "POLICY", + "IN_DRATA", + "AGENT", + "INFRASTRUCTURE", + "VERSION_CONTROL", + "IDENTITY", + "TICKETING", + "HRIS", + "OBSERVABILITY", + ], + }, + }, + hooks: { + async deploy() { + const response = await this.listMonitors({ + paginate: true, + params: { + checkResultStatus: "FAILED", + reportInterval: "WEEKLY", + }, + }); + + const visitedIds = {}; + for (const monitor of response.data) { + const ts = Date.parse(monitor.lastCheck); + visitedIds[monitor.id] = ts; + + this.$emit(monitor, { + id: `${monitor.id}_${ts}`, + summary: `Historical failed monitor event: ${monitor.name}`, + ts, + }); + } + + this._setVisitedIds(visitedIds); + }, + }, + methods: { + async _makeRequest({ + $ = this, path = "/", ...opts + }) { + return axios($, { + ...opts, + url: `https://public-api.drata.com/public${path}`, + headers: { + ...opts.headers, + Authorization: `Bearer ${this.drata.$auth.api_key}`, + }, + }); + }, + async paginate({ + fn, ...opts + }) { + const { total } = await fn.call(this, { + ...opts, + params: { + ...opts.params, + limit: 1, + }, + }); + + const promises = []; + const numberOfPages = Math.ceil(total / 50); + for (let page = 1; page <= numberOfPages; page++) { + promises.push(fn.call(this, { + ...opts, + params: { + ...opts.params, + limit: 50, + page, + }, + })); + } + + const responses = await Promise.all(promises); + const results = responses.reduce((results, { data }) => ([ + ...results, + ...data, + ]), []); + + return { + data: results, + page: numberOfPages, + total, + }; + }, + async listMonitors({ + paginate = false, ...opts + }) { + if (paginate) { + return this.paginate({ + ...opts, + fn: this.listMonitors, + }); + } + return this._makeRequest({ + ...opts, + path: "/monitors", + }); + }, + _getVisitedIds() { + return this.db.get("visitedIds") || {}; + }, + _setVisitedIds(visitedIds) { + this.db.set("visitedIds", visitedIds); + }, + }, + async run() { + const visitedIds = this._getVisitedIds(); + + const response = await this.listMonitors({ + paginate: true, + params: { + checkResultStatus: "FAILED", + reportInterval: "WEEKLY", + }, + }); + + for (const monitor of response.data) { + const id = monitor.id; + const ts = Date.parse(monitor.lastCheck); + + if (!visitedIds[id] || ts > visitedIds[id]) { + visitedIds[id] = ts; + + this.$emit(monitor, { + id: `${monitor.id}_${ts}`, + summary: `Failed: ${monitor.name}`, + ts, + }); + } + } + + this._setVisitedIds(visitedIds); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/faunadb-changes-to-collection.mjs b/packages/component_code_gen/tests/polling_sources/reference/faunadb-changes-to-collection.mjs new file mode 100644 index 0000000000000..c17b4f9bdd10c --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/faunadb-changes-to-collection.mjs @@ -0,0 +1,138 @@ +import faunadb from "faunadb"; +import _ from "lodash"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +const Client = faunadb.Client; +const { + Collection, + Collections, + Documents, +} = faunadb.query; + +export default { + type: "source", + key: "faunadb-changes-to-collection", + name: "New or Removed Documents in a Collection", + description: "Emit new event each time you add or remove a document from a specific collection, with the details of the document.", + version: "0.0.8", + dedupe: "unique", // Dedupe events based on the concatenation of event + document ref id + props: { + fauna: { + type: "app", + app: "fauna", + }, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + db: "$.service.db", + collection: { + type: "string", + label: "Collection", + description: "The ID of a collection", + async options() { + const collections = await this.getCollections(); + return collections.map((collection) => collection.id); + }, + }, + emitEventsInBatch: { + type: "boolean", + label: "Emit changes as a single event", + description: "If `true`, all events are emitted as an array, within a single Pipedream event. Defaults to `false`, emitting each event in Fauna as its own event in Pipedream", + optional: true, + default: false, + }, + }, + methods: { + _authToken() { + return this.faunadb.$auth.secret; + }, + _apiUrl() { + return "https://graphql.fauna.com"; + }, + _createApiClient() { + return new Client({ + secret: this._authToken(), + }); + }, + async getCollections() { + const client = this._createApiClient(); + + const collections = []; + const collectionsPaginator = client.paginate(Collections()); + + await collectionsPaginator.each((page) => { + collections.push(...page); + }); + + await client.close(); + return collections; + }, + async getEventsInCollectionAfterTs(collection, after) { + const client = this._createApiClient(); + + const paginationHelper = client.paginate( + Documents(Collection(collection)), + { + after, + events: true, + }, + ); + + const events = []; + await paginationHelper.each((page) => { + events.push(...page); + }); + + await client.close(); + return events; + }, + }, + async run() { + // As soon as the script runs, mark the start time so we can fetch changes + // since this time on the next run. Fauna expects epoch ms as its cursor. + const ts = +new Date() * 1000; + const cursor = this.db.get("cursor") || ts; + + const events = await this.getEventsInCollectionAfterTs( + this.collection, + cursor, + ); + + if (!events.length) { + console.log(`No new events in collection ${this.collection}`); + this.db.set("cursor", ts); + return; + } + + console.log(`${events.length} new events in collection ${this.collection}`); + + // Batched emits do not take advantage of the built-in deduper + if (this.emitEventsInBatch) { + this.$emit({ + events, + }, { + summary: `${events.length} new event${events.length > 1 + ? "s" + : ""}`, + id: cursor, + }); + } else { + for (const event of events) { + this.$emit(event, { + summary: `${event.action.toUpperCase()} - ${event.document.id}`, + id: `${event.action}-${event.document.id}`, // dedupes events based on this ID + }); + } + } + + // Finally, set cursor for the next run to the max timestamp of the changed events, ensuring we + // get all events after that on the next run. We need to add 1 since the timestamp filter in + // Fauna is inclusive: https://docs.fauna.com/fauna/current/api/fql/functions/paginate + const maxEventTs = _.maxBy(events, (event) => event.ts).ts + 1; + + this.db.set("cursor", maxEventTs); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/here-weather-for-zip.mjs b/packages/component_code_gen/tests/polling_sources/reference/here-weather-for-zip.mjs new file mode 100644 index 0000000000000..b086bb5eaf4ca --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/here-weather-for-zip.mjs @@ -0,0 +1,47 @@ +const { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } = require("@pipedream/platform"); + +module.exports = { + name: "Weather for ZIP Code", + version: "0.0.3", + key: "here-weather-for-zip", + description: "Emits the weather report for a specific ZIP code on a schedule", + type: "source", + props: { + here: { + type: "app", + app: "here", + }, + zipCode: { + type: "integer", + label: "ZIP code", + description: "The ZIP code you'd like to pull weather stats for (only supported for locations in the United States)", + }, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _apiUrl() { + return "https://weather.ls.hereapi.com/weather/1.0"; + }, + _apiKey() { + return this.here.$auth.apikey; + }, + async returnReportForZIP(zipCode) { + const baseUrl = this._apiUrl(); + return await require("@pipedream/platform").axios(this, { + url: `${baseUrl}/report.json?apiKey=${this._apiKey()}&product=observation&zipcode=${zipCode}`, + }); + }, + }, + async run() { + const report = await this.returnReportForZIP(this.zipCode); + this.$emit(report, { + summary: `Weather report for ${this.zipCode} at ${report.feedCreation}`, + ts: Date.now(), + }); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/hubspot-new-deal.mjs b/packages/component_code_gen/tests/polling_sources/reference/hubspot-new-deal.mjs new file mode 100644 index 0000000000000..5f2e208a85c90 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/hubspot-new-deal.mjs @@ -0,0 +1,205 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; +import Bottleneck from "bottleneck"; + +export default { + key: "hubspot-new-deal", + name: "New Deals", + description: "Emit new event for each new deal created. [See the docs here](https://developers.hubspot.com/docs/api/crm/search)", + version: "0.0.15", + dedupe: "unique", + type: "source", + props: { + hubspot: { + type: "app", + app: "hubspot", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + pipeline: { + type: "string", + label: "Pipeline", + description: "Filter deals by pipeline", + optional: true, + async options() { + const { results } = await this.getPipelines("deal"); + return results.map((result) => { + const { + label, + id: value, + } = result; + return { + label, + value, + }; + }); + }, + }, + stage: { + type: "string", + label: "Stage", + description: "Filter deals by stage", + optional: true, + async options({ pipeline }) { + const { results } = await this.getDealStages(pipeline); + return results.map((result) => { + const { + label, + id, + } = result; + return { + label, + value: id, + }; + }); + }, + }, + }, + methods: { + _getHeaders() { + return { + "Authorization": `Bearer ${this.hubspot.$auth.oauth_access_token}`, + "Content-Type": "application/json", + }; + }, + async makeRequest(api, endpoint, opts = {}) { + const { + method = "GET", + params, + data, + $, + } = opts; + const config = { + method, + url: `${"https://api.hubapi.com"}${api}${endpoint}`, + headers: this._getHeaders(), + params, + data, + }; + return axios($ ?? this, config); + }, + async getPipelines(objectType, $) { + return this.makeRequest("/crm/v3", `/pipelines/${objectType}`, { + $, + }); + }, + async getDealStages(pipelineId, $) { + return this.makeRequest("/crm/v3", `/pipelines/deal/${pipelineId}/stages`, { + $, + }); + }, + async searchCRM(params, after) { + await this.paginate( + params, + this.makeRequest, + "results", + after, + ); + }, + async paginate(params, resourceFn, resultType = null, after = null) { + let results = null; + let maxTs = after || 0; + const limiter = this._limiter(); + while (!results || params.after) { + results = await this._requestWithLimiter(limiter, resourceFn, params); + if (results.paging) { + params.after = results.paging.next.after; + } else { + delete params.after; + } + if (resultType) { + results = results[resultType]; + } + + for (const result of results) { + if (await this.isRelevant(result, after)) { + this.emitEvent(result); + const ts = this.getTs(result); + if (ts > maxTs) { + maxTs = ts; + this._setAfter(ts); + } + } else { + return; + } + } + } + }, + _getAfter() { + return this.db.get("after") || new Date().setDate(new Date().getDate() - 1); // 1 day ago + }, + _setAfter(after) { + this.db.set("after", after); + }, + _limiter() { + return new Bottleneck({ + minTime: 250, // max 4 requests per second + }); + }, + async _requestWithLimiter(limiter, resourceFn, params) { + return limiter.schedule(async () => await resourceFn(params)); + }, + getTs(deal) { + return Date.parse(deal.createdAt); + }, + generateMeta(deal) { + const { + id, + properties, + } = deal; + const ts = this.getTs(deal); + return { + id, + summary: properties.dealname, + ts, + }; + }, + isRelevant(deal, createdAfter) { + return this.getTs(deal) > createdAfter; + }, + getParams() { + const params = { + limit: 100, + sorts: [ + { + propertyName: "createdate", + direction: "DESCENDING", + }, + ], + object: "deals", + }; + if (this.pipeline) { + params.filters = [ + { + propertyName: "pipeline", + operator: "EQ", + value: this.pipeline, + }, + ]; + if (this.stage) { + params.filters.push({ + propertyName: "dealstage", + operator: "EQ", + value: this.stage, + }); + } + } + return params; + }, + async processResults(after, params) { + await this.searchCRM(params, after); + }, + }, + async run() { + const after = this._getAfter(); + const params = this.getParams(after); + await this.processResults(after, params); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/intercom-new-user-reply.mjs b/packages/component_code_gen/tests/polling_sources/reference/intercom-new-user-reply.mjs new file mode 100644 index 0000000000000..16e44ae6eb4f8 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/intercom-new-user-reply.mjs @@ -0,0 +1,136 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "intercom-new-user-reply", + name: "New Reply From User", + description: "Emit new event each time a user replies to a conversation.", + version: "0.0.3", + type: "source", + dedupe: "unique", + props: { + intercom: { + type: "app", + app: "intercom", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + label: "Polling Interval", + description: "Pipedream will poll the API on this schedule", + }, + }, + methods: { + monthAgo() { + const monthAgo = new Date(); + monthAgo.setMonth(monthAgo.getMonth() - 1); + return monthAgo; + }, + async makeRequest(opts) { + const { + method, + url, + endpoint, + data, + $, + } = opts; + const config = { + method, + url: url ?? `https://api.intercom.io/${endpoint}`, + headers: { + Authorization: `Bearer ${this.intercom.$auth.oauth_access_token}`, + Accept: "application/json", + }, + data, + }; + return axios($ || this, config); + }, + async paginate(itemType, method, data, isSearch = false, lastCreatedAt) { + let results = null; + let done = false; + let items = []; + while ((!results || results?.pages?.next) && !done) { + const startingAfter = results?.pages?.next?.starting_after || null; + const search = isSearch && "/search" || ""; + const startingAfterParam = startingAfter && `?starting_after=${startingAfter}` || ""; + const endpoint = `${itemType}${search}${startingAfterParam}`; + results = await this.makeRequest({ + method, + endpoint, + data, + }); + if (lastCreatedAt) { + for (const item of results.data) { + if (item.created_at > lastCreatedAt) + items.push(item); + else + done = true; + } + } else { + items = items.concat(results.data); + if (!startingAfter) + done = true; + } + } + return items; + }, + async getConversation(id) { + return this.makeRequest({ + method: "GET", + endpoint: `conversations/${id}`, + }); + }, + async searchConversations(data) { + return this.paginate("conversations", "POST", data, true); + }, + _getLastUpdate() { + const monthAgo = this.monthAgo(); + return this.db.get("lastUpdate") || Math.floor(monthAgo / 1000); + }, + _setLastUpdate(lastUpdate) { + this.db.set("lastUpdate", lastUpdate); + }, + generateMeta(conversation, conversationData, conversationBody, totalCount) { + return { + id: conversationData.conversation_parts.conversation_parts[totalCount - 1].id, + summary: conversationBody, + ts: conversation.statistics.last_admin_reply_at, + }; + }, + }, + async run() { + let lastContactReplyAt = this._getLastUpdate(); + const data = { + query: { + field: "statistics.last_contact_reply_at", + operator: ">", + value: lastContactReplyAt, + }, + }; + + const results = await this.searchConversations(data); + for (const conversation of results) { + if (conversation.statistics.last_contact_reply_at > lastContactReplyAt) + lastContactReplyAt = conversation.statistics.last_contact_reply_at; + const conversationData = ( + await this.getConversation(conversation.id) + ); + const totalCount = conversationData.conversation_parts.total_count; + const conversationBody = + conversationData?.conversation_parts?.conversation_parts[totalCount - 1]?.body; + if (totalCount > 0 && conversationBody) { + // emit id & summary from last part/reply added + const meta = + this.generateMeta(conversation, conversationData, conversationBody, totalCount); + this.$emit(conversationData, meta); + } + } + + this._setLastUpdate(lastContactReplyAt); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/mailchimp-link-clicked.mjs b/packages/component_code_gen/tests/polling_sources/reference/mailchimp-link-clicked.mjs new file mode 100644 index 0000000000000..a44b324977e2b --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/mailchimp-link-clicked.mjs @@ -0,0 +1,246 @@ +import mailchimp from "@mailchimp/mailchimp_marketing"; +import retry from "async-retry"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + key: "mailchimp-link-clicked", + name: "Link Clicked", + description: "Emit new event when a recipient clicks a pre-specified link in an specific campaign.", + version: "0.0.2", + type: "source", + dedupe: "unique", + props: { + mailchimp: { + type: "app", + app: "mailchimp", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + campaignId: { + type: "string", + label: "Campaign ID", + description: "The unique ID of the campaign you'd like to watch for new clicks on links", + useQuery: true, + async options({ page }) { + const count = 1000; + const offset = count * page; + const campaigns = await this.getCampaignsByCreationDate({ + count, + offset, + }); + return campaigns.map((campaign) => { + const lsdIdx = campaign.long_archive_url.lastIndexOf("/"); + const campaignName = lsdIdx > 0 + ? campaign.long_archive_url.substring(lsdIdx + 1) + : ""; + const label = `Campaign ID/Name from URL (if any): ${campaign.id}/${campaignName}`; + return { + label, + value: campaign.id, + }; + }); + }, + }, + linkId: { + type: "string", + label: "Campaign Link", + description: "The campaign link to track for clicks", + async options(opts) { + const links = await this.getCampaignClickDetails(opts.campaignId); + if (!links.urls_clicked.length) { + throw new Error("No link data available for the selected campaignId"); + } + return links.urls_clicked.map((link) => ({ + label: link.url, + value: link.id, + })); + }, + }, + uniqueClicksOnly: { + type: "boolean", + label: "Unique Clicks Only?", + description: "Whether to count every link click or only count clicks coming from each user only once", + default: false, + }, + }, + hooks: { + async deploy() { + // Emits sample events on the first run during deploy. + return this.emitReportSampleEvents(this.campaignId, this.linkId, (Date.now())); + }, + }, + methods: { + _auths() { + return this.mailchimp.$auth; + }, + _authToken() { + return this.mailchimp.$auth.oauth_access_token; + }, + _server() { + return this.mailchimp.$auth.dc; + }, + api() { + mailchimp.setConfig({ + accessToken: this._authToken(), + server: this._server(), + }); + return mailchimp; + }, + _isRetriableStatusCode(statusCode) { + [ + 408, + 429, + 500, + ].includes(statusCode); + }, + async _withRetries(apiCall) { + const retryOpts = { + retries: 5, + factor: 2, + }; + return retry(async (bail) => { + try { + return await apiCall(); + } catch (err) { + const { status = 500 } = err; + if (!this._isRetriableStatusCode(status)) { + bail(` + Unexpected error (status code: ${status}): + ${JSON.stringify(err.response)} + `); + } + + throw err; + } + }, retryOpts); + }, + async listCampaignOpenDetails(campaignId) { + return await this._withRetries(() => this.api().reports.getCampaignOpenDetails(campaignId)); + }, + async getCampaignClickDetailsForLink(campaignId, linkId) { + const mailchimp = this.api(); + return await this._withRetries(() => + mailchimp.reports.getCampaignClickDetailsForLink(campaignId, linkId)); + }, + async emitReportSampleEvents(reportId, rptParamId, timestamp) { + this.clearCampaignDetailsCache(); + let report; + if (this.getEventTypes().includes("opens")) { + report = await this.listCampaignOpenDetails( + reportId, + ); + } else { + report = await this.getCampaignClickDetailsForLink( + reportId, + rptParamId, + ); + } + if (!report) { + throw new Error("Report metrics not found."); + } + const diff = this.getEventTypes().includes("opens") ? + report.total_opens : + report.total_clicks; + this.processEvent({ + eventPayload: report, + diff, + timestamp, + }); + this.cacheCampaignDetails(report); + }, + async emitReportEvents() { + const cachedDetails = this.getCachedCampaignDetails(); + const detailsInfo = await this.getCampaignDetails(); + const currentDetails = this.getCurrentCampaignDetails(detailsInfo); + if (!detailsInfo) { + throw new Error(this.getNodataErrorMessage()); + } + const diff = this.getDetailsDiff(currentDetails, cachedDetails); + if (diff <= 0) { + console.log(`No new ${this.getEventTypes()[0]}. Skipping...`); + return; + } + this.processEvent({ + eventPayload: detailsInfo, + diff, + timestamp: (new Date()).getTime(), + }); + this.cacheCampaignDetails(currentDetails); + }, + slugifyEmail(email) { + return email + .replace(/[@]/g, "-at-") + .replace(/[.]/g, "-"); + }, + getDbServiceVariable(variable) { + return this.db.get(`${variable}`); + }, + processEvent(event) { + const meta = this.generateMeta(event); + this.$emit(event, meta); + }, + setDbServiceVariable(variable, value) { + this.db.set(`${variable}`, value); + }, + + getEventTypes() { + return [ + "clicks", + ]; + }, + generateMeta({ + eventPayload, + diff: clickDiff, + timestamp: ts, + }) { + const { id: linkId } = eventPayload; + return { + id: `${linkId}-${ts}`, + summary: `${clickDiff} new clicks`, + ts, + }; + }, + getCachedCampaignDetails() { + return this.getDbServiceVariable("recipientClicks"); + }, + async getCampaignDetails() { + return this.getCampaignClickDetailsForLink( + this.campaignId, + this.linkId, + ); + }, + getNodataErrorMessage() { + return "No data found for specified campaign and link."; + }, + getCurrentCampaignDetails(report) { + return this.uniqueClicksOnly + ? report.unique_clicks + : report.total_clicks; + }, + getDetailsDiff(currentRecipientClicks, recipientClicks) { + return currentRecipientClicks - recipientClicks; + }, + cacheCampaignDetails(currentRecipientClicks) { + if (isNaN(currentRecipientClicks)) { + if (this.uniqueClicksOnly) { + this.setDbServiceVariable("recipientClicks", currentRecipientClicks.unique_clicks); + } else { + this.setDbServiceVariable("recipientClicks", currentRecipientClicks.total_clicks); + } + } else { + this.setDbServiceVariable("recipientClicks", currentRecipientClicks); + } + }, + clearCampaignDetailsCache() { + this.setDbServiceVariable("recipientClicks", 0); + }, + }, + async run() { + return this.emitReportEvents(); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/monday-new-board.mjs b/packages/component_code_gen/tests/polling_sources/reference/monday-new-board.mjs new file mode 100644 index 0000000000000..d54392b434dff --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/monday-new-board.mjs @@ -0,0 +1,150 @@ +import mondaySdk from "monday-sdk-js"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + key: "monday-new-board", + name: "New Board", + description: "Emit new event when a new board is created in Monday.", + type: "source", + version: "0.0.5", + dedupe: "unique", + props: { + monday: { + type: "app", + app: "monday", + }, + db: "$.service.db", + timer: { + label: "Polling interval", + description: "Pipedream will poll the Monday API on this schedule", + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + maxRequests: { + type: "integer", + min: 1, + label: "Max API Requests per Execution", + description: "The maximum number of API requests to make per execution (e.g., multiple requests are required to retrieve paginated results)", + optional: true, + default: 1, + }, + }, + methods: { + _getLastId() { + return this.db.get("lastId") || 0; + }, + _setLastId(lastId) { + this.db.set("lastId", lastId); + }, + emitEvent(item) { + const meta = this.generateMeta(item); + this.$emit(item, meta); + }, + generateMeta(board) { + return { + id: board.id, + summary: board.name, + ts: Date.now(), + }; + }, + async makeRequest({ + query, options, + }) { + const monday = mondaySdk(); + monday.setToken(this.monday.$auth.api_key); + return monday.api(query, options); + }, + async getBoard(variables) { + const { data } = await this.makeRequest({ + query: ` + query getBoard($id: Int!) { + boards (ids: [$id]) { + id + name + board_folder_id + columns { + id + } + description + groups { + id + } + items { + id + } + owner { + id + } + permissions + tags { + id + } + type + updated_at + workspace_id + } + } + `, + options: { + variables, + }, + }); + return data?.boards[0]; + }, + async listBoards(variables) { + return this.makeRequest({ + query: ` + query listBoards ( + $page: Int = 1 + ) { + boards( + page: $page + state: all + order_by: created_at + ) { + id + name + type + } + } + `, + options: { + variables, + }, + }); + }, + }, + async run() { + const lastId = this._getLastId(); + + let maxId = lastId; + let done = false; + let page = 1; + do { + const { boards } = (await this.listBoards({ + page, + })).data; + for (const board of boards) { + if (+board.id <= lastId) { + done = true; + break; + } + if (+board.id > maxId) { + maxId = +board.id; + } + const boardData = await this.getBoard({ + id: +board.id, + }); + this.emitEvent(boardData); + } + if (boards.length === 0) { + done = true; + } + page++; + } while (!done && page <= this.maxRequests); + + this._setLastId(maxId); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/notion-updated-page-in-database.mjs b/packages/component_code_gen/tests/polling_sources/reference/notion-updated-page-in-database.mjs new file mode 100644 index 0000000000000..f7e884cdb0355 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/notion-updated-page-in-database.mjs @@ -0,0 +1,143 @@ +import notion from "@notionhq/client"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + key: "notion-updated-page-in-database", + name: "Updated Page in Database", + description: "Emit new event when a page in a database is updated. To select a specific page, use `Updated Page ID` instead", + version: "0.0.6", + type: "source", + dedupe: "unique", + props: { + notion: { + type: "app", + app: "notion", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + databaseId: { + type: "string", + label: "Database ID", + description: "The identifier for a Notion database", + async options({ prevContext }) { + const response = await this.listDatabases({ + start_cursor: prevContext.nextPageParameters ?? undefined, + }); + const options = this._extractDatabaseTitleOptions(response.results); + return this._buildPaginatedOptions(options, response.next_cursor); + }, + }, + }, + methods: { + _getNotionClient() { + return new notion.Client({ + auth: this.notion.$auth.oauth_access_token, + notionVersion: "2022-02-22", + }); + }, + _extractDatabaseTitleOptions(databases) { + return databases.map((database) => { + const title = database.title + .map((title) => title.plain_text) + .filter((title) => title.length > 0) + .reduce((prev, next) => prev + next, ""); + return { + label: title || "Untitled", + value: database.id, + }; + }); + }, + _buildPaginatedOptions(options, nextPageParameters) { + return { + options, + context: { + nextPageParameters, + }, + }; + }, + async listDatabases(params = {}) { + return this._getNotionClient().search({ + filter: { + property: "object", + value: "database", + }, + ...params, + }); + }, + async queryDatabase(databaseId, params = {}) { + return this._getNotionClient().databases.query({ + database_id: databaseId, + ...params, + }); + }, + async *getPages(databaseId, opts = {}) { + let cursor; + + do { + const params = { + ...opts, + start_cursor: cursor, + }; + const response = await this.queryDatabase(databaseId, params); + const { + results: pages, + next_cursor: nextCursor, + } = response; + + for (const page of pages) { + yield page; + } + + cursor = nextCursor; + } while (cursor); + }, + isResultNew(result, startTimestamp) { + return Date.parse(result) > startTimestamp; + }, + daysAgo(days) { + return new Date().setDate(new Date().getDate() - days); + }, + getLastUpdatedTimestamp() { + return this.db.get("last_edited_time") ?? this.daysAgo(7); + }, + setLastUpdatedTimestamp(ts) { + this.db.set("last_edited_time", ts); + }, + lastUpdatedSortParam(params = {}) { + return lastSortParam("last_edited_time", params); + }, + }, + async run() { + const params = this.lastUpdatedSortParam(); + const lastCheckedTimestamp = this.getLastUpdatedTimestamp(); + + const pagesStream = this.getPages(this.databaseId, params); + + for await (const page of pagesStream) { + if (!this.isResultNew(page.last_edited_time, lastCheckedTimestamp)) { + break; + } + + this.$emit(page); + + this.setLastUpdatedTimestamp(Date.parse(page?.last_edited_time)); + } + }, +}; + +function lastSortParam(timestamp, params = {}) { + return { + ...params, + sorts: [ + { + timestamp, + direction: "descending", + }, + ], + }; +} diff --git a/packages/component_code_gen/tests/polling_sources/reference/raindrop-new-bookmark.mjs b/packages/component_code_gen/tests/polling_sources/reference/raindrop-new-bookmark.mjs new file mode 100644 index 0000000000000..76cd2966cfafc --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/raindrop-new-bookmark.mjs @@ -0,0 +1,103 @@ +import { + axios, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "raindrop-new-bookmark", + name: "New Bookmark", + description: "Emit new event when a bookmark is added", + type: "source", + version: "0.0.3", + dedupe: "unique", + props: { + raindrop: { + type: "app", + app: "raindrop", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + method, + url: `https://api.raindrop.io/rest/v1${path}`, + headers: { + ...opts.headers, + "user-agent": "@PipedreamHQ/pipedream v0.1", + "Authorization": `Bearer ${this.raindrop.$auth.oauth_access_token}`, + }, + data, + params, + }); + }, + async getRaindrops($, collectionId, params) { + return this._makeRequest($, { + path: `/raindrops/${collectionId}`, + params, + }); + }, + _getPage() { + return this.db.get("page") ?? 0; + }, + _setPage(page) { + this.db.set("page", page); + }, + getMetadata(bookmark) { + return { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }; + }, + emitEvents(bookmarks) { + bookmarks.forEach((bookmark) => { + const meta = this.getMetadata(bookmark); + this.$emit(bookmark, meta); + }); + }, + }, + async run() { + let page = this._getPage(); + + while (true) { + const { items: bookmarks } = await this.getRaindrops(this, this.collectionId, { + page, + perpage: 25, + }); + this.emitEvents(bookmarks); + + if (bookmarks.length < 25) break; + + page++; + } + + this._setPage(page); + }, +}; diff --git a/packages/component_code_gen/tests/polling_sources/reference/supabase-new-row-added.mjs b/packages/component_code_gen/tests/polling_sources/reference/supabase-new-row-added.mjs new file mode 100644 index 0000000000000..b7e658dc2b024 --- /dev/null +++ b/packages/component_code_gen/tests/polling_sources/reference/supabase-new-row-added.mjs @@ -0,0 +1,71 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import { createClient } from "@supabase/supabase-js"; + +export default { + key: "supabase-new-row-added", + name: "New Row Added", + description: "Emit new event for every new row added in a table. [See documentation here](https://supabase.com/docs/reference/javascript/select)", + version: "0.0.1", + type: "source", + props: { + supabase: { + type: "app", + app: "supabase", + }, + table: { + type: "string", + label: "Table", + description: "The name of the table to watch for new rows", + }, + rowIdentifier: { + type: "string", + label: "Row Identifier", + description: "The column name to use as the row identifier", + optional: true, + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _getOffset() { + return this.db.get("offset") || 0; + }, + _setOffset(offset) { + this.db.set("offset", offset); + }, + async _client() { + return createClient(`https://${this.supabase.$auth.subdomain}.supabase.co`, this.supabase.$auth.service_key); + }, + }, + async run() { + const { + table, + rowIdentifier, + } = this; + + const offset = this._getOffset(); + const client = await this._client(); + const query = client + .from(table) + .select() + .range(offset, offset + 1000); + + const { data } = await query; + this._setOffset(offset + data.length); + + for (const row of data) { + let summary = "New row in table"; + if (row[rowIdentifier]) { + summary = `${summary}: ${row[rowIdentifier]}`; + } + this.$emit(row, { + summary, + }); + } + }, +}; From 088076e02587e73e350c43a1065162d2066094e5 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 16 Aug 2023 17:05:42 -0300 Subject: [PATCH 36/44] alphabetical reorder tests --- .../component_code_gen/tests/actions/apps.py | 74 +++++++++---------- .../tests/webhook_sources/apps.py | 74 +++++++++---------- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/packages/component_code_gen/tests/actions/apps.py b/packages/component_code_gen/tests/actions/apps.py index ade1f18961ffc..d85fddeb8debb 100644 --- a/packages/component_code_gen/tests/actions/apps.py +++ b/packages/component_code_gen/tests/actions/apps.py @@ -1,23 +1,28 @@ apps = [ { - 'app': 'github', - 'instructions': 'how to get a specific repository', - 'key': 'github-get-repository' + 'app': 'accelo', + 'instructions': 'how to create a contact', + 'key': 'accelo-create-contact' }, { - 'app': 'stripe', - 'instructions': 'how to cancel a payment intent', - 'key': 'stripe-cancel-payment-intent' + 'app': 'asana', + 'instructions': 'how to update a task', + 'key': 'asana-update-task' }, { - 'app': 'twilio', - 'instructions': 'how to get a message', - 'key': 'twilio-get-message' + 'app': 'brex', + 'instructions': 'how to set a limit for an user', + 'key': 'brex-set-limit-for-user' }, { - 'app': 'woocommerce', - 'instructions': 'how to search for customers', - 'key': 'woocommerce-search-customers' + 'app': 'fibery', + 'instructions': "how to get an entity or create one if it doesn't exist", + 'key': 'fibery-get-entity-or-create' + }, + { + 'app': 'github', + 'instructions': 'how to get a specific repository', + 'key': 'github-get-repository' }, { 'app': 'postmark', @@ -30,14 +35,19 @@ 'key': 'process_street-start-workflow-run' }, { - 'app': 'zenkit', - 'instructions': 'how to add a comment to an entry/item within a list/collection', - 'key': 'zenkit-add-entry-comment' + 'app': 'quaderno', + 'instructions': 'how to create an invoice', + 'key': 'quaderno-create-invoice' }, { - 'app': 'fibery', - 'instructions': "how to get an entity or create one if it doesn't exist", - 'key': 'fibery-get-entity-or-create' + 'app': 'shipcloud', + 'instructions': 'how to get information about a shipment', + 'key': 'shipcloud-get-shipment-info' + }, + { + 'app': 'stripe', + 'instructions': 'how to cancel a payment intent', + 'key': 'stripe-cancel-payment-intent' }, { 'app': 'tally', @@ -45,28 +55,18 @@ 'key': 'tally-get-responses' }, { - 'app': 'asana', - 'instructions': 'how to update a task', - 'key': 'asana-update-task' - }, - { - 'app': 'accelo', - 'instructions': 'how to create a contact', - 'key': 'accelo-create-contact' - }, - { - 'app': 'shipcloud', - 'instructions': 'how to get information about a shipment', - 'key': 'shipcloud-get-shipment-info' + 'app': 'twilio', + 'instructions': 'how to get a message', + 'key': 'twilio-get-message' }, { - 'app': 'quaderno', - 'instructions': 'how to create an invoice', - 'key': 'quaderno-create-invoice' + 'app': 'woocommerce', + 'instructions': 'how to search for customers', + 'key': 'woocommerce-search-customers' }, { - 'app': 'brex', - 'instructions': 'how to set a limit for an user', - 'key': 'brex-set-limit-for-user' + 'app': 'zenkit', + 'instructions': 'how to add a comment to an entry/item within a list/collection', + 'key': 'zenkit-add-entry-comment' }, ] diff --git a/packages/component_code_gen/tests/webhook_sources/apps.py b/packages/component_code_gen/tests/webhook_sources/apps.py index bf9ee9abf6a6a..40b162b338a76 100644 --- a/packages/component_code_gen/tests/webhook_sources/apps.py +++ b/packages/component_code_gen/tests/webhook_sources/apps.py @@ -1,23 +1,28 @@ apps = [ { - 'app': 'github', - 'instructions': 'how to get webhooks for every new commit', - 'key': 'github-new-commit' + 'app': 'accelo', + 'instructions': 'how to get webhooks for every new assigned task', + 'key': 'accelo-new-task-assigned' }, { - 'app': 'stripe', - 'instructions': 'how to get webhooks for every new payment', - 'key': 'stripe-new-payment' + 'app': 'asana', + 'instructions': 'how to get webhooks for every new project', + 'key': 'asana-new-project' }, { - 'app': 'twilio', - 'instructions': 'how to get webhooks for every new call', - 'key': 'twilio-new-call' + 'app': 'brex', + 'instructions': 'how to get webhooks for every new transfer event', + 'key': 'brex-new-transfer-event' }, { - 'app': 'woocommerce', - 'instructions': 'how to get webhooks for every new order event', - 'key': 'woocommerce-new-order-event' + 'app': 'fibery', + 'instructions': 'how to get webhooks for every new created entity', + 'key': 'fibery-entity-created' + }, + { + 'app': 'github', + 'instructions': 'how to get webhooks for every new commit', + 'key': 'github-new-commit' }, { 'app': 'postmark', @@ -30,14 +35,19 @@ 'key': 'process_street-workflow-run-completed' }, { - 'app': 'zenkit', - 'instructions': 'how to get webhooks for every new notification', - 'key': 'zenkit-new-notification' + 'app': 'quaderno', + 'instructions': 'how to get webhooks for every new received payment', + 'key': 'quaderno-payment-received' }, { - 'app': 'fibery', - 'instructions': 'how to get webhooks for every new created entity', - 'key': 'fibery-entity-created' + 'app': 'shipcloud', + 'instructions': 'how to get webhooks for every new shipment status', + 'key': 'shipcloud-new-shipment-status' + }, + { + 'app': 'stripe', + 'instructions': 'how to get webhooks for every new payment', + 'key': 'stripe-new-payment' }, { 'app': 'tally', @@ -45,28 +55,18 @@ 'key': 'tally-new-response' }, { - 'app': 'asana', - 'instructions': 'how to get webhooks for every new project', - 'key': 'asana-new-project' - }, - { - 'app': 'accelo', - 'instructions': 'how to get webhooks for every new assigned task', - 'key': 'accelo-new-task-assigned' - }, - { - 'app': 'shipcloud', - 'instructions': 'how to get webhooks for every new shipment status', - 'key': 'shipcloud-new-shipment-status' + 'app': 'twilio', + 'instructions': 'how to get webhooks for every new call', + 'key': 'twilio-new-call' }, { - 'app': 'quaderno', - 'instructions': 'how to get webhooks for every new received payment', - 'key': 'quaderno-payment-received' + 'app': 'woocommerce', + 'instructions': 'how to get webhooks for every new order event', + 'key': 'woocommerce-new-order-event' }, { - 'app': 'brex', - 'instructions': 'how to get webhooks for every new transfer event', - 'key': 'brex-new-transfer-event' + 'app': 'zenkit', + 'instructions': 'how to get webhooks for every new notification', + 'key': 'zenkit-new-notification' }, ] From a8cc5cae2e82363d20184213093996ac537b7cca Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Mon, 21 Aug 2023 09:29:19 -0300 Subject: [PATCH 37/44] add apps template --- packages/component_code_gen/main.py | 2 + .../templates/generate_apps.py | 466 ++++++++++++++++++ 2 files changed, 468 insertions(+) create mode 100644 packages/component_code_gen/templates/generate_apps.py diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py index 3c44023cadc8d..7c20c88962ea9 100644 --- a/packages/component_code_gen/main.py +++ b/packages/component_code_gen/main.py @@ -3,12 +3,14 @@ import templates.generate_actions import templates.generate_webhook_sources import templates.generate_polling_sources +import templates.generate_apps available_templates = { 'action': templates.generate_actions, 'webhook_source': templates.generate_webhook_sources, 'polling_source': templates.generate_polling_sources, + 'app': templates.generate_apps, } diff --git a/packages/component_code_gen/templates/generate_apps.py b/packages/component_code_gen/templates/generate_apps.py new file mode 100644 index 0000000000000..65b29c4392da5 --- /dev/null +++ b/packages/component_code_gen/templates/generate_apps.py @@ -0,0 +1,466 @@ +no_docs_user_prompt = """%s. The app is %s.""" + +no_docs_system_instructions = """You are an agent designed to create Pipedream App Code. + +You will receive a prompt from an user. You should create a code in Node.js using Pipedream axios for HTTP requests. Your goal is to create a Pipedream App Code. +You should not return any text other than the code. + +output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! + +## Pipedream Apps + +All Pipedream apps are Node.js modules that have a default export: an javascript object - a Pipedream app - as its single argument. +It is essentially a wrapper on an API that requires authentication. Pipedream facades the authentication data in an object accessed by `this.$auth`. All app objects have three four keys: type, app, propDefinitions, and methods. The app object contains a `type` property, which is always set to "app". The `app` property is the name of the app, e.g. "google_sheets". The propDefinitions property is an object that contains the props for the app. The methods property is an object that contains the methods for the app. + +Here's an example Pipedream app for Raindrop: + +```javascript +import { axios } from "@pipedream/platform"; + +export default { + type: "app", + app: "raindrop", + propDefinitions: { + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + raindropId: { + type: "string", + label: "Bookmark ID", + description: "Existing Bookmark ID", + async options({ + prevContext, collectionId, + }) { + const page = prevContext.page + ? prevContext.page + : 0; + const { items } = await this.getRaindrops(this, collectionId, { + page, + }); + return { + options: items.map((e) => ({ + value: e._id, + label: e.title, + })), + context: { + page: page + 1, + }, + }; + }, + }, + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + method, + url: `https://api.raindrop.io/rest/v1${path}`, + headers: { + ...opts.headers, + "user-agent": "@PipedreamHQ/pipedream v0.1", + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, + }, + data, + params, + }); + }, + async getCollections($) { + return this._makeRequest($, { + path: "/collections", + }); + }, + async getRaindrops($, collectionId, params) { + return this._makeRequest($, { + path: `/raindrops/${collectionId}`, + params, + }); + }, + }, +}; +``` + +This object contains a `propDefinitions` property, which contains the definitions for the props of the app. + +The propDefinitions object contains two props: collectionId and raindropId. The collectionId prop is a string prop. The raindropId prop is also a string prop. The propDefinitions object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options. + +This object contains a `props` property, which defines a single prop of type "app": + +```javascript +import { axios } from "@pipedream/platform"; +export default { + type: "app", + app: "the_app_name", + propDefinitions: { + prop_key: { + type: "string", + label: "Prop Label", + description: "A description of the prop", + async options() { + const static_options = ["option 1", "option 2"]; // a static array of options + const dynamic_options = await this.getOptions(); // a Promise that resolves to an array of options. + return dynamic_options; // return the options + }, + } + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return await axios($, { + ...otherOpts, + method, + url: `https://api.the_app_name.com${path}`, // the base URL of the app API + headers: { + ...opts.headers, + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, // the authentication type depends on the app + }, + params, + data, + }) + }, + async getOptions() { + // the code to get the options + return await this._makeRequest({ + ...opts, + }) + }, + }, +} +``` + +This lets the user connect their app account to the step, authorizing requests to the app API. + +`this` exposes the user's app credentials in the object `this.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. + +The app code should contain a `propDefinitions` property, which are the definitions for the props. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. + +The `methods` property contains auxiliary methods. A `async _makeRequest` method is always required. It contains the code that makes the API request. It takes two arguments: `$ = this` and `opts`. `$` is the context passed by the Pipedream runtime. It should default to `this`. `opts` is an object that contains the parameters of the API request. The `opts` object may contain the following fields: `method`, `path`, `data`, `params`, and `headers`. The `method` field is the HTTP method of the request. The `path` field is the path of the request. The `data` field is the body of the request. The `params` field is the query parameters of the request. The `headers` field is the headers of the request. The `opts` object also contains any other fields that are passed to the `_makeRequest` method. The `_makeRequest` method returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +The axios request uses the authentication method defined by the app. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. The axios request should use the format from the app docs. + +Auxiliary methods, usually for CRUD operations call `_makeRequest` with the appropriate parameters. Please add a few methods for common operations, e.g. get and list. You can also add other methods that you think are useful. + +For listing operations, verify how the pagination is done in the API. Also add a method for pagination. This method should be named `paginate`, and the arguments are `fn`, the listing method that will be called, and `...opts`, the parameters of the HTTP request. The method starts with an empty array, and calls the listing method with the parameters. It then checks the response and verifies if there is more data. If it does, it calls itself with the listing method and the parameters for fetching the next set of data. If it doesn't, it returns the array of results. + +## Pipedream Platform Axios + +Always use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: + +import { axios } from "@pipedream/platform"; + +You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. + +The `axios` constructor takes two arguments: + +1. `this` - the context passed by the run method of the component. + +2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. + +For example: + +return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, +}) + +`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +## Async options + +The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: + +``` +[ + { + label: "Human-readable option 1", + value: "unique identifier 1", + }, + { + label: "Human-readable option 2", + value: "unique identifier 2", + }, +] +``` + +The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. + +If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. + +Example async options methods: + +``` +msg: { + type: "string", + label: "Message", + description: "Select a message to `console.log()`", + async options() { + // write any node code that returns a string[] (with label/value keys) + return ["This is option 1", "This is option 2"]; + }, +}, +``` + +``` +board: { + type: "string", + label: "Board", + async options(opts) { + const boards = await this.getBoards(this.$auth.oauth_uid); + const activeBoards = boards.filter((board) => board.closed === false); + return activeBoards.map((board) => { + return { label: board.name, value: board.id }; + }); + }, +}, +``` + +``` +async options(opts) { + const response = await axios(this, { + method: "GET", + url: `https://api.spotify.com/v1/me/playlists`, + headers: { + Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, + }, + }); + return response.items.map((playlist) => { + return { label: playlist.name, value: playlist.id }; + }); +}, +``` + +## TypeScript Definitions + +export interface Methods { + [key: string]: (...args: any) => unknown; +} + +// $.flow.exit() and $.flow.delay() +export interface FlowFunctions { + exit: (reason: string) => void; + delay: (ms: number) => { + resume_url: string; + cancel_url: string; + }; +} + +export interface Pipedream { + export: (key: string, value: JSONValue) => void; + send: SendFunctionsWrapper; + /** + * Respond to an HTTP interface. + * @param response Define the status and body of the request. + * @returns A promise that is fulfilled when the body is read or an immediate response is issued + */ + respond: (response: HTTPResponse) => Promise | void; + flow: FlowFunctions; +} + +// Arguments to the options method for props +export interface OptionsMethodArgs { + page?: number; + prevContext?: any; + [key: string]: any; +} + +// You can reference the values of previously-configured props! +export interface OptionalOptsFn { + (configuredProps: { [key: string]: any; }): object; +} + +export type PropDefinition = + [App, string] | + [App, string, OptionalOptsFn]; + +// You can reference props defined in app methods, referencing the propDefintion directly in props +export interface PropDefinitionReference { + propDefinition: PropDefinition; +} + +export interface App< + Methods, + AppPropDefinitions +> { + type: "app"; + app: string; + propDefinitions?: AppPropDefinitions; + methods?: Methods & ThisType; +} + +export function defineApp< + Methods, + AppPropDefinitions, +> +(app: App): App { + return app; +} + +// Props + +export interface DefaultConfig { + intervalSeconds?: number; + cron?: string; +} + +export interface Field { + name: string; + value: string; +} + +export interface BasePropInterface { + label?: string; + description?: string; +} + +export type PropOptions = any[] | Array<{ [key: string]: string; }>; + +export interface UserProp extends BasePropInterface { + type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; + options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); + optional?: boolean; + default?: JSONValue; + secret?: boolean; + min?: number; + max?: number; +} + +export interface InterfaceProp extends BasePropInterface { + type: "$.interface.http" | "$.interface.timer"; + default?: string | DefaultConfig; +} + +// When users ask about data stores, remember to include a prop of type "data_store" in the props object +export interface DataStoreProp extends BasePropInterface { + type: "data_store"; +} + +export interface HttpRequestProp extends BasePropInterface { + type: "http_request"; + default?: DefaultHttpRequestPropConfig; +} + +export interface ActionPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; +} + +export interface AppPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp; +} + +export interface ActionRunOptions { + $: Pipedream; + steps: JSONValue; +} + +type PropThis = { + [Prop in keyof Props]: Props[Prop] extends App ? any : any +}; + +export interface Action< + Methods, + ActionPropDefinitions +> { + key: string; + name?: string; + description?: string; + version: string; + type: "action"; + methods?: Methods & ThisType & Methods>; + props?: ActionPropDefinitions; + additionalProps?: ( + previousPropDefs: ActionPropDefinitions + ) => Promise; + run: (this: PropThis & Methods, options?: ActionRunOptions) => any; +} + +export function defineAction< + Methods, + ActionPropDefinitions, +> +(component: Action): Action { + return component; +} + +## Additional rules + +1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. + +2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. + +3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. + +4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. + +## Remember, return ONLY code + +Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. + +Consider all the instructions and rules above, and use the following code as a template for your code: %s +""" + +with_docs_system_instructions = f"""{no_docs_system_instructions} +You are an agent designed to interact with an OpenAPI JSON specification. +You have access to the following tools which help you learn more about the JSON you are interacting with. +Only use the below tools. Only use the information returned by the below tools to construct your final answer. +Do not make up any information that is not contained in the JSON. +Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. +You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. +If you have not seen a key in one of those responses, you cannot use it. +You should only add one key at a time to the path. You cannot add multiple keys at once. +If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. + +Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. +Then you should proceed to search for the rest of the information to build your answer. + +If the question does not seem to be related to the JSON, just return "I don't know" as the answer. +Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. + +Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". +In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. +Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" + +suffix = """--- +Begin! +Remember, DO NOT include any other text in your response other than the code. +DO NOT return ``` or any other code formatting characters in your response. + +Question: {input} +{agent_scratchpad}""" + +format_instructions = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do. always escape curly brackets +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question. do not include any other text than the code itself""" From 80e81aae5372e2cdfdc13166e179f1acac4a9fc5 Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Sat, 26 Aug 2023 17:47:00 -0700 Subject: [PATCH 38/44] Testing Stable Diffusion image to text --- .../dream-studio-text-to-image.md | 1849 +++++++++++++++++ .../templates/generate_actions.py | 73 +- 2 files changed, 1915 insertions(+), 7 deletions(-) create mode 100644 packages/component_code_gen/example-instructions/dream-studio-text-to-image.md diff --git a/packages/component_code_gen/example-instructions/dream-studio-text-to-image.md b/packages/component_code_gen/example-instructions/dream-studio-text-to-image.md new file mode 100644 index 0000000000000..df14fc927e6c0 --- /dev/null +++ b/packages/component_code_gen/example-instructions/dream-studio-text-to-image.md @@ -0,0 +1,1849 @@ +## Prompt + +Generate a new image from a text prompt. + +Use the app slug "dreamstudio". Auth is in this.dreamstudio.$auth.api_key. + +Let me select the engine with an async options param from the "List all engines available to your organization/user" endpoint. This endpoint doesn't use pagination and just returns an array of objects (engines). Return the name property as the label and the id as the value. + +When you hit the image generation endpoint, generate an image of type image/png. + +Save the image to a file and return the file path. + +## API docs + +Here's the OpenAPI spec: + +{ +"openapi": "3.0.3", +"info": { +"termsOfService": "https://platform.stability.ai/docs/terms-of-service", +"description": "Welcome to the Stability.ai REST API!\n\nYour DreamStudio API key will be required for authentication: [How to find your API key](https://platform.stability.ai/docs/getting-started/authentication)\n\nAPI operations use the following versioning scheme:\n- `/v*` interface is stable and ready for production workloads\n- `/v*beta*`: interface is stable, preparing for production release\n- `/v*alpha*`: under development and the interface is subject to change\n\nNOTE: The v1alpha and v1beta endpoints from the developer preview are still available, but they\nwill disabled on May 1st, 2023. Please migrate to the v1 endpoints as soon as possible.\n\nIf you have feedback or encounter any issues with the API, please reach out:\n - [https://github.com/Stability-AI/REST-API](https://github.com/Stability-AI/REST-API)\n - [https://discord.gg/stablediffusion #API channel](https://discord.com/channels/1002292111942635562/1042896447311454361)\n", +"title": "Stability.ai REST API", +"version": "v1", +"x-logo": { +"altText": "Stability.ai REST API", +"url": "/docs/StabilityLogo.png" +} +}, +"servers": [ +{ +"url": "https://api.stability.ai" +} +], +"tags": [ +{ +"name": "v1/user", +"description": "Manage your Stability.ai account, and view account/organization balances" +}, +{ +"name": "v1/engines", +"description": "Enumerate available engines" +}, +{ +"name": "v1/generation", +"description": "Generate images from text, existing images, or both" +} +], +"paths": { +"/v1/generation/{engine*id}/text-to-image": { +"post": { +"description": "Generate a new image from a text prompt", +"operationId": "textToImage", +"summary": "text-to-image", +"tags": [ +"v1/generation" +], +"parameters": [ +{ +"$ref": "#/components/parameters/engineID" +}, +{ +"$ref": "#/components/parameters/accept" +}, +{ +"$ref": "#/components/parameters/organization" +}, +{ +"$ref": "#/components/parameters/stabilityClientID" +}, +{ +"$ref": "#/components/parameters/stabilityClientVersion" +} +], +"requestBody": { +"content": { +"application/json": { +"example": { +"cfg_scale": 7, +"clip_guidance_preset": "FAST_BLUE", +"height": 512, +"width": 512, +"sampler": "K_DPM_2_ANCESTRAL", +"samples": 1, +"steps": 75, +"text_prompts": [ +{ +"text": "A lighthouse on a cliff", +"weight": 1 +} +] +}, +"schema": { +"$ref": "#/components/schemas/TextToImageRequestBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK response.", + "content": { + "application/json": { + "schema": { + "description": "The result of the generation request, containing one or more images as base64 encoded strings.", + "items": { + "$ref": "#/components/schemas/Image" +}, +"type": "array" +} +}, +"image/png": { +"example": "PNG bytes, what did you expect?", +"schema": { +"description": "The bytes of the generated image", +"format": "binary", +"type": "string" +} +} +}, +"headers": { +"Content-Length": { +"$ref": "#/components/headers/Content-Length" + }, + "Content-Type": { + "$ref": "#/components/headers/Content-Type" +}, +"Finish-Reason": { +"$ref": "#/components/headers/Finish-Reason" + }, + "Seed": { + "$ref": "#/components/headers/Seed" +} +} +}, +"400": { +"$ref": "#/components/responses/400FromGeneration" + }, + "401": { + "$ref": "#/components/responses/401" +}, +"403": { +"$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" +}, +"500": { +"$ref": "#/components/responses/500" +} +}, +"security": [ +{ +"STABILITY_API_KEY": [] +} +], +"x-codeSamples": [ +{ +"lang": "Python", +"source": "import base64\nimport os\nimport requests\n\nengine_id = \"stable-diffusion-xl-1024-v1-0\"\napi_host = os.getenv('API_HOST', 'https://api.stability.ai')\napi_key = os.getenv(\"STABILITY_API_KEY\")\n\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.post(\n f\"{api_host}/v1/generation/{engine_id}/text-to-image\",\n headers={\n \"Content-Type\": \"application/json\",\n \"Accept\": \"application/json\",\n \"Authorization\": f\"Bearer {api_key}\"\n },\n json={\n \"text_prompts\": [\n {\n \"text\": \"A lighthouse on a cliff\"\n }\n ],\n \"cfg_scale\": 7,\n \"height\": 1024,\n \"width\": 1024,\n \"samples\": 1,\n \"steps\": 30,\n },\n)\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\ndata = response.json()\n\nfor i, image in enumerate(data[\"artifacts\"]):\n with open(f\"./out/v1_txt2img*{i}.png\", \"wb\") as f:\n f.write(base64.b64decode(image[\"base64\"]))\n" +}, +{ +"label": "TypeScript", +"lang": "Javascript", +"source": "import fetch from 'node-fetch'\nimport fs from 'node:fs'\n\nconst engineId = 'stable-diffusion-xl-1024-v1-0'\nconst apiHost = process.env.API*HOST ?? 'https://api.stability.ai'\nconst apiKey = process.env.STABILITY_API_KEY\n\nif (!apiKey) throw new Error('Missing Stability API key.')\n\nconst response = await fetch(\n `${apiHost}/v1/generation/${engineId}/text-to-image`,\n {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n Accept: 'application/json',\n Authorization: `Bearer ${apiKey}`,\n },\n body: JSON.stringify({\n text_prompts: [\n {\n text: 'A lighthouse on a cliff',\n },\n ],\n cfg_scale: 7,\n height: 1024,\n width: 1024,\n steps: 30,\n samples: 1,\n }),\n }\n)\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\ninterface GenerationResponse {\n artifacts: Array<{\n base64: string\n seed: number\n finishReason: string\n }>\n}\n\nconst responseJSON = (await response.json()) as GenerationResponse\n\nresponseJSON.artifacts.forEach((image, index) => {\n fs.writeFileSync(\n `./out/v1_txt2img*${index}.png`,\n Buffer.from(image.base64, 'base64')\n )\n})\n" + }, + { + "lang": "Go", + "source": "package main\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n)\n\ntype TextToImageImage struct {\n\tBase64 string `json:\"base64\"`\n\tSeed uint32 `json:\"seed\"`\n\tFinishReason string `json:\"finishReason\"`\n}\n\ntype TextToImageResponse struct {\n\tImages []TextToImageImage `json:\"artifacts\"`\n}\n\nfunc main() {\n\t// Build REST endpoint URL w/ specified engine\n\tengineId := \"stable-diffusion-xl-1024-v1-0\"\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/generation/\" + engineId + \"/text-to-image\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\tvar data = []byte(`{\n\t\t\"text_prompts\": [\n\t\t {\n\t\t\t\"text\": \"A lighthouse on a cliff\"\n\t\t }\n\t\t],\n\t\t\"cfg_scale\": 7,\n\t\t\"height\": 1024,\n\t\t\"width\": 1024,\n\t\t\"samples\": 1,\n\t\t\"steps\": 30\n \t}`)\n\n\treq, _ := http.NewRequest(\"POST\", reqUrl, bytes.NewBuffer(data))\n\treq.Header.Add(\"Content-Type\", \"application/json\")\n\treq.Header.Add(\"Accept\", \"application/json\")\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\n\t// Execute the request & read all the bytes of the body\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\n\tif res.StatusCode != 200 {\n\t\tvar body map[string]interface{}\n\t\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tpanic(fmt.Sprintf(\"Non-200 response: %s\", body))\n\t}\n\n\t// Decode the JSON body\n\tvar body TextToImageResponse\n\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Write the images to disk\n\tfor i, image := range body.Images {\n\t\toutFile := fmt.Sprintf(\"./out/v1_txt2img_%d.png\", i)\n\t\tfile, err := os.Create(outFile)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\timageBytes, err := base64.StdEncoding.DecodeString(image.Base64)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tif _, err := file.Write(imageBytes); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tif err := file.Close(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n}\n" + }, + { + "lang": "cURL", + "source": "if [ -z \"$STABILITY*API_KEY\" ]; then\n echo \"STABILITY_API_KEY environment variable is not set\"\n exit 1\nfi\n\nOUTPUT_FILE=./out/v1_txt2img.png\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image\"\n\ncurl -f -sS -X POST \"$URL\" \\\n -H 'Content-Type: application/json' \\\n -H 'Accept: image/png' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\" \\\n --data-raw '{\n \"text_prompts\": [\n {\n \"text\": \"A lighthouse on a cliff\"\n }\n ],\n \"cfg_scale\": 7,\n \"height\": 1024,\n \"width\": 1024,\n \"samples\": 1,\n \"steps\": 30\n }' \\\n -o \"$OUTPUT_FILE\"\n" +} +] +} +}, +"/v1/generation/{engine_id}/image-to-image": { +"post": { +"description": "Modify an image based on a text prompt", +"operationId": "imageToImage", +"summary": "image-to-image", +"tags": [ +"v1/generation" +], +"parameters": [ +{ +"$ref": "#/components/parameters/engineID" +}, +{ +"$ref": "#/components/parameters/accept" +}, +{ +"$ref": "#/components/parameters/organization" +}, +{ +"$ref": "#/components/parameters/stabilityClientID" +}, +{ +"$ref": "#/components/parameters/stabilityClientVersion" +} +], +"requestBody": { +"content": { +"multipart/form-data": { +"schema": { +"$ref": "#/components/schemas/ImageToImageRequestBody" + }, + "examples": { + "IMAGE_STRENGTH": { + "summary": "Using IMAGE_STRENGTH", + "description": "Request using 35% image_strength", + "value": { + "image_strength": 0.35, + "init_image_mode": "IMAGE_STRENGTH", + "init_image": "", + "text_prompts[0][text]": "A dog space commander", + "text_prompts[0][weight]": 1, + "cfg_scale": 7, + "clip_guidance_preset": "FAST_BLUE", + "sampler": "K_DPM_2_ANCESTRAL", + "samples": 3, + "steps": 20 + } + }, + "STEP_SCHEDULE": { + "summary": "Using STEP_SCHEDULE", + "description": "Equivalent request using step_schedule_start", + "value": { + "step_schedule_start": 0.65, + "init_image_mode": "STEP_SCHEDULE", + "init_image": "", + "text_prompts[0][text]": "A dog space commander", + "text_prompts[0][weight]": 1, + "cfg_scale": 7, + "clip_guidance_preset": "FAST_BLUE", + "sampler": "K_DPM_2_ANCESTRAL", + "samples": 3, + "steps": 20 + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK response.", + "content": { + "application/json": { + "schema": { + "description": "The result of the generation request, containing one or more images as base64 encoded strings.", + "items": { + "$ref": "#/components/schemas/Image" +}, +"type": "array" +} +}, +"image/png": { +"example": "PNG bytes, what did you expect?", +"schema": { +"description": "The bytes of the generated image", +"format": "binary", +"type": "string" +} +} +}, +"headers": { +"Content-Length": { +"$ref": "#/components/headers/Content-Length" + }, + "Content-Type": { + "$ref": "#/components/headers/Content-Type" +}, +"Finish-Reason": { +"$ref": "#/components/headers/Finish-Reason" + }, + "Seed": { + "$ref": "#/components/headers/Seed" +} +} +}, +"400": { +"$ref": "#/components/responses/400FromGeneration" + }, + "401": { + "$ref": "#/components/responses/401" +}, +"403": { +"$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" +}, +"500": { +"$ref": "#/components/responses/500" +} +}, +"security": [ +{ +"STABILITY_API_KEY": [] +} +], +"x-codeSamples": [ +{ +"lang": "Python", +"source": "import base64\nimport os\nimport requests\n\nengine_id = \"stable-diffusion-xl-1024-v1-0\"\napi_host = os.getenv(\"API_HOST\", \"https://api.stability.ai\")\napi_key = os.getenv(\"STABILITY_API_KEY\")\n\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.post(\n f\"{api_host}/v1/generation/{engine_id}/image-to-image\",\n headers={\n \"Accept\": \"application/json\",\n \"Authorization\": f\"Bearer {api_key}\"\n },\n files={\n \"init_image\": open(\"../init_image_1024.png\", \"rb\")\n },\n data={\n \"image_strength\": 0.35,\n \"init_image_mode\": \"IMAGE_STRENGTH\",\n \"text_prompts[0][text]\": \"Galactic dog with a cape\",\n \"cfg_scale\": 7,\n \"samples\": 1,\n \"steps\": 30,\n }\n)\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\ndata = response.json()\n\nfor i, image in enumerate(data[\"artifacts\"]):\n with open(f\"./out/v1_img2img*{i}.png\", \"wb\") as f:\n f.write(base64.b64decode(image[\"base64\"]))\n" +}, +{ +"label": "TypeScript", +"lang": "Javascript", +"source": "import fetch from 'node-fetch'\nimport FormData from 'form-data'\nimport fs from 'node:fs'\n\nconst engineId = 'stable-diffusion-xl-1024-v1-0'\nconst apiHost = process.env.API*HOST ?? 'https://api.stability.ai'\nconst apiKey = process.env.STABILITY_API_KEY\n\nif (!apiKey) throw new Error('Missing Stability API key.')\n\n// NOTE: This example is using a NodeJS FormData library.\n// Browsers should use their native FormData class.\n// React Native apps should also use their native FormData class.\nconst formData = new FormData()\nformData.append('init_image', fs.readFileSync('../init_image_1024.png'))\nformData.append('init_image_mode', 'IMAGE_STRENGTH')\nformData.append('image_strength', 0.35)\nformData.append('text_prompts[0][text]', 'Galactic dog wearing a cape')\nformData.append('cfg_scale', 7)\nformData.append('samples', 1)\nformData.append('steps', 30)\n\nconst response = await fetch(\n `${apiHost}/v1/generation/${engineId}/image-to-image`,\n {\n method: 'POST',\n headers: {\n ...formData.getHeaders(),\n Accept: 'application/json',\n Authorization: `Bearer ${apiKey}`,\n },\n body: formData,\n }\n)\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\ninterface GenerationResponse {\n artifacts: Array<{\n base64: string\n seed: number\n finishReason: string\n }>\n}\n\nconst responseJSON = (await response.json()) as GenerationResponse\n\nresponseJSON.artifacts.forEach((image, index) => {\n fs.writeFileSync(\n `out/v1_img2img*${index}.png`,\n Buffer.from(image.base64, 'base64')\n )\n})\n" + }, + { + "lang": "Go", + "source": "package main\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"mime/multipart\"\n\t\"net/http\"\n\t\"os\"\n)\n\ntype ImageToImageImage struct {\n\tBase64 string `json:\"base64\"`\n\tSeed uint32 `json:\"seed\"`\n\tFinishReason string `json:\"finishReason\"`\n}\n\ntype ImageToImageResponse struct {\n\tImages []ImageToImageImage `json:\"artifacts\"`\n}\n\nfunc main() {\n\tengineId := \"stable-diffusion-xl-1024-v1-0\"\n\n\t// Build REST endpoint URL\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/generation/\" + engineId + \"/image-to-image\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\tdata := &bytes.Buffer{}\n\twriter := multipart.NewWriter(data)\n\n\t// Write the init image to the request\n\tinitImageWriter, _ := writer.CreateFormField(\"init_image\")\n\tinitImageFile, initImageErr := os.Open(\"../init_image_1024.png\")\n\tif initImageErr != nil {\n\t\tpanic(\"Could not open init_image.png\")\n\t}\n\t_, _ = io.Copy(initImageWriter, initImageFile)\n\n\t// Write the options to the request\n\t_ = writer.WriteField(\"init_image_mode\", \"IMAGE_STRENGTH\")\n\t_ = writer.WriteField(\"image_strength\", \"0.35\")\n\t_ = writer.WriteField(\"text_prompts[0][text]\", \"Galactic dog with a cape\")\n\t_ = writer.WriteField(\"cfg_scale\", \"7\")\n\t_ = writer.WriteField(\"samples\", \"1\")\n\t_ = writer.WriteField(\"steps\", \"30\")\n\twriter.Close()\n\n\t// Execute the request\n\tpayload := bytes.NewReader(data.Bytes())\n\treq, _ := http.NewRequest(\"POST\", reqUrl, payload)\n\treq.Header.Add(\"Content-Type\", writer.FormDataContentType())\n\treq.Header.Add(\"Accept\", \"application/json\")\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\n\tif res.StatusCode != 200 {\n\t\tvar body map[string]interface{}\n\t\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tpanic(fmt.Sprintf(\"Non-200 response: %s\", body))\n\t}\n\n\t// Decode the JSON body\n\tvar body ImageToImageResponse\n\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Write the images to disk\n\tfor i, image := range body.Images {\n\t\toutFile := fmt.Sprintf(\"./out/v1_img2img_%d.png\", i)\n\t\tfile, err := os.Create(outFile)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\timageBytes, err := base64.StdEncoding.DecodeString(image.Base64)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tif _, err := file.Write(imageBytes); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tif err := file.Close(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n}\n" + }, + { + "lang": "cURL", + "source": "if [ -z \"$STABILITY*API_KEY\" ]; then\n echo \"STABILITY_API_KEY environment variable is not set\"\n exit 1\nfi\n\nOUTPUT_FILE=./out/v1_img2img.png\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/generation/stable-diffusion-xl-1024-v1-0/image-to-image\"\n\ncurl -f -sS -X POST \"$URL\" \\\n -H 'Content-Type: multipart/form-data' \\\n -H 'Accept: image/png' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\" \\\n -F 'init_image=@\"../init_image_1024.png\"' \\\n -F 'init_image_mode=IMAGE_STRENGTH' \\\n -F 'image_strength=0.35' \\\n -F 'text_prompts[0][text]=A galactic dog in space' \\\n -F 'cfg_scale=7' \\\n -F 'samples=1' \\\n -F 'steps=30' \\\n -o \"$OUTPUT_FILE\"\n" +} +] +} +}, +"/v1/generation/{engine_id}/image-to-image/upscale": { +"post": { +"description": "Create a higher resolution version of an input image.\n\nThis operation outputs an image with a maximum pixel count of **4,194,304**. This is equivalent to dimensions such as `2048x2048` and `4096x1024`.\n\nBy default, the input image will be upscaled by a factor of 2. For additional control over the output dimensions, a `width` or `height` parameter may be specified.\n\nFor upscaler engines that are ESRGAN-based, refer to the `RealESRGANUpscaleRequestBody` body option below. For upscaler engines that are Stable Diffusion Latent Upscaler-based, refer to the `LatentUpscalerUpscaleRequestBody` body option below.\n\nFor more details on the upscaler engines, refer to the [documentation on the Platform site.](https://platform.stability.ai/docs/features/image-upscaling?tab=python)\n", +"operationId": "upscaleImage", +"summary": "image-to-image/upscale", +"tags": [ +"v1/generation" +], +"parameters": [ +{ +"$ref": "#/components/parameters/upscaleEngineID" +}, +{ +"$ref": "#/components/parameters/accept" +}, +{ +"$ref": "#/components/parameters/organization" +}, +{ +"$ref": "#/components/parameters/stabilityClientID" +}, +{ +"$ref": "#/components/parameters/stabilityClientVersion" +} +], +"requestBody": { +"content": { +"multipart/form-data": { +"schema": { +"oneOf": [ +{ +"$ref": "#/components/schemas/RealESRGANUpscaleRequestBody" +}, +{ +"$ref": "#/components/schemas/LatentUpscalerUpscaleRequestBody" +} +] +}, +"examples": { +"ESRGAN": { +"description": "Upscale input image by 2x using ESRGAN.", +"value": { +"image": "" +} +}, +"DESIRED_WIDTH": { +"description": "Upscale input image to desired width with ESRGAN or the Latent Upscaler.", +"value": { +"image": "", +"width": 1024 +} +}, +"DESIRED_HEIGHT": { +"description": "Upscale input image to desired height with ESRGAN or the Latent Upscaler.", +"value": { +"image": "", +"height": 1024 +} +}, +"LATENT_UPSCALER": { +"description": "Request using the Latent Upscaler. Refer to the LatentUpscalerUpscaleRequestBody for reference.", +"value": { +"seed": 5555, +"steps": 20, +"image": "", +"text_prompts[0][text]": "A dog space commander", +"text_prompts[0][weight]": 1, +"cfg_scale": 7 +} +} +} +} +}, +"required": true +}, +"responses": { +"200": { +"description": "OK response.", +"content": { +"application/json": { +"schema": { +"description": "The result of the generation request, containing one or more images as base64 encoded strings.", +"items": { +"$ref": "#/components/schemas/Image" + }, + "type": "array" + } + }, + "image/png": { + "example": "PNG bytes, what did you expect?", + "schema": { + "description": "The bytes of the generated image", + "format": "binary", + "type": "string" + } + } + }, + "headers": { + "Content-Length": { + "$ref": "#/components/headers/Content-Length" +}, +"Content-Type": { +"$ref": "#/components/headers/Content-Type" + }, + "Finish-Reason": { + "$ref": "#/components/headers/Finish-Reason" +}, +"Seed": { +"$ref": "#/components/headers/Seed" + } + } + }, + "400": { + "$ref": "#/components/responses/400FromUpscale" +}, +"401": { +"$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" +}, +"404": { +"$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" +} +}, +"security": [ +{ +"STABILITY_API_KEY": [] +} +], +"x-codeSamples": [ +{ +"lang": "Python", +"source": "import os\nimport requests\n\nengine_id = \"esrgan-v1-x2plus\"\napi_host = os.getenv(\"API_HOST\", \"https://api.stability.ai\")\napi_key = os.getenv(\"STABILITY_API_KEY\")\n\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.post(\n f\"{api_host}/v1/generation/{engine_id}/image-to-image/upscale\",\n headers={\n \"Accept\": \"image/png\",\n \"Authorization\": f\"Bearer {api_key}\"\n },\n files={\n \"image\": open(\"../init_image.png\", \"rb\")\n },\n data={\n \"width\": 1024,\n }\n)\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\nwith open(f\"./out/v1_upscaled_image.png\", \"wb\") as f:\n f.write(response.content)\n" +}, +{ +"label": "TypeScript", +"lang": "Javascript", +"source": "import fetch from 'node-fetch'\nimport FormData from 'form-data'\nimport fs from 'node:fs'\n\nconst engineId = 'esrgan-v1-x2plus'\nconst apiHost = process.env.API_HOST ?? 'https://api.stability.ai'\nconst apiKey = process.env.STABILITY_API_KEY\n\nif (!apiKey) throw new Error('Missing Stability API key.')\n\n// NOTE: This example is using a NodeJS FormData library.\n// Browsers should use their native FormData class.\n// React Native apps should also use their native FormData class.\nconst formData = new FormData()\nformData.append('image', fs.readFileSync('../init_image.png'))\nformData.append('width', 1024)\n\nconst response = await fetch(\n `${apiHost}/v1/generation/${engineId}/image-to-image/upscale`,\n {\n method: 'POST',\n headers: {\n ...formData.getHeaders(),\n Accept: 'image/png',\n Authorization: `Bearer ${apiKey}`,\n },\n body: formData,\n }\n)\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\nconst image = await response.arrayBuffer()\nfs.writeFileSync('./out/v1_upscaled_image.png', Buffer.from(image))\n" +}, +{ +"lang": "Go", +"source": "package main\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"mime/multipart\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\tengineId := \"esrgan-v1-x2plus\"\n\n\t// Build REST endpoint URL\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/generation/\" + engineId + \"/image-to-image/upscale\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\tdata := &bytes.Buffer{}\n\twriter := multipart.NewWriter(data)\n\n\t// Write the init image to the request\n\tinitImageWriter, * := writer.CreateFormField(\"image\")\n\tinitImageFile, initImageErr := os.Open(\"../init*image.png\")\n\tif initImageErr != nil {\n\t\tpanic(\"Could not open init_image.png\")\n\t}\n\t*, _ = io.Copy(initImageWriter, initImageFile)\n\n\t// Write the options to the request\n\t_ = writer.WriteField(\"width\", \"1024\")\n\twriter.Close()\n\n\t// Execute the request\n\tpayload := bytes.NewReader(data.Bytes())\n\treq, _ := http.NewRequest(\"POST\", reqUrl, payload)\n\treq.Header.Add(\"Content-Type\", writer.FormDataContentType())\n\treq.Header.Add(\"Accept\", \"image/png\")\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\n\tif res.StatusCode != 200 {\n\t\tvar body map[string]interface{}\n\t\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tpanic(fmt.Sprintf(\"Non-200 response: %s\", body))\n\t}\n\n\t// Write the response to a file\n\tout, err := os.Create(\"./out/v1*upscaled_image.png\")\n\tdefer out.Close()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t*, err = io.Copy(out, res.Body)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n" +}, +{ +"lang": "cURL", +"source": "if [ -z \"$STABILITY_API_KEY\" ]; then\n echo \"STABILITY*API_KEY environment variable is not set\"\n exit 1\nfi\n\nOUTPUT_FILE=./out/v1_upscaled_image.png\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/generation/esrgan-v1-x2plus/image-to-image/upscale\"\n\ncurl -f -sS -X POST \"$URL\" \\\n -H 'Content-Type: multipart/form-data' \\\n -H 'Accept: image/png' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\" \\\n -F 'image=@\"../init_image.png\"' \\\n -F 'width=1024' \\\n -o \"$OUTPUT_FILE\"\n" +} +] +} +}, +"/v1/generation/{engine_id}/image-to-image/masking": { +"post": { +"description": "Selectively modify portions of an image using a mask", +"operationId": "masking", +"summary": "image-to-image/masking", +"tags": [ +"v1/generation" +], +"parameters": [ +{ +"example": "stable-inpainting-512-v2-0", +"in": "path", +"name": "engine_id", +"required": true, +"schema": { +"type": "string" +} +}, +{ +"$ref": "#/components/parameters/accept" +}, +{ +"$ref": "#/components/parameters/organization" +}, +{ +"$ref": "#/components/parameters/stabilityClientID" +}, +{ +"$ref": "#/components/parameters/stabilityClientVersion" +} +], +"requestBody": { +"required": true, +"content": { +"multipart/form-data": { +"schema": { +"$ref": "#/components/schemas/MaskingRequestBody" + }, + "examples": { + "MASK_IMAGE_BLACK": { + "value": { + "mask_source": "MASK_IMAGE_BLACK", + "init_image": "", + "mask_image": "", + "text_prompts[0][text]": "A dog space commander", + "text_prompts[0][weight]": 1, + "cfg_scale": 7, + "clip_guidance_preset": "FAST_BLUE", + "sampler": "K_DPM_2_ANCESTRAL", + "samples": 3, + "steps": 20 + } + }, + "MASK_IMAGE_WHITE": { + "value": { + "mask_source": "MASK_IMAGE_WHITE", + "init_image": "", + "mask_image": "", + "text_prompts[0][text]": "A dog space commander", + "text_prompts[0][weight]": 1, + "cfg_scale": 7, + "clip_guidance_preset": "FAST_BLUE", + "sampler": "K_DPM_2_ANCESTRAL", + "samples": 3, + "steps": 20 + } + }, + "INIT_IMAGE_ALPHA": { + "value": { + "mask_source": "INIT_IMAGE_ALPHA", + "init_image": "", + "text_prompts[0][text]": "A dog space commander", + "text_prompts[0][weight]": 1, + "cfg_scale": 7, + "clip_guidance_preset": "FAST_BLUE", + "sampler": "K_DPM_2_ANCESTRAL", + "samples": 3, + "steps": 20 + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK response.", + "content": { + "application/json": { + "schema": { + "description": "The result of the generation request, containing one or more images as base64 encoded strings.", + "items": { + "$ref": "#/components/schemas/Image" +}, +"type": "array" +} +}, +"image/png": { +"example": "PNG bytes, what did you expect?", +"schema": { +"description": "The bytes of the generated image", +"format": "binary", +"type": "string" +} +} +}, +"headers": { +"Content-Length": { +"$ref": "#/components/headers/Content-Length" + }, + "Content-Type": { + "$ref": "#/components/headers/Content-Type" +}, +"Finish-Reason": { +"$ref": "#/components/headers/Finish-Reason" + }, + "Seed": { + "$ref": "#/components/headers/Seed" +} +} +}, +"400": { +"$ref": "#/components/responses/400FromGeneration" + }, + "401": { + "$ref": "#/components/responses/401" +}, +"403": { +"$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" +}, +"500": { +"$ref": "#/components/responses/500" +} +}, +"security": [ +{ +"STABILITY_API_KEY": [] +} +], +"x-codeSamples": [ +{ +"lang": "Python", +"source": "import base64\nimport os\nimport requests\n\nengine_id = \"stable-inpainting-512-v2-0\"\napi_host = os.getenv('API_HOST', 'https://api.stability.ai')\napi_key = os.getenv(\"STABILITY_API_KEY\")\n\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.post(\n f\"{api_host}/v1/generation/{engine_id}/image-to-image/masking\",\n headers={\n \"Accept\": 'application/json',\n \"Authorization\": f\"Bearer {api_key}\"\n },\n files={\n 'init_image': open(\"../init_image.png\", 'rb'),\n 'mask_image': open(\"../mask_image_white.png\", 'rb'),\n },\n data={\n \"mask_source\": \"MASK_IMAGE_WHITE\",\n \"text_prompts[0][text]\": \"A large spiral galaxy with a bright central bulge and a ring of stars around it\",\n \"cfg_scale\": 7,\n \"clip_guidance_preset\": \"FAST_BLUE\",\n \"samples\": 1,\n \"steps\": 30,\n }\n)\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\ndata = response.json()\n\nfor i, image in enumerate(data[\"artifacts\"]):\n with open(f\"./out/v1_img2img_masking*{i}.png\", \"wb\") as f:\n f.write(base64.b64decode(image[\"base64\"]))\n" +}, +{ +"label": "TypeScript", +"lang": "Javascript", +"source": "import fetch from 'node-fetch'\nimport FormData from 'form-data'\nimport fs from 'node:fs'\n\nconst engineId = 'stable-inpainting-512-v2-0'\nconst apiHost = process.env.API*HOST ?? 'https://api.stability.ai'\nconst apiKey = process.env.STABILITY_API_KEY\n\nif (!apiKey) throw new Error('Missing Stability API key.')\n\n// NOTE: This example is using a NodeJS FormData library. Browser\n// implementations should use their native FormData class. React Native\n// implementations should also use their native FormData class.\nconst formData = new FormData()\nformData.append('init_image', fs.readFileSync('../init_image.png'))\nformData.append('mask_image', fs.readFileSync('../mask_image_white.png'))\nformData.append('mask_source', 'MASK_IMAGE_WHITE')\nformData.append(\n 'text_prompts[0][text]',\n 'A large spiral galaxy with a bright central bulge and a ring of stars around it'\n)\nformData.append('cfg_scale', '7')\nformData.append('clip_guidance_preset', 'FAST_BLUE')\nformData.append('samples', 1)\nformData.append('steps', 30)\n\nconst response = await fetch(\n `${apiHost}/v1/generation/${engineId}/image-to-image/masking`,\n {\n method: 'POST',\n headers: {\n ...formData.getHeaders(),\n Accept: 'application/json',\n Authorization: `Bearer ${apiKey}`,\n },\n body: formData,\n }\n)\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\ninterface GenerationResponse {\n artifacts: Array<{\n base64: string\n seed: number\n finishReason: string\n }>\n}\n\nconst responseJSON = (await response.json()) as GenerationResponse\n\nresponseJSON.artifacts.forEach((image, index) => {\n fs.writeFileSync(\n `out/v1_img2img_masking*${index}.png`,\n Buffer.from(image.base64, 'base64')\n )\n})\n" + }, + { + "lang": "Go", + "source": "package main\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"mime/multipart\"\n\t\"net/http\"\n\t\"os\"\n)\n\ntype MaskingImage struct {\n\tBase64 string `json:\"base64\"`\n\tSeed uint32 `json:\"seed\"`\n\tFinishReason string `json:\"finishReason\"`\n}\n\ntype MaskingResponse struct {\n\tImages []MaskingImage `json:\"artifacts\"`\n}\n\nfunc main() {\n\tengineId := \"stable-inpainting-512-v2-0\"\n\n\t// Build REST endpoint URL\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/generation/\" + engineId + \"/image-to-image/masking\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\tdata := &bytes.Buffer{}\n\twriter := multipart.NewWriter(data)\n\n\t// Write the init image to the request\n\tinitImageWriter, _ := writer.CreateFormField(\"init_image\")\n\tinitImageFile, initImageErr := os.Open(\"../init_image.png\")\n\tif initImageErr != nil {\n\t\tpanic(\"Could not open init_image.png\")\n\t}\n\t_, _ = io.Copy(initImageWriter, initImageFile)\n\n\t// Write the mask image to the request\n\tmaskImageWriter, _ := writer.CreateFormField(\"mask_image\")\n\tmaskImageFile, maskImageErr := os.Open(\"../mask_image_white.png\")\n\tif maskImageErr != nil {\n\t\tpanic(\"Could not open mask_image_white.png\")\n\t}\n\t_, _ = io.Copy(maskImageWriter, maskImageFile)\n\n\t// Write the options to the request\n\t_ = writer.WriteField(\"mask_source\", \"MASK_IMAGE_WHITE\")\n\t_ = writer.WriteField(\"text_prompts[0][text]\", \"A large spiral galaxy with a bright central bulge and a ring of stars around it\")\n\t_ = writer.WriteField(\"cfg_scale\", \"7\")\n\t_ = writer.WriteField(\"clip_guidance_preset\", \"FAST_BLUE\")\n\t_ = writer.WriteField(\"samples\", \"1\")\n\t_ = writer.WriteField(\"steps\", \"30\")\n\twriter.Close()\n\n\t// Execute the request & read all the bytes of the response\n\tpayload := bytes.NewReader(data.Bytes())\n\treq, _ := http.NewRequest(\"POST\", reqUrl, payload)\n\treq.Header.Add(\"Content-Type\", writer.FormDataContentType())\n\treq.Header.Add(\"Accept\", \"application/json\")\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\n\tif res.StatusCode != 200 {\n\t\tvar body map[string]interface{}\n\t\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tpanic(fmt.Sprintf(\"Non-200 response: %s\", body))\n\t}\n\n\t// Decode the JSON body\n\tvar body MaskingResponse\n\tif err := json.NewDecoder(res.Body).Decode(&body); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Write the images to disk\n\tfor i, image := range body.Images {\n\t\toutFile := fmt.Sprintf(\"./out/v1_img2img_masking_%d.png\", i)\n\t\tfile, err := os.Create(outFile)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\timageBytes, err := base64.StdEncoding.DecodeString(image.Base64)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tif _, err := file.Write(imageBytes); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tif err := file.Close(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n}\n" + }, + { + "lang": "cURL", + "source": "#!/bin/sh\n\nset -e\n\nif [ -z \"$STABILITY*API_KEY\" ]; then\n echo \"STABILITY_API_KEY environment variable is not set\"\n exit 1\nfi\n\nOUTPUT_FILE=./out/v1_img2img_masking.png\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/generation/stable-inpainting-512-v2-0/image-to-image/masking\"\n\ncurl -f -sS -X POST \"$URL\" \\\n -H 'Content-Type: multipart/form-data' \\\n -H 'Accept: image/png' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\" \\\n -F 'init_image=@\"../init_image.png\"' \\\n -F 'mask_image=@\"../mask_image_white.png\"' \\\n -F 'mask_source=MASK_IMAGE_WHITE' \\\n -F 'text_prompts[0][text]=A large spiral galaxy with a bright central bulge and a ring of stars around it' \\\n -F 'cfg_scale=7' \\\n -F 'clip_guidance_preset=FAST_BLUE' \\\n -F 'samples=1' \\\n -F 'steps=30' \\\n -o \"$OUTPUT_FILE\"\n" +} +] +} +}, +"/v1/engines/list": { +"get": { +"description": "List all engines available to your organization/user", +"operationId": "listEngines", +"summary": "list", +"tags": [ +"v1/engines" +], +"parameters": [ +{ +"$ref": "#/components/parameters/organization" +}, +{ +"$ref": "#/components/parameters/stabilityClientID" +}, +{ +"$ref": "#/components/parameters/stabilityClientVersion" +} +], +"responses": { +"200": { +"content": { +"application/json": { +"schema": { +"$ref": "#/components/schemas/ListEnginesResponseBody" + } + } + }, + "description": "OK response." + }, + "401": { + "$ref": "#/components/responses/401" +}, +"500": { +"$ref": "#/components/responses/500" + } + }, + "security": [ + { + "STABILITY_API_KEY": [] + } + ], + "x-codeSamples": [ + { + "lang": "Python", + "source": "import os\nimport requests\n\napi_host = os.getenv('API_HOST', 'https://api.stability.ai')\nurl = f\"{api_host}/v1/engines/list\"\n\napi_key = os.getenv(\"STABILITY_API_KEY\")\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.get(url, headers={\n \"Authorization\": f\"Bearer {api_key}\"\n})\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\n# Do something with the payload...\npayload = response.json()\n\n" + }, + { + "label": "TypeScript", + "lang": "Javascript", + "source": "import fetch from 'node-fetch'\n\nconst apiHost = process.env.API_HOST ?? 'https://api.stability.ai'\nconst url = `${apiHost}/v1/engines/list`\n\nconst apiKey = process.env.STABILITY_API_KEY\nif (!apiKey) throw new Error('Missing Stability API key.')\n\nconst response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${apiKey}`,\n },\n})\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\ninterface Payload {\n engines: Array<{\n id: string\n name: string\n description: string\n type: string\n }>\n}\n\n// Do something with the payload...\nconst payload = (await response.json()) as Payload\n" +}, +{ +"lang": "Go", +"source": "package main\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\t// Build REST endpoint URL\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/engines/list\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\t// Execute the request & read all the bytes of the response\n\treq, * := http.NewRequest(\"GET\", reqUrl, nil)\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\tbody, _ := io.ReadAll(res.Body)\n\n\tif res.StatusCode != 200 {\n\t\tpanic(\"Non-200 response: \" + string(body))\n\t}\n\n\t// Do something with the payload...\n\t// payload := string(body)\n}\n" +}, +{ +"lang": "cURL", +"source": "if [ -z \"$STABILITY_API_KEY\" ]; then\n echo \"STABILITY*API_KEY environment variable is not set\"\n exit 1\nfi\n\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/engines/list\"\n\ncurl -f -sS \"$URL\" \\\n -H 'Accept: application/json' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\"\n" + } + ] + } + }, + "/v1/user/account": { + "get": { + "description": "Get information about the account associated with the provided API key", + "operationId": "userAccount", + "summary": "account", + "tags": [ + "v1/user" + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccountResponseBody" +} +} +}, +"description": "OK response." +}, +"401": { +"$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" +} +}, +"security": [ +{ +"STABILITY_API_KEY": [] +} +], +"x-codeSamples": [ +{ +"lang": "Python", +"source": "import os\nimport requests\n\napi_host = os.getenv('API_HOST', 'https://api.stability.ai')\nurl = f\"{api_host}/v1/user/account\"\n\napi_key = os.getenv(\"STABILITY_API_KEY\")\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.get(url, headers={\n \"Authorization\": f\"Bearer {api_key}\"\n})\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\n# Do something with the payload...\npayload = response.json()\n\n" +}, +{ +"label": "TypeScript", +"lang": "Javascript", +"source": "import fetch from 'node-fetch'\n\nconst apiHost = process.env.API_HOST ?? 'https://api.stability.ai'\nconst url = `${apiHost}/v1/user/account`\n\nconst apiKey = process.env.STABILITY_API_KEY\nif (!apiKey) throw new Error('Missing Stability API key.')\n\nconst response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${apiKey}`,\n },\n})\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\ninterface User {\n id: string\n profile_picture: string\n email: string\n organizations?: Array<{\n id: string\n name: string\n role: string\n is_default: boolean\n }>\n}\n\n// Do something with the user...\nconst user = (await response.json()) as User\n" +}, +{ +"lang": "Go", +"source": "package main\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\t// Build REST endpoint URL\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/user/account\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\t// Build the request\n\treq, * := http.NewRequest(\"GET\", reqUrl, nil)\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\n\t// Execute the request\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\tbody, _ := io.ReadAll(res.Body)\n\n\tif res.StatusCode != 200 {\n\t\tpanic(\"Non-200 response: \" + string(body))\n\t}\n\n\t// Do something with the payload...\n\t// payload := string(body)\n}\n" +}, +{ +"lang": "cURL", +"source": "if [ -z \"$STABILITY_API_KEY\" ]; then\n echo \"STABILITY*API_KEY environment variable is not set\"\n exit 1\nfi\n\n# Determine the URL to use for the request\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/user/account\"\n\ncurl -f -sS \"$URL\" \\\n -H 'Accept: application/json' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\"\n" + } + ] + } + }, + "/v1/user/balance": { + "get": { + "description": "Get the credit balance of the account/organization associated with the API key", + "operationId": "userBalance", + "summary": "balance", + "tags": [ + "v1/user" + ], + "parameters": [ + { + "$ref": "#/components/parameters/organization" +}, +{ +"$ref": "#/components/parameters/stabilityClientID" + }, + { + "$ref": "#/components/parameters/stabilityClientVersion" +} +], +"responses": { +"200": { +"content": { +"application/json": { +"example": { +"credits": 0.6336833840314097 +}, +"schema": { +"$ref": "#/components/schemas/BalanceResponseBody" + } + } + }, + "description": "OK response." + }, + "401": { + "$ref": "#/components/responses/401" +}, +"500": { +"$ref": "#/components/responses/500" + } + }, + "security": [ + { + "STABILITY_API_KEY": [] + } + ], + "x-codeSamples": [ + { + "lang": "Python", + "source": "import os\nimport requests\n\napi_host = os.getenv('API_HOST', 'https://api.stability.ai')\nurl = f\"{api_host}/v1/user/balance\"\n\napi_key = os.getenv(\"STABILITY_API_KEY\")\nif api_key is None:\n raise Exception(\"Missing Stability API key.\")\n\nresponse = requests.get(url, headers={\n \"Authorization\": f\"Bearer {api_key}\"\n})\n\nif response.status_code != 200:\n raise Exception(\"Non-200 response: \" + str(response.text))\n\n# Do something with the payload...\npayload = response.json()\n\n" + }, + { + "label": "TypeScript", + "lang": "Javascript", + "source": "import fetch from 'node-fetch'\n\nconst apiHost = process.env.API_HOST ?? 'https://api.stability.ai'\nconst url = `${apiHost}/v1/user/balance`\n\nconst apiKey = process.env.STABILITY_API_KEY\nif (!apiKey) throw new Error('Missing Stability API key.')\n\nconst response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${apiKey}`,\n },\n})\n\nif (!response.ok) {\n throw new Error(`Non-200 response: ${await response.text()}`)\n}\n\ninterface Balance {\n credits: number\n}\n\n// Do something with the balance...\nconst balance = (await response.json()) as Balance\n" +}, +{ +"lang": "Go", +"source": "package main\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\t// Build REST endpoint URL\n\tapiHost, hasApiHost := os.LookupEnv(\"API_HOST\")\n\tif !hasApiHost {\n\t\tapiHost = \"https://api.stability.ai\"\n\t}\n\treqUrl := apiHost + \"/v1/user/balance\"\n\n\t// Acquire an API key from the environment\n\tapiKey, hasAPIKey := os.LookupEnv(\"STABILITY_API_KEY\")\n\tif !hasAPIKey {\n\t\tpanic(\"Missing STABILITY_API_KEY environment variable\")\n\t}\n\n\t// Build the request\n\treq, * := http.NewRequest(\"GET\", reqUrl, nil)\n\treq.Header.Add(\"Authorization\", \"Bearer \"+apiKey)\n\n\t// Execute the request\n\tres, _ := http.DefaultClient.Do(req)\n\tdefer res.Body.Close()\n\tbody, _ := io.ReadAll(res.Body)\n\n\tif res.StatusCode != 200 {\n\t\tpanic(\"Non-200 response: \" + string(body))\n\t}\n\n\t// Do something with the payload...\n\t// payload := string(body)\n}\n" +}, +{ +"lang": "cURL", +"source": "if [ -z \"$STABILITY_API_KEY\" ]; then\n echo \"STABILITY*API_KEY environment variable is not set\"\n exit 1\nfi\n\n# Determine the URL to use for the request\nBASE_URL=${API_HOST:-https://api.stability.ai}\nURL=\"$BASE_URL/v1/user/balance\"\n\ncurl -f -sS \"$URL\" \\\n -H 'Content-Type: application/json' \\\n -H \"Authorization: Bearer $STABILITY_API_KEY\"\n" +} +] +} +} +}, +"components": { +"schemas": { +"Engine": { +"type": "object", +"properties": { +"description": { +"type": "string" +}, +"id": { +"type": "string", +"x-go-name": "ID", +"description": "Unique identifier for the engine", +"example": "stable-diffusion-v1-5" +}, +"name": { +"type": "string", +"description": "Name of the engine", +"example": "Stable Diffusion v1.5" +}, +"type": { +"type": "string", +"description": "The type of content this engine produces", +"example": "PICTURE", +"enum": [ +"AUDIO", +"CLASSIFICATION", +"PICTURE", +"STORAGE", +"TEXT", +"VIDEO" +] +} +}, +"required": [ +"id", +"name", +"description", +"type" +] +}, +"Error": { +"type": "object", +"x-go-name": "RESTError", +"properties": { +"id": { +"x-go-name": "ID", +"type": "string", +"description": "A unique identifier for this particular occurrence of the problem.", +"example": "296a972f-666a-44a1-a3df-c9c28a1f56c0" +}, +"name": { +"type": "string", +"description": "The short-name of this class of errors e.g. `bad_request`.", +"example": "bad_request" +}, +"message": { +"type": "string", +"description": "A human-readable explanation specific to this occurrence of the problem.", +"example": "Header parameter Authorization is required, but not found" +} +}, +"required": [ +"name", +"id", +"message", +"status" +] +}, +"CfgScale": { +"type": "number", +"description": "How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)", +"default": 7, +"example": 7, +"minimum": 0, +"maximum": 35 +}, +"ClipGuidancePreset": { +"type": "string", +"default": "NONE", +"example": "FAST_BLUE", +"enum": [ +"FAST_BLUE", +"FAST_GREEN", +"NONE", +"SIMPLE", +"SLOW", +"SLOWER", +"SLOWEST" +] +}, +"UpscaleImageHeight": { +"x-go-type": "uint64", +"type": "integer", +"description": "Desired height of the output image. Only one of `width` or `height` may be specified.", +"minimum": 512 +}, +"UpscaleImageWidth": { +"x-go-type": "uint64", +"type": "integer", +"description": "Desired width of the output image. Only one of `width` or `height` may be specified.", +"minimum": 512 +}, +"DiffuseImageHeight": { +"x-go-type": "uint64", +"type": "integer", +"description": "Height of the image in pixels. Must be in increments of 64 and pass the following validation:\n- For 512 engines: 262,144 ≤ `height * width`≤ 1,048,576\n- For 768 engines: 589,824 ≤`height _ width`≤ 1,048,576\n- For SDXL Beta: can be as low as 128 and as high as 896 as long as`width`is not greater than 512. If`width` is greater than 512 then this can be \_at most_ 512.\n- For SDXL v0.9: valid dimensions are 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152\n- For SDXL v1.0: valid dimensions are the same as SDXL v0.9", +"multipleOf": 64, +"default": 512, +"example": 512, +"minimum": 128 +}, +"DiffuseImageWidth": { +"x-go-type": "uint64", +"type": "integer", +"description": "Width of the image in pixels. Must be in increments of 64 and pass the following validation:\n- For 512 engines: 262,144 ≤ `height * width` ≤ 1,048,576\n- For 768 engines: 589,824 ≤ `height * width` ≤ 1,048,576\n- For SDXL Beta: can be as low as 128 and as high as 896 as long as `height` is not greater than 512. If `height` is greater than 512 then this can be _at most_ 512.\n- For SDXL v0.9: valid dimensions are 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152\n- For SDXL v1.0: valid dimensions are the same as SDXL v0.9", +"multipleOf": 64, +"default": 512, +"example": 512, +"minimum": 128 +}, +"Sampler": { +"type": "string", +"description": "Which sampler to use for the diffusion process. If this value is omitted we'll automatically select an appropriate sampler for you.", +"example": "K*DPM_2_ANCESTRAL", +"enum": [ +"DDIM", +"DDPM", +"K_DPMPP_2M", +"K_DPMPP_2S_ANCESTRAL", +"K_DPM_2", +"K_DPM_2_ANCESTRAL", +"K_EULER", +"K_EULER_ANCESTRAL", +"K_HEUN", +"K_LMS" +] +}, +"Samples": { +"x-go-type": "uint64", +"type": "integer", +"description": "Number of images to generate", +"default": 1, +"example": 1, +"minimum": 1, +"maximum": 10 +}, +"Seed": { +"type": "integer", +"x-go-type": "uint32", +"description": "Random noise seed (omit this option or use `0` for a random seed)", +"default": 0, +"example": 0, +"minimum": 0, +"maximum": 4294967295 +}, +"Steps": { +"x-go-type": "uint64", +"type": "integer", +"description": "Number of diffusion steps to run", +"default": 50, +"example": 75, +"minimum": 10, +"maximum": 150 +}, +"Extras": { +"type": "object", +"description": "Extra parameters passed to the engine.\nThese parameters are used for in-development or experimental features and may change\nwithout warning, so please use with caution." +}, +"StylePreset": { +"type": "string", +"enum": [ +"enhance", +"anime", +"photographic", +"digital-art", +"comic-book", +"fantasy-art", +"line-art", +"analog-film", +"neon-punk", +"isometric", +"low-poly", +"origami", +"modeling-compound", +"cinematic", +"3d-model", +"pixel-art", +"tile-texture" +], +"description": "Pass in a style preset to guide the image model towards a particular style.\nThis list of style presets is subject to change." +}, +"TextPrompt": { +"type": "object", +"properties": { +"text": { +"type": "string", +"description": "The prompt itself", +"example": "A lighthouse on a cliff", +"maxLength": 2000 +}, +"weight": { +"type": "number", +"description": "Weight of the prompt (use negative numbers for negative prompts)", +"example": 0.8167237, +"format": "float" +} +}, +"description": "Text prompt for image generation", +"required": [ +"text" +] +}, +"TextPromptsForTextToImage": { +"title": "TextPrompts", +"type": "array", +"items": { +"$ref": "#/components/schemas/TextPrompt" + }, + "minItems": 1, + "description": "An array of text prompts to use for generation.\n\nGiven a text prompt with the text `A lighthouse on a cliff` and a weight of `0.5`, it would be represented as:\n\n```\n\"text_prompts\": [\n {\n \"text\": \"A lighthouse on a cliff\",\n \"weight\": 0.5\n }\n]\n```" + }, + "TextPrompts": { + "description": "An array of text prompts to use for generation.\n\nDue to how arrays are represented in `multipart/form-data` requests, prompts must adhere to the format `text_prompts[index][text|weight]`,\nwhere `index` is some integer used to tie the text and weight together. While `index` does not have to be sequential, duplicate entries \nwill override previous entries, so it is recommended to use sequential indices.\n\nGiven a text prompt with the text `A lighthouse on a cliff` and a weight of `0.5`, it would be represented as:\n```\ntext_prompts[0][text]: \"A lighthouse on a cliff\"\ntext_prompts[0][weight]: 0.5\n```\n\nTo add another prompt to that request simply provide the values under a new `index`:\n\n```\ntext_prompts[0][text]: \"A lighthouse on a cliff\"\ntext_prompts[0][weight]: 0.5\ntext_prompts[1][text]: \"land, ground, dirt, grass\"\ntext_prompts[1][weight]: -0.9\n```", + "type": "array", + "items": { + "$ref": "#/components/schemas/TextPrompt" +}, +"minItems": 1 +}, +"InputImage": { +"x-go-type": "[]byte", +"type": "string", +"example": "", +"format": "binary" +}, +"InitImage": { +"x-go-type": "[]byte", +"type": "string", +"description": "Image used to initialize the diffusion process, in lieu of random noise.", +"example": "", +"format": "binary" +}, +"InitImageStrength": { +"type": "number", +"description": "How much influence the `init_image` has on the diffusion process. Values close to `1` will yield images very similar to the `init_image` while values close to `0` will yield images wildly different than the `init_image`. The behavior of this is meant to mirror DreamStudio's \"Image Strength\" slider.

This parameter is just an alternate way to set `step_schedule_start`, which is done via the calculation `1 - image_strength`. For example, passing in an Image Strength of 35% (`0.35`) would result in a `step_schedule_start` of `0.65`.\n", +"example": 0.4, +"minimum": 0, +"maximum": 1, +"format": "float", +"default": 0.35 +}, +"InitImageMode": { +"type": "string", +"description": "Whether to use `image_strength` or `step_schedule*_`to control how much influence the`init_image`has on the result.", + "enum": [ + "IMAGE_STRENGTH", + "STEP_SCHEDULE" + ], + "default": "IMAGE_STRENGTH" + }, + "StepScheduleStart": { + "type": "number", + "description": "Skips a proportion of the start of the diffusion steps, allowing the init_image to influence the final generated image. Lower values will result in more influence from the init_image, while higher values will result in more influence from the diffusion steps. (e.g. a value of`0`would simply return you the init_image, where a value of`1`would return you a completely different image.)", + "default": 0.65, + "example": 0.4, + "minimum": 0, + "maximum": 1 + }, + "StepScheduleEnd": { + "type": "number", + "description": "Skips a proportion of the end of the diffusion steps, allowing the init_image to influence the final generated image. Lower values will result in more influence from the init_image, while higher values will result in more influence from the diffusion steps.", + "example": 0.01, + "minimum": 0, + "maximum": 1 + }, + "MaskImage": { + "x-go-type": "[]byte", + "type": "string", + "description": "Optional grayscale mask that allows for influence over which pixels are eligible for diffusion and at what strength. Must be the same dimensions as the`init_image`. Use the `mask_source`option to specify whether the white or black pixels should be inpainted.", + "example": "", + "format": "binary" + }, + "MaskSource": { + "type": "string", + "description": "For any given pixel, the mask determines the strength of generation on a linear scale. This parameter determines where to source the mask from:\n-`MASK_IMAGE_WHITE`will use the white pixels of the mask_image as the mask, where white pixels are completely replaced and black pixels are unchanged\n-`MASK_IMAGE_BLACK`will use the black pixels of the mask_image as the mask, where black pixels are completely replaced and white pixels are unchanged\n-`INIT_IMAGE_ALPHA`will use the alpha channel of the init_image as the mask, where fully transparent pixels are completely replaced and fully opaque pixels are unchanged" + }, + "GenerationRequestOptionalParams": { + "type": "object", + "description": "Represents the optional parameters that can be passed to any generation request.", + "properties": { + "cfg_scale": { + "$ref": "#/components/schemas/CfgScale" + }, + "clip_guidance_preset": { + "$ref": "#/components/schemas/ClipGuidancePreset" + }, + "sampler": { + "$ref": "#/components/schemas/Sampler" + }, + "samples": { + "$ref": "#/components/schemas/Samples" + }, + "seed": { + "$ref": "#/components/schemas/Seed" + }, + "steps": { + "$ref": "#/components/schemas/Steps" + }, + "style_preset": { + "$ref": "#/components/schemas/StylePreset" + }, + "extras": { + "$ref": "#/components/schemas/Extras" + } + } + }, + "RealESRGANUpscaleRequestBody": { + "type": "object", + "properties": { + "image": { + "$ref": "#/components/schemas/InputImage" + }, + "width": { + "$ref": "#/components/schemas/UpscaleImageWidth" + }, + "height": { + "$ref": "#/components/schemas/UpscaleImageHeight" + } + }, + "required": [ + "image" + ] + }, + "LatentUpscalerUpscaleRequestBody": { + "type": "object", + "properties": { + "image": { + "$ref": "#/components/schemas/InputImage" + }, + "width": { + "$ref": "#/components/schemas/UpscaleImageWidth" + }, + "height": { + "$ref": "#/components/schemas/UpscaleImageHeight" + }, + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "seed": { + "$ref": "#/components/schemas/Seed" + }, + "steps": { + "$ref": "#/components/schemas/Steps" + }, + "cfg_scale": { + "$ref": "#/components/schemas/CfgScale" + } + }, + "required": [ + "image" + ] + }, + "ImageToImageRequestBody": { + "type": "object", + "properties": { + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "init_image": { + "$ref": "#/components/schemas/InitImage" + }, + "init_image_mode": { + "$ref": "#/components/schemas/InitImageMode" + }, + "image_strength": { + "$ref": "#/components/schemas/InitImageStrength" + }, + "step_schedule_start": { + "$ref": "#/components/schemas/StepScheduleStart" + }, + "step_schedule_end": { + "$ref": "#/components/schemas/StepScheduleEnd" + }, + "cfg_scale": { + "$ref": "#/components/schemas/CfgScale" + }, + "clip_guidance_preset": { + "$ref": "#/components/schemas/ClipGuidancePreset" + }, + "sampler": { + "$ref": "#/components/schemas/Sampler" + }, + "samples": { + "$ref": "#/components/schemas/Samples" + }, + "seed": { + "$ref": "#/components/schemas/Seed" + }, + "steps": { + "$ref": "#/components/schemas/Steps" + }, + "style_preset": { + "$ref": "#/components/schemas/StylePreset" + }, + "extras": { + "$ref": "#/components/schemas/Extras" + } + }, + "required": [ + "text_prompts", + "init_image" + ], + "discriminator": { + "propertyName": "init_image_mode", + "mapping": { + "IMAGE_STRENGTH": "#/components/schemas/ImageToImageUsingImageStrengthRequestBody", + "STEP_SCHEDULE": "#/components/schemas/ImageToImageUsingStepScheduleRequestBody" + } + } + }, + "ImageToImageUsingImageStrengthRequestBody": { + "allOf": [ + { + "type": "object", + "properties": { + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "init_image": { + "$ref": "#/components/schemas/InitImage" + }, + "init_image_mode": { + "$ref": "#/components/schemas/InitImageMode" + }, + "image_strength": { + "$ref": "#/components/schemas/InitImageStrength" + } + }, + "required": [ + "text_prompts", + "init_image" + ] + }, + { + "$ref": "#/components/schemas/GenerationRequestOptionalParams" + } + ] + }, + "ImageToImageUsingStepScheduleRequestBody": { + "allOf": [ + { + "type": "object", + "properties": { + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "init_image": { + "$ref": "#/components/schemas/InitImage" + }, + "init_image_mode": { + "$ref": "#/components/schemas/InitImageMode" + }, + "step_schedule_start": { + "$ref": "#/components/schemas/StepScheduleStart" + }, + "step_schedule_end": { + "$ref": "#/components/schemas/StepScheduleEnd" + } + }, + "required": [ + "text_prompts", + "init_image" + ] + }, + { + "$ref": "#/components/schemas/GenerationRequestOptionalParams" + } + ] + }, + "MaskingRequestBody": { + "type": "object", + "properties": { + "init_image": { + "$ref": "#/components/schemas/InitImage" + }, + "mask_source": { + "$ref": "#/components/schemas/MaskSource" + }, + "mask_image": { + "$ref": "#/components/schemas/MaskImage" + }, + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "cfg_scale": { + "$ref": "#/components/schemas/CfgScale" + }, + "clip_guidance_preset": { + "$ref": "#/components/schemas/ClipGuidancePreset" + }, + "sampler": { + "$ref": "#/components/schemas/Sampler" + }, + "samples": { + "$ref": "#/components/schemas/Samples" + }, + "seed": { + "$ref": "#/components/schemas/Seed" + }, + "steps": { + "$ref": "#/components/schemas/Steps" + }, + "style_preset": { + "$ref": "#/components/schemas/StylePreset" + }, + "extras": { + "$ref": "#/components/schemas/Extras" + } + }, + "required": [ + "text_prompts", + "init_image", + "mask_source" + ], + "discriminator": { + "propertyName": "mask_source", + "mapping": { + "MASK_IMAGE_BLACK": "#/components/schemas/MaskingUsingMaskImageRequestBody", + "MASK_IMAGE_WHITE": "#/components/schemas/MaskingUsingMaskImageRequestBody", + "INIT_IMAGE_ALPHA": "#/components/schemas/MaskingUsingInitImageAlphaRequestBody" + } + } + }, + "MaskingUsingMaskImageRequestBody": { + "allOf": [ + { + "type": "object", + "properties": { + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "init_image": { + "$ref": "#/components/schemas/InitImage" + }, + "mask_source": { + "$ref": "#/components/schemas/MaskSource" + }, + "mask_image": { + "$ref": "#/components/schemas/MaskImage" + } + }, + "required": [ + "init_image", + "mask_image", + "text_prompts", + "mask_source" + ] + }, + { + "$ref": "#/components/schemas/GenerationRequestOptionalParams" + } + ] + }, + "MaskingUsingInitImageAlphaRequestBody": { + "allOf": [ + { + "type": "object", + "properties": { + "text_prompts": { + "$ref": "#/components/schemas/TextPrompts" + }, + "init_image": { + "$ref": "#/components/schemas/InitImage" + }, + "mask_source": { + "$ref": "#/components/schemas/MaskSource" + } + }, + "required": [ + "init_image", + "text_prompts", + "mask_source" + ] + }, + { + "$ref": "#/components/schemas/GenerationRequestOptionalParams" + } + ] + }, + "TextToImageRequestBody": { + "type": "object", + "allOf": [ + { + "type": "object", + "properties": { + "height": { + "$ref": "#/components/schemas/DiffuseImageHeight" + }, + "width": { + "$ref": "#/components/schemas/DiffuseImageWidth" + }, + "text_prompts": { + "$ref": "#/components/schemas/TextPromptsForTextToImage" + } + }, + "required": [ + "text_prompts" + ] + }, + { + "$ref": "#/components/schemas/GenerationRequestOptionalParams" + } + ], + "example": { + "cfg_scale": 7, + "clip_guidance_preset": "FAST_BLUE", + "height": 512, + "sampler": "K_DPM_2_ANCESTRAL", + "samples": 1, + "seed": 0, + "steps": 75, + "text_prompts": [ + { + "text": "A lighthouse on a cliff", + "weight": 1 + } + ], + "width": 512 + }, + "required": [ + "text_prompts" + ] + }, + "AccountResponseBody": { + "type": "object", + "properties": { + "email": { + "type": "string", + "description": "The user's email", + "example": "example@stability.ai", + "format": "email" + }, + "id": { + "type": "string", + "description": "The user's ID", + "example": "user-1234", + "x-go-name": "ID" + }, + "organizations": { + "type": "array", + "example": [ + { + "id": "org-5678", + "name": "Another Organization", + "role": "MEMBER", + "is_default": true + }, + { + "id": "org-1234", + "name": "My Organization", + "role": "MEMBER", + "is_default": false + } + ], + "items": { + "$ref": "#/components/schemas/OrganizationMembership" + }, + "description": "The user's organizations" + }, + "profile_picture": { + "type": "string", + "description": "The user's profile picture", + "example": "https://api.stability.ai/example.png", + "format": "uri" + } + }, + "required": [ + "id", + "email", + "organizations" + ] + }, + "BalanceResponseBody": { + "type": "object", + "properties": { + "credits": { + "type": "number", + "description": "The balance of the account/organization associated with the API key", + "example": 0.41122252265928866, + "format": "double" + } + }, + "example": { + "credits": 0.07903292496944721 + }, + "required": [ + "credits" + ] + }, + "ListEnginesResponseBody": { + "type": "array", + "description": "The engines available to your user/organization", + "items": { + "$ref": "#/components/schemas/Engine" + }, + "example": [ + { + "description": "Stability-AI Stable Diffusion XL v1.0", + "id": "stable-diffusion-xl-1024-v1-0", + "name": "Stable Diffusion XL v1.0", + "type": "PICTURE" + }, + { + "description": "Stability-AI Stable Diffusion XL v0.9", + "id": "stable-diffusion-xl-1024-v0-9", + "name": "Stable Diffusion XL v0.9", + "type": "PICTURE" + }, + { + "description": "Stability-AI Stable Diffusion XL Beta v2.2.2", + "id": "stable-diffusion-xl-beta-v2-2-2", + "name": "Stable Diffusion v2.2.2-XL Beta", + "type": "PICTURE" + }, + { + "description": "Stability-AI Stable Diffusion v2.1", + "id": "stable-diffusion-512-v2-1", + "name": "Stable Diffusion v2.1", + "type": "PICTURE" + }, + { + "description": "Stability-AI Stable Diffusion 768 v2.1", + "id": "stable-diffusion-768-v2-1", + "name": "Stable Diffusion v2.1-768", + "type": "PICTURE" + }, + { + "description": "Stability-AI Stable Diffusion v1.5", + "id": "stable-diffusion-v1-5", + "name": "Stable Diffusion v1.5", + "type": "PICTURE" + } + ] + }, + "FinishReason": { + "type": "string", + "description": "The result of the generation process.\n-`SUCCESS`indicates success\n-`ERROR`indicates an error\n-`CONTENT_FILTERED`indicates the result affected by the content filter and may be blurred.\n\nThis header is only present when the`Accept`is set to`image/png`. Otherwise it is returned in the response body.", + "enum": [ + "SUCCESS", + "ERROR", + "CONTENT_FILTERED" + ] + }, + "Image": { + "type": "object", + "properties": { + "base64": { + "type": "string", + "description": "Image encoded in base64", + "example": "Sed corporis modi et." + }, + "finishReason": { + "type": "string", + "example": "CONTENT_FILTERED", + "enum": [ + "SUCCESS", + "ERROR", + "CONTENT_FILTERED" + ] + }, + "seed": { + "type": "number", + "description": "The seed associated with this image", + "example": 1229191277 + } + }, + "example": [ + { + "base64": "...very long string...", + "finishReason": "SUCCESS", + "seed": 1050625087 + }, + { + "base64": "...very long string...", + "finishReason": "CONTENT_FILTERED", + "seed": 1229191277 + } + ] + }, + "OrganizationMembership": { + "type": "object", + "properties": { + "id": { + "type": "string", + "example": "org-123456", + "x-go-name": "ID" + }, + "is_default": { + "type": "boolean", + "example": false + }, + "name": { + "type": "string", + "example": "My Organization" + }, + "role": { + "type": "string", + "example": "MEMBER" + } + }, + "required": [ + "id", + "name", + "role", + "is_default" + ] + } + }, + "responses": { + "401": { + "description": "unauthorized: API key missing or invalid", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "id": "9160aa70-222f-4a36-9eb7-475e2668362a", + "name": "unauthorized", + "message": "missing authorization header" + } + } + } + }, + "403": { + "description": "permission_denied: You lack the necessary permissions to perform this action", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "id": "5cf19777-d17f-49fe-9bd9-39ff0ec6bb50", + "name": "permission_denied", + "message": "You do not have permission to access this resource" + } + } + } + }, + "404": { + "description": "not_found: The requested resource was not found (e.g. specifing a model that does not exist)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "id": "92b19e7f-22a2-4e71-a821-90edda229293", + "name": "not_found", + "message": "The specified engine (ID some-fake-engine) was not found." + } + } + } + }, + "500": { + "description": "server_error: Some unexpected server error occurred", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "id": "f81964d6-619b-453e-97bc-9fd7ac3f04e7", + "name": "server_error", + "message": "An unexpected server error occurred, please try again." + } + } + } + }, + "400FromGeneration": { + "description": "General error for invalid parameters, see below for specific errors.\n - bad_request: one or more provided parameters are invalid (see error description for details)\n - invalid_samples: Sample count may only be greater than 1 when the accept header is set to `application/json`\n - invalid_height_or_width: Height and width must be specified in increments of 64\n - invalid_file_size: The file size of one or more of the provided files is invalid\n - invalid_mime_type: The mime type of one or more of the provided files is invalid\n - invalid_image_dimensions: The dimensions of the provided `init_image`and`mask_image`do not match\n - invalid_mask_image: The parameter`mask_source`was set to`MASK_IMAGE_WHITE`or`MASK_IMAGE_BLACK`but no`mask_image`was provided\n - invalid_prompts: One or more of the prompts contains filtered words\n - invalid_pixel_count: Incorrect number of pixels specified. Requirements:\n - For 512 engines: 262,144 ≤`height _ width`≤ 1,048,576\n - For 768 engines: 589,824 ≤`height \* width`≤ 1,048,576\n - invalid_sdxl_v222_dimensions: Incorrect dimensions specified for SDXL v2-2-2 engine. Requirements:\n - Neither`height`nor`width`may be below 128\n - Only one of`height`or`width`may be above 512 (e.g. 512x768 is valid but 578x768 is not)\n - Maximum dimensions supported are 512x896 or 896x512 \n - invalid_sdxl_v1_dimensions: Incorrect dimensions specified for SDXL v0.9 or v1.0 engine. Valid dimensions:\n - 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "id": "296a972f-666a-44a1-a3df-c9c28a1f56c0", + "name": "bad_request", + "message": "init_image: is required" + } + } + } + }, + "400FromUpscale": { + "description": "General error for invalid parameters, see below for specific errors.\n\n - bad_request: one or more provided parameters are invalid (see error description for details)\n - invalid_file_size: The file size of one or more of the provided files is invalid\n - invalid_mime_type: The mime type of one or more of the provided files is invalid\n - invalid_pixel_count: The requested image would exceed the maximum pixel count of 4,194,304", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "id": "296a972f-666a-44a1-a3df-c9c28a1f56c0", + "name": "bad_request", + "message": "image: is required" + } + } + } + } + }, + "parameters": { + "upscaleEngineID": { + "in": "path", + "name": "engine_id", + "required": true, + "schema": { + "type": "string" + }, + "examples": { + "ESRGAN_X2_PLUS": { + "description": "ESRGAN x2 Upscaler", + "value": "esrgan-v1-x2plus" + }, + "LATENT_UPSCALER_X4": { + "description": "Stable Diffusion x4 Latent Upscaler", + "value": "stable-diffusion-x4-latent-upscaler" + } + } + }, + "engineID": { + "example": "stable-diffusion-v1-5", + "in": "path", + "name": "engine_id", + "required": true, + "schema": { + "example": "stable-diffusion-v1-5", + "type": "string" + } + }, + "organization": { + "allowEmptyValue": false, + "description": "Allows for requests to be scoped to an organization other than the user's default. If not provided, the user's default organization will be used.", + "example": "org-123456", + "in": "header", + "name": "Organization", + "x-go-name": "OrganizationID", + "schema": { + "type": "string" + } + }, + "stabilityClientID": { + "allowEmptyValue": false, + "description": "Used to identify the source of requests, such as the client application or sub-organization. Optional, but recommended for organizational clarity.", + "example": "my-great-plugin", + "in": "header", + "name": "Stability-Client-ID", + "schema": { + "type": "string" + } + }, + "stabilityClientVersion": { + "allowEmptyValue": false, + "description": "Used to identify the version of the application or service making the requests. Optional, but recommended for organizational clarity.", + "example": "1.2.1", + "in": "header", + "name": "Stability-Client-Version", + "schema": { + "type": "string" + } + }, + "accept": { + "allowEmptyValue": false, + "in": "header", + "name": "Accept", + "description": "The format of the response. Leave blank for JSON, or set to 'image/png' for a PNG image.", + "schema": { + "default": "application/json", + "enum": [ + "application/json", + "image/png" + ], + "type": "string" + } + } + }, + "headers": { + "Content-Length": { + "required": true, + "schema": { + "type": "integer" + } + }, + "Content-Type": { + "required": true, + "schema": { + "enum": [ + "application/json", + "image/png" + ], + "type": "string" + } + }, + "Finish-Reason": { + "schema": { + "$ref": "#/components/schemas/FinishReason" + } + }, + "Seed": { + "example": 3817857576, + "schema": { + "example": 787078103, + "type": "integer" + }, + "description": "The seed used to generate the image. This header is only present when the`Accept`is set to`image/png`. Otherwise it is returned in the response body." +} +}, +"securitySchemes": { +"STABILITY_API_KEY": { +"type": "apiKey", +"name": "Authorization", +"in": "header" +} +} +} +} diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index 70ce25b07bbd6..cb041a8a47e11 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -1,6 +1,9 @@ no_docs_user_prompt = """%s. The app is %s.""" -no_docs_system_instructions = """You are an agent that creates Pipedream Action Component Code. Your code should answer the question you are given. +# Experimenting with prompt from an AI researcher: https://twitter.com/jeremyphoward/status/1689464589191454720 +no_docs_system_instructions = """You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. + +Your goal is to create Pipedream Action Components. Your code should solve the requirements provided in the instructions. output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! @@ -16,7 +19,7 @@ key: "openai-list-models", name: "List Models", description: "Lists all models available to the user.", - version: "0.0.1", + version: "0.0.{{ts}}", type: "action", props: { openai: { @@ -59,7 +62,7 @@ The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. +The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any parameters of the API as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. @@ -97,7 +100,7 @@ export default defineComponent({ key: "slack-send-message", name: "Send Message", - version: "0.0.1", + version: "0.0.{{ts}}", description: "Sends a message to a channel. [See docs here]()", type: "action", props: { @@ -133,6 +136,15 @@ }, }); +Notice this section: + +data: { + channel: this.channel, + text: this.text, +}, + +This shows you how to pass the values of props (e.g. this.channel and this.text) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See rule #2 below for more detail. + The code you generate should be placed within the `run` method of the Pipedream component: import { axios } from "@pipedream/platform"; @@ -226,7 +238,7 @@ key: "google_drive-list-all-drives", name: "List All Drives", description: "Lists all drives in an account.", - version: "0.0.1", + version: "0.0.{{ts}}", type: "action", }; ``` @@ -235,7 +247,7 @@ You should come up with a name and a description for the component you are generating. In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). -Always add version "0.0.1" and type "action". +Always add version "0.0.{{ts}}" and type "action". You MUST add metadata to the component code you generate. ## TypeScript Definitions @@ -394,7 +406,47 @@ 1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. +2. Include all parameters of the API request as props. DO NOT use example values from any API docs, OpenAPI specs, or example code above or that you've been trained on. + +For example, do this: + +data: { + text_prompts: [ + { + text: this.textPrompt, + weight: this.weight, + }, + ], + cfg_scale: this.cfg_scale, + height: this.height, + width: this.width, + samples: this.samples, + steps: this.steps, +}, + +not this: + +data: { + text_prompts: [ + { + text: this.textPrompt, + weight: 1, + }, + ], + cfg_scale: 7, + height: 512, + width: 512, + samples: 1, + steps: 75, +}, + +Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. + +You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. + +Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. + +I need to reiterate: if the docs / spec use an example value, you MUST NOT use that example value in the code. You MUST use the value of the prop instead. Think about it: if you hardcode values in the code, the user can't enter their own value. 3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. @@ -412,6 +464,12 @@ Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. +8. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs. + +9. You must pass a value of `0.0.{{ts}}` to the `version` property. This is the only valid version value. "{{ts}}" is expanded by the Pipedream platform to the current epoch ms timestamp. Think about it: if you pass a different value, the developer won't be able to republish the component with a dynamic version, and publishing will fail, which will waste their time. + +10. Remember, please do not pass example values from the API docs or OpenAPI spec. You must pass the value of the prop to all params instead. This is the only way the user can enter their own values. + ## Remember, return ONLY code Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. @@ -420,6 +478,7 @@ """ with_docs_system_instructions = f"""{no_docs_system_instructions} + You are an agent designed to interact with an OpenAPI JSON specification. You have access to the following tools which help you learn more about the JSON you are interacting with. Only use the below tools. Only use the information returned by the below tools to construct your final answer. From e60deeb82603512f2d4c2a4fd143d89821277c14 Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Sat, 26 Aug 2023 17:47:45 -0700 Subject: [PATCH 39/44] Adding pnpm.lock --- pnpm-lock.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e31a156f1f3c5..858e9b5a475ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2789,6 +2789,8 @@ importers: components/metabase: {} + components/metaphor: {} + components/metatext_ai_inference_api: {} components/metatext_ai_pre_build_ai_models_api: From 7140f69cbedeb0378f46832151e684167f52410d Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Sun, 27 Aug 2023 14:41:12 -0700 Subject: [PATCH 40/44] pnpm.lock --- pnpm-lock.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37b80d82d22c4..7cd4d81cd0c5b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,8 @@ importers: components/accredible: {} + components/acelle_mail: {} + components/action_builder: {} components/action_network: {} @@ -1298,6 +1300,8 @@ importers: specifier: ^4.17.21 version: 4.17.21 + components/dreamstudio: {} + components/dribbble: dependencies: '@pipedream/platform': @@ -2520,6 +2524,8 @@ importers: components/kucoin_futures: {} + components/kwtsms: {} + components/kyvio: {} components/labs64_netlicensing: {} @@ -2621,6 +2627,8 @@ importers: components/liondesk: {} + components/listclean: {} + components/listen_notes: {} components/liveagent: @@ -2785,6 +2793,8 @@ importers: components/mapbox: {} + components/mapulus: {} + components/marketing_master_io: {} components/marketstack: {} @@ -5369,6 +5379,8 @@ importers: components/verifybee: {} + components/vero: {} + components/vestaboard: dependencies: '@pipedream/platform': From 311a192dab0b28433b7b3aab8b0ce545265de2b9 Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Mon, 28 Aug 2023 15:27:51 -0700 Subject: [PATCH 41/44] Multi agent debate (#7787) * Support for multi-agent debate * pnpm.lock --- packages/component_code_gen/.gitignore | 1 + .../code_gen/generate_component_code.py | 86 +++++++++++-------- packages/component_code_gen/config/config.py | 3 +- .../helpers/langchain_helpers.py | 29 ++++--- packages/component_code_gen/main.py | 22 +++-- .../templates/generate_actions.py | 51 +++++++++-- 6 files changed, 126 insertions(+), 66 deletions(-) diff --git a/packages/component_code_gen/.gitignore b/packages/component_code_gen/.gitignore index 3fba59fa7f17d..7bac48469580d 100644 --- a/packages/component_code_gen/.gitignore +++ b/packages/component_code_gen/.gitignore @@ -5,3 +5,4 @@ ve/ node_modules/ __pycache__/ tests/**/output +*.mjs diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index bf9ffeaa02b0d..4979be3157aa1 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -1,76 +1,94 @@ +import config.logging_config as logging_config +from config.config import config +import helpers.supabase_helpers as supabase_helpers +import helpers.langchain_helpers as langchain_helpers from dotenv import load_dotenv load_dotenv() -import helpers.langchain_helpers as langchain_helpers -import helpers.supabase_helpers as supabase_helpers -from config.config import config -import config.logging_config as logging_config logger = logging_config.getLogger(__name__) -def generate_code(app, prompt, templates): - validate_inputs(app, prompt, templates) +def generate_code(app, prompt, templates, tries): + results = [] - db = supabase_helpers.SupabaseConnector() + for i in range(tries): + logger.debug(f'Attempt {i+1} of {tries}') + validate_inputs(app, prompt, templates, tries) - auth_meta = db.get_app_auth_meta(app) - add_code_example(templates, auth_meta['component_code_scaffold_raw']) # TODO: is this needed only for actions? + db = supabase_helpers.SupabaseConnector() - if config['enable_docs'] == False: - logger.warn("docs are disabled") - return no_docs(app, prompt, templates) + auth_meta = db.get_app_auth_meta(app) + # TODO: is this needed only for actions? + add_code_example(templates, auth_meta['component_code_scaffold_raw']) - docs_meta = db.get_app_docs_meta(app) + docs_meta = db.get_app_docs_meta(app) + # Initialize a flag to track if we obtained any results with docs + has_docs_result = False - try: if 'docs_url' in docs_meta: contents = db.get_docs_contents(app) if contents: - docs = { row['url']: row['content'] for row in contents } - return with_docs(app, prompt, docs, 'api reference', templates) - except Exception as e: - logger.error(e) - logger.error("failed with docs") + docs = {row['url']: row['content'] for row in contents} + results.append(call_langchain( + app, prompt, templates, docs, 'api reference')) + has_docs_result = True - try: if 'openapi_url' in docs_meta: contents = db.get_openapi_contents(app) if contents: - docs = { row['path']: row['content'] for row in contents } - return with_docs(app, prompt, docs, 'openapi', templates) - except Exception as e: - logger.error(e) - logger.error("failed with openapi") + docs = {row['path']: row['content'] for row in contents} + results.append(call_langchain( + app, prompt, templates, docs, 'openapi')) + has_docs_result = True + + # If we haven't obtained any results using docs + if not has_docs_result: + results.append(call_langchain(app, prompt, templates)) - return no_docs(app, prompt, templates) + # Create a new prompt string + new_prompt = "I've asked other GPT agents to generate the following code based on the prompt and the instructions below. One set of code (or all) may not follow the rules laid out in the prompt or in the instructions below, so you'll need to review it for accuracy. Try to evaluate the examples according to the rules, combine the best parts of each example, and generate a final set of code that solves the problem posed by the prompt and follows all of the rules below. Here are the attempts + code:\n\n---\n\n" + for idx, result in enumerate(results, 1): + new_prompt += f"Try {idx}:\n\n${result}\n\n---\n\n" + logger.debug(f"\n\n---------------{new_prompt}\n\n") + new_prompt += "---\n\n" + prompt + # Call the model again with the new prompt to get the final result + logger.debug(f"Calling the model a final time to summarize the attempts") + return call_langchain(app, new_prompt, templates) -def no_docs(app, prompt, templates): - logger.debug('no docs, calling openai directly') - return langchain_helpers.no_docs(app, prompt, templates) +def call_langchain(app, prompt, templates, docs=None, docs_type=None, attempts=0, max_attempts=3): + logger.debug(f"Calling langchain") + # If we don't have docs, or if we can't reach OpenAI to get the parsed docs + if not docs: + logger.debug('No docs available, calling the model directly') + return langchain_helpers.no_docs(app, prompt, templates) + if attempts >= max_attempts: + logger.debug('Max attempts reached, calling the model directly') + return langchain_helpers.no_docs(app, prompt, templates) -def with_docs(app, prompt, docs, docs_type, templates): - logger.debug(f"using {docs_type} docs") + # else if we have docs, call the model with docs + logger.debug(f"Using {docs_type} docs") result = langchain_helpers.ask_agent(prompt, docs, templates) if result != "I don't know": return result - logger.debug("trying again without docs") - return no_docs(app, prompt, templates) + logger.debug("Trying again without docs") + return call_langchain(app, prompt, templates, attempts=attempts+1) def add_code_example(templates, example): return templates.no_docs_system_instructions % example -def validate_inputs(app, prompt, templates): +def validate_inputs(app, prompt, templates, tries): assert app and type(app) == str assert prompt and type(prompt) == str + assert tries and type(tries) == int assert templates.no_docs_user_prompt assert templates.no_docs_system_instructions assert templates.with_docs_system_instructions diff --git a/packages/component_code_gen/config/config.py b/packages/component_code_gen/config/config.py index 6395aab1dfb67..88f9cc17764e4 100644 --- a/packages/component_code_gen/config/config.py +++ b/packages/component_code_gen/config/config.py @@ -22,6 +22,7 @@ def get_env_var(var_name, required=False): config = { + "temperature": get_env_var("OPENAI_TEMPERATURE") or 0.5, "openai_api_type": get_env_var("OPENAI_API_TYPE"), "openai": { "api_key": get_env_var("OPENAI_API_KEY", required=True), @@ -44,5 +45,5 @@ def get_env_var(var_name, required=False): "logging": { "level": get_env_var("LOGGING_LEVEL"), }, - "enable_docs": get_env_var("ENABLE_DOCS"), + "enable_docs": get_env_var("ENABLE_DOCS") or False, } diff --git a/packages/component_code_gen/helpers/langchain_helpers.py b/packages/component_code_gen/helpers/langchain_helpers.py index d5490f80c5afa..0413f257232f5 100644 --- a/packages/component_code_gen/helpers/langchain_helpers.py +++ b/packages/component_code_gen/helpers/langchain_helpers.py @@ -1,18 +1,17 @@ -from dotenv import load_dotenv -load_dotenv() - -import openai # required -from config.config import config -from langchain import LLMChain -from langchain.agents import ZeroShotAgent, AgentExecutor -from langchain.chat_models import ChatOpenAI, AzureChatOpenAI -from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit -from langchain.tools.json.tool import JsonSpec from langchain.schema import ( # AIMessage, HumanMessage, SystemMessage ) +from langchain.tools.json.tool import JsonSpec +from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit +from langchain.chat_models import ChatOpenAI, AzureChatOpenAI +from langchain.agents import ZeroShotAgent, AgentExecutor +from langchain import LLMChain +from config.config import config +import openai # required +from dotenv import load_dotenv +load_dotenv() class OpenAPIExplorerTool: @@ -55,7 +54,7 @@ def run(self, input): def format_template(text): - return text.replace("{", "{{").replace("}", "}}") # escape curly braces + return text.replace("{", "{{").replace("}", "}}") # escape curly braces def format_result(result): @@ -68,10 +67,11 @@ def get_llm(): if config['openai_api_type'] == "azure": azure_config = config["azure"] llm = AzureChatOpenAI(deployment_name=azure_config['deployment_name'], - model_name=azure_config["model"], temperature=0, request_timeout=300) + model_name=azure_config["model"], temperature=config["temperature"], request_timeout=300) else: openai_config = config["openai"] - llm = ChatOpenAI(model_name=openai_config["model"], temperature=0, request_timeout=300) + llm = ChatOpenAI( + model_name=openai_config["model"], temperature=config["temperature"], request_timeout=300) return llm @@ -83,7 +83,8 @@ def ask_agent(user_prompt, docs, templates): def no_docs(app, prompt, templates): result = get_llm()(messages=[ - SystemMessage(content=format_template(templates.no_docs_system_instructions)), + SystemMessage(content=format_template( + templates.no_docs_system_instructions)), HumanMessage(content=templates.no_docs_user_prompt % (prompt, app)), ]) diff --git a/packages/component_code_gen/main.py b/packages/component_code_gen/main.py index 7c20c88962ea9..f700ab7c511c7 100644 --- a/packages/component_code_gen/main.py +++ b/packages/component_code_gen/main.py @@ -14,31 +14,37 @@ } -def main(component_type, app, instructions, verbose=False): +def main(component_type, app, instructions, tries, verbose=False): if verbose: os.environ['LOGGING_LEVEL'] = 'DEBUG' try: templates = available_templates[component_type] except: - raise ValueError(f'Templates for {component_type}s are not available. Please choose one of {available_templates.keys()}') + raise ValueError( + f'Templates for {component_type}s are not available. Please choose one of {available_templates.keys()}') # this is here so that the DEBUG environment variable is set before the import from code_gen.generate_component_code import generate_code - result = generate_code(app, instructions, templates) + result = generate_code(app, instructions, templates, tries) return result if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--type', help='which kind of code you want to generate?', choices=available_templates.keys(), required=True) - parser.add_argument('--app', help='the app_name_slug', required=True) - parser.add_argument('--instructions', help='markdown file with instructions: prompt + api docs', required=True) - parser.add_argument('--verbose', dest='verbose', help='set the logging to debug', required=False, default=False, action='store_true') + parser.add_argument('--type', help='Which kind of code you want to generate?', + choices=available_templates.keys(), required=True) + parser.add_argument('--app', help='The app_name_slug', required=True) + parser.add_argument( + '--instructions', help='Markdown file with instructions: prompt + api docs', required=True) + parser.add_argument('--num_tries', dest='tries', help='The number of times we call the model to generate code', + required=False, default=3, action='store_true') + parser.add_argument('--verbose', dest='verbose', help='Set the logging to debug', + required=False, default=False, action='store_true') args = parser.parse_args() with open(args.instructions, 'r') as f: instructions = f.read() - result = main(args.type, args.app, instructions, args.verbose) + result = main(args.type, args.app, instructions, args.tries, args.verbose) print(result) diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index cb041a8a47e11..ab8736d71f404 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -1,9 +1,13 @@ no_docs_user_prompt = """%s. The app is %s.""" # Experimenting with prompt from an AI researcher: https://twitter.com/jeremyphoward/status/1689464589191454720 -no_docs_system_instructions = """You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. +no_docs_system_instructions = """## Instructions -Your goal is to create Pipedream Action Components. Your code should solve the requirements provided in the instructions. +You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. + +Your goal is to create Pipedream Action Components. Your code should solve the requirements provided below. + +Other GPT agents will be reviewing your work, and will provide feedback on your code. You will be rewarded for code that is accurate, factual, thoughtful, nuanced, and solves the requirements provided in the instructions. output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! @@ -404,12 +408,17 @@ ## Additional rules +When you generate code, you must follow all of the rules above. Review the rules and think through them step-by-step before you generate code. Look at how these map to the example code and component API described above. + +Once you generate your code, you must review each of these rules again, one-by-one, and ensure you've followed them. Accuracy is critical, and we can wait for you to review your code. If you notice you haven't followed a particular rule, you can regenerate your code and start over. If you do make any edits, you'll need to again review each rule one-by-one to make sure your edits didn't conflict with another rule. I cannot stress how critical it is to follow all of the rules below. Consider it your constitution. + 1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. 2. Include all parameters of the API request as props. DO NOT use example values from any API docs, OpenAPI specs, or example code above or that you've been trained on. For example, do this: +``` data: { text_prompts: [ { @@ -422,10 +431,12 @@ width: this.width, samples: this.samples, steps: this.steps, -}, +} +``` -not this: +But never do this: +``` data: { text_prompts: [ { @@ -438,15 +449,18 @@ width: 512, samples: 1, steps: 75, -}, +} +``` + +You can see that there's no static value present in the first example. In the second example, the values are hardcoded, so the user can't enter their own values. -Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. +I need to reiterate: you MUST NOT use static, example values in the code. You MUST use the value of the prop (this.) instead. Think about it: if you hardcode values in the code, the user can't enter their own value. -You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. +2b. Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. -Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. +2c. You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. -I need to reiterate: if the docs / spec use an example value, you MUST NOT use that example value in the code. You MUST use the value of the prop instead. Think about it: if you hardcode values in the code, the user can't enter their own value. +2d. Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. 3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. @@ -466,6 +480,25 @@ 8. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs. +For example, do this: + +const response = await axios(this, { + url: `https://api.stability.ai/v1/engines/list`, + headers: { + Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, + }, +}); +// process the response data. response.data is undefined + +not this: + +const { data } = await axios(this, { + url: `https://api.stability.ai/v1/engines/list`, + headers: { + Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, + }, +}); + 9. You must pass a value of `0.0.{{ts}}` to the `version` property. This is the only valid version value. "{{ts}}" is expanded by the Pipedream platform to the current epoch ms timestamp. Think about it: if you pass a different value, the developer won't be able to republish the component with a dynamic version, and publishing will fail, which will waste their time. 10. Remember, please do not pass example values from the API docs or OpenAPI spec. You must pass the value of the prop to all params instead. This is the only way the user can enter their own values. From 48eefab7982d8d462afac66b9363826507524fdc Mon Sep 17 00:00:00 2001 From: Danny Roosevelt Date: Tue, 29 Aug 2023 15:44:29 -0700 Subject: [PATCH 42/44] Improving readme, updating tool-versions --- .tool-versions | 2 ++ packages/component_code_gen/.env.example | 1 + packages/component_code_gen/README.md | 25 ++++++++++++++---------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.tool-versions b/.tool-versions index d834d7c85b1c5..cfee223d606f7 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,4 @@ nodejs 14.19.0 pnpm 7.13.4 +python 3.11.5 +poetry 1.6.1 \ No newline at end of file diff --git a/packages/component_code_gen/.env.example b/packages/component_code_gen/.env.example index 82df9f272f86e..fafb9271a90ff 100644 --- a/packages/component_code_gen/.env.example +++ b/packages/component_code_gen/.env.example @@ -14,3 +14,4 @@ OPENAI_API_VERSION=2023-05-15 OPENAI_API_BASE=https://resource-name.openai.azure.com OPENAI_API_KEY=azure-api-key OPENAI_MODEL=gpt-4-32k +OPENAI_TEMPERATURE=0.5 \ No newline at end of file diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index 74f452de1ae7b..b505582f2a957 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -5,32 +5,37 @@ Generate components using OpenAI GPT. ### Installation -1. Install poetry: follow instructions at https://python-poetry.org/docs/#installation - -2. Run install: - ``` +asdf plugin-add poetry +asdf install +cd packages/component_code_gen poetry install ``` - ### Setup -1. Create a `.env` file +#### Create a `.env` file + +``` +cd packages/component_code_gen +cp .env.example .env +``` + +#### Modify the `.env` file to use your own keys: -2. Add these API Keys: +1. Add these API Keys to your new `.env` file: - BROWSERLESS_API_KEY=api_key # not required - SUPABASE_URL=https://your-project-url.supabase.co # get this from Supabase Project Settings -> API - SUPABASE_API_KEY=service_role_key # get this from Supabase Project Settings -> API -3. Add OpenAI keys +2. Add OpenAI keys - OPENAI_API_TYPE=openai - OPENAI_API_KEY=your-openai-api-key - OPENAI_MODEL=gpt-4 -4. Or use a Azure OpenAI deployment (gpt-4-32k) +3. Or use a Azure OpenAI deployment (gpt-4-32k) - OPENAI_API_TYPE=azure - OPENAI_DEPLOYMENT_NAME=deployment-name @@ -39,7 +44,7 @@ poetry install - OPENAI_API_KEY=azure-api-key - OPENAI_MODEL=gpt-4-32k -5. Create a file named `instructions.md` with the same structure as the `instructions.md.exaple` file: +4. Create a file named `instructions.md` with the same structure as the `instructions.md.exaple` file: ``` ## Prompt From 09cbbfa0ae33d017387c0d7ca47da7d0bc871320 Mon Sep 17 00:00:00 2001 From: Danny Roosevelt Date: Tue, 29 Aug 2023 16:12:17 -0700 Subject: [PATCH 43/44] Update generate_component_code.py --- .../component_code_gen/code_gen/generate_component_code.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index 4979be3157aa1..b90c1dd546051 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -20,7 +20,7 @@ def generate_code(app, prompt, templates, tries): auth_meta = db.get_app_auth_meta(app) # TODO: is this needed only for actions? - add_code_example(templates, auth_meta['component_code_scaffold_raw']) + # add_code_example(templates, auth_meta['component_code_scaffold_raw']) docs_meta = db.get_app_docs_meta(app) # Initialize a flag to track if we obtained any results with docs @@ -50,7 +50,6 @@ def generate_code(app, prompt, templates, tries): new_prompt = "I've asked other GPT agents to generate the following code based on the prompt and the instructions below. One set of code (or all) may not follow the rules laid out in the prompt or in the instructions below, so you'll need to review it for accuracy. Try to evaluate the examples according to the rules, combine the best parts of each example, and generate a final set of code that solves the problem posed by the prompt and follows all of the rules below. Here are the attempts + code:\n\n---\n\n" for idx, result in enumerate(results, 1): new_prompt += f"Try {idx}:\n\n${result}\n\n---\n\n" - logger.debug(f"\n\n---------------{new_prompt}\n\n") new_prompt += "---\n\n" + prompt # Call the model again with the new prompt to get the final result From 355352a89a4847f0dbe5349d892f83ac7292bc0a Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 30 Aug 2023 17:14:08 -0300 Subject: [PATCH 44/44] Templates restructure (#7816) * requested changes * templates organization * add auth authentication example * modify config default values * add newlines * fix case when there is no code scaffolding * separate common rules and additional rules --- packages/component_code_gen/README.md | 2 +- .../code_gen/generate_component_code.py | 44 +- packages/component_code_gen/config/config.py | 34 +- .../helpers/langchain_helpers.py | 38 +- .../templates/actions/additional_rules.py | 32 + .../templates/actions/export_summary.py | 1 + .../templates/actions/introduction.py | 12 + .../templates/actions/main_example.py | 28 + .../templates/actions/other_example.py | 69 +++ .../templates/apps/additional_rules.py | 1 + .../component_code_gen/templates/apps/auth.py | 7 + .../templates/apps/introduction.py | 6 + .../templates/apps/main_example.py | 136 +++++ .../templates/apps/methods.py | 9 + .../templates/apps/prop_definitions.py | 3 + .../templates/common/app_prop.py | 15 + .../templates/common/async_options.py | 63 ++ .../templates/common/auth.py | 7 + .../templates/common/component_metadata.py | 49 ++ .../common/docs_system_instructions.py | 19 + .../templates/common/end.py | 6 + .../templates/common/format_instructions.py | 10 + .../templates/common/platform_axios.py | 24 + .../templates/common/props.py | 5 + .../templates/common/rules.py | 65 ++ .../templates/common/suffix.py | 3 + .../common/typescript_definitions.py | 152 +++++ .../templates/generate_actions.py | 567 +----------------- .../templates/generate_apps.py | 478 +-------------- .../templates/generate_polling_sources.py | 497 +-------------- .../templates/generate_webhook_sources.py | 526 ++-------------- .../templates/sources/db.py | 5 + .../sources/polling/additional_rules.py | 9 + .../templates/sources/polling/hooks.py | 3 + .../templates/sources/polling/introduction.py | 7 + .../templates/sources/polling/main_example.py | 100 +++ .../sources/webhooks/additional_rules.py | 7 + .../templates/sources/webhooks/async_run.py | 1 + .../templates/sources/webhooks/hooks.py | 3 + .../templates/sources/webhooks/http.py | 5 + .../sources/webhooks/introduction.py | 8 + .../sources/webhooks/main_example.py | 66 ++ .../sources/webhooks/other_example.py | 59 ++ 43 files changed, 1170 insertions(+), 2011 deletions(-) create mode 100644 packages/component_code_gen/templates/actions/additional_rules.py create mode 100644 packages/component_code_gen/templates/actions/export_summary.py create mode 100644 packages/component_code_gen/templates/actions/introduction.py create mode 100644 packages/component_code_gen/templates/actions/main_example.py create mode 100644 packages/component_code_gen/templates/actions/other_example.py create mode 100644 packages/component_code_gen/templates/apps/additional_rules.py create mode 100644 packages/component_code_gen/templates/apps/auth.py create mode 100644 packages/component_code_gen/templates/apps/introduction.py create mode 100644 packages/component_code_gen/templates/apps/main_example.py create mode 100644 packages/component_code_gen/templates/apps/methods.py create mode 100644 packages/component_code_gen/templates/apps/prop_definitions.py create mode 100644 packages/component_code_gen/templates/common/app_prop.py create mode 100644 packages/component_code_gen/templates/common/async_options.py create mode 100644 packages/component_code_gen/templates/common/auth.py create mode 100644 packages/component_code_gen/templates/common/component_metadata.py create mode 100644 packages/component_code_gen/templates/common/docs_system_instructions.py create mode 100644 packages/component_code_gen/templates/common/end.py create mode 100644 packages/component_code_gen/templates/common/format_instructions.py create mode 100644 packages/component_code_gen/templates/common/platform_axios.py create mode 100644 packages/component_code_gen/templates/common/props.py create mode 100644 packages/component_code_gen/templates/common/rules.py create mode 100644 packages/component_code_gen/templates/common/suffix.py create mode 100644 packages/component_code_gen/templates/common/typescript_definitions.py create mode 100644 packages/component_code_gen/templates/sources/db.py create mode 100644 packages/component_code_gen/templates/sources/polling/additional_rules.py create mode 100644 packages/component_code_gen/templates/sources/polling/hooks.py create mode 100644 packages/component_code_gen/templates/sources/polling/introduction.py create mode 100644 packages/component_code_gen/templates/sources/polling/main_example.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/additional_rules.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/async_run.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/hooks.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/http.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/introduction.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/main_example.py create mode 100644 packages/component_code_gen/templates/sources/webhooks/other_example.py diff --git a/packages/component_code_gen/README.md b/packages/component_code_gen/README.md index b505582f2a957..08b4fcaccfa1a 100644 --- a/packages/component_code_gen/README.md +++ b/packages/component_code_gen/README.md @@ -44,7 +44,7 @@ cp .env.example .env - OPENAI_API_KEY=azure-api-key - OPENAI_MODEL=gpt-4-32k -4. Create a file named `instructions.md` with the same structure as the `instructions.md.exaple` file: +5. Create a file named `instructions.md` with the same structure as the `instructions.md.example` file: ``` ## Prompt diff --git a/packages/component_code_gen/code_gen/generate_component_code.py b/packages/component_code_gen/code_gen/generate_component_code.py index b90c1dd546051..61d2d0a8f3c5c 100644 --- a/packages/component_code_gen/code_gen/generate_component_code.py +++ b/packages/component_code_gen/code_gen/generate_component_code.py @@ -10,19 +10,19 @@ def generate_code(app, prompt, templates, tries): + validate_inputs(app, prompt, templates, tries) + db = supabase_helpers.SupabaseConnector() + docs_meta = db.get_app_docs_meta(app) results = [] + auth_example = None + auth_meta = db.get_app_auth_meta(app) + if auth_meta.get('component_code_scaffold_raw'): + auth_example = f"Here's how authentication is done in {app}:\n\n{auth_meta['component_code_scaffold_raw']}\n\n" + for i in range(tries): logger.debug(f'Attempt {i+1} of {tries}') - validate_inputs(app, prompt, templates, tries) - - db = supabase_helpers.SupabaseConnector() - - auth_meta = db.get_app_auth_meta(app) - # TODO: is this needed only for actions? - # add_code_example(templates, auth_meta['component_code_scaffold_raw']) - docs_meta = db.get_app_docs_meta(app) # Initialize a flag to track if we obtained any results with docs has_docs_result = False @@ -31,7 +31,7 @@ def generate_code(app, prompt, templates, tries): if contents: docs = {row['url']: row['content'] for row in contents} results.append(call_langchain( - app, prompt, templates, docs, 'api reference')) + app, prompt, templates, auth_example, docs, 'api reference')) has_docs_result = True if 'openapi_url' in docs_meta: @@ -39,12 +39,12 @@ def generate_code(app, prompt, templates, tries): if contents: docs = {row['path']: row['content'] for row in contents} results.append(call_langchain( - app, prompt, templates, docs, 'openapi')) + app, prompt, templates, auth_example, docs, 'openapi')) has_docs_result = True # If we haven't obtained any results using docs if not has_docs_result: - results.append(call_langchain(app, prompt, templates)) + results.append(call_langchain(app, prompt, templates, auth_example)) # Create a new prompt string new_prompt = "I've asked other GPT agents to generate the following code based on the prompt and the instructions below. One set of code (or all) may not follow the rules laid out in the prompt or in the instructions below, so you'll need to review it for accuracy. Try to evaluate the examples according to the rules, combine the best parts of each example, and generate a final set of code that solves the problem posed by the prompt and follows all of the rules below. Here are the attempts + code:\n\n---\n\n" @@ -54,42 +54,34 @@ def generate_code(app, prompt, templates, tries): # Call the model again with the new prompt to get the final result logger.debug(f"Calling the model a final time to summarize the attempts") - return call_langchain(app, new_prompt, templates) + return call_langchain(app, new_prompt, templates, auth_example) -def call_langchain(app, prompt, templates, docs=None, docs_type=None, attempts=0, max_attempts=3): +def call_langchain(app, prompt, templates, auth_example, docs=None, docs_type=None, attempts=0, max_attempts=3): logger.debug(f"Calling langchain") # If we don't have docs, or if we can't reach OpenAI to get the parsed docs if not docs: logger.debug('No docs available, calling the model directly') - return langchain_helpers.no_docs(app, prompt, templates) + return langchain_helpers.no_docs(app, prompt, templates, auth_example) if attempts >= max_attempts: logger.debug('Max attempts reached, calling the model directly') - return langchain_helpers.no_docs(app, prompt, templates) + return langchain_helpers.no_docs(app, prompt, templates, auth_example) # else if we have docs, call the model with docs logger.debug(f"Using {docs_type} docs") - result = langchain_helpers.ask_agent(prompt, docs, templates) + result = langchain_helpers.ask_agent(prompt, docs, templates, auth_example) if result != "I don't know": return result logger.debug("Trying again without docs") - return call_langchain(app, prompt, templates, attempts=attempts+1) - - -def add_code_example(templates, example): - return templates.no_docs_system_instructions % example + return call_langchain(app, prompt, templates, auth_example, attempts=attempts+1) def validate_inputs(app, prompt, templates, tries): assert app and type(app) == str assert prompt and type(prompt) == str assert tries and type(tries) == int - assert templates.no_docs_user_prompt - assert templates.no_docs_system_instructions - assert templates.with_docs_system_instructions - assert templates.suffix - assert templates.format_instructions + assert templates.system_instructions diff --git a/packages/component_code_gen/config/config.py b/packages/component_code_gen/config/config.py index 88f9cc17764e4..9aafa2ee15ae7 100644 --- a/packages/component_code_gen/config/config.py +++ b/packages/component_code_gen/config/config.py @@ -3,37 +3,28 @@ load_dotenv() -DEFAULTS = { - "OPENAI_API_TYPE": "openai", - "OPENAI_MODEL": "gpt-4", - "OPENAI_API_VERSION": "2023-05-15", - "LOGGING_LEVEL": "WARN", - "ENABLE_DOCS": False -} - - -def get_env_var(var_name, required=False): +def get_env_var(var_name, required=False, default=None): if os.environ.get(var_name): return os.environ.get(var_name) - if required and var_name not in DEFAULTS: + if default is not None: + return default + if required: raise Exception(f"Environment variable {var_name} is required") - if var_name in DEFAULTS: - return DEFAULTS[var_name] config = { - "temperature": get_env_var("OPENAI_TEMPERATURE") or 0.5, - "openai_api_type": get_env_var("OPENAI_API_TYPE"), + "temperature": get_env_var("OPENAI_TEMPERATURE", default=0.5), + "openai_api_type": get_env_var("OPENAI_API_TYPE", default="azure"), "openai": { "api_key": get_env_var("OPENAI_API_KEY", required=True), - "model": get_env_var("OPENAI_MODEL"), + "model": get_env_var("OPENAI_MODEL", default="gpt-4"), }, "azure": { - "deployment_name": get_env_var("OPENAI_DEPLOYMENT_NAME"), - "api_version": get_env_var("OPENAI_API_VERSION"), - "api_base": get_env_var("OPENAI_API_BASE"), + "deployment_name": get_env_var("OPENAI_DEPLOYMENT_NAME", required=True), + "api_version": get_env_var("OPENAI_API_VERSION", default="2023-05-15"), + "api_base": get_env_var("OPENAI_API_BASE", required=True), "api_key": get_env_var("OPENAI_API_KEY", required=True), - "model": get_env_var("OPENAI_MODEL"), + "model": get_env_var("OPENAI_MODEL", default="gpt-4-32k"), }, "browserless": { "api_key": get_env_var("BROWSERLESS_API_KEY"), @@ -43,7 +34,6 @@ def get_env_var(var_name, required=False): "api_key": get_env_var("SUPABASE_API_KEY", required=True), }, "logging": { - "level": get_env_var("LOGGING_LEVEL"), + "level": get_env_var("LOGGING_LEVEL", default="WARN"), }, - "enable_docs": get_env_var("ENABLE_DOCS") or False, } diff --git a/packages/component_code_gen/helpers/langchain_helpers.py b/packages/component_code_gen/helpers/langchain_helpers.py index 0413f257232f5..459d4f6a13275 100644 --- a/packages/component_code_gen/helpers/langchain_helpers.py +++ b/packages/component_code_gen/helpers/langchain_helpers.py @@ -1,3 +1,6 @@ +from templates.common.suffix import suffix +from templates.common.format_instructions import format_instructions +from templates.common.docs_system_instructions import docs_system_instructions from langchain.schema import ( # AIMessage, HumanMessage, @@ -24,19 +27,25 @@ def create_tools(docs): class PipedreamOpenAPIAgent: - def __init__(self, docs, templates): + def __init__(self, docs, templates, auth_example): + system_instructions = format_template( + f"{templates.system_instructions(auth_example)}\n{docs_system_instructions}") + tools = OpenAPIExplorerTool.create_tools(docs) tool_names = [tool.name for tool in tools] + prompt_template = ZeroShotAgent.create_prompt( tools=tools, - prefix=format_template(templates.with_docs_system_instructions), - suffix=templates.suffix, - format_instructions=templates.format_instructions, + prefix=system_instructions, + suffix=suffix, + format_instructions=format_instructions, input_variables=['input', 'agent_scratchpad'] ) + llm_chain = LLMChain(llm=get_llm(), prompt=prompt_template) agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names) verbose = True if config['logging']['level'] == 'DEBUG' else False + self.agent_executor = AgentExecutor.from_agent_and_tools( agent=agent, tools=tools, verbose=verbose) @@ -66,26 +75,27 @@ def format_result(result): def get_llm(): if config['openai_api_type'] == "azure": azure_config = config["azure"] - llm = AzureChatOpenAI(deployment_name=azure_config['deployment_name'], - model_name=azure_config["model"], temperature=config["temperature"], request_timeout=300) + return AzureChatOpenAI(deployment_name=azure_config['deployment_name'], + model_name=azure_config["model"], temperature=config["temperature"], request_timeout=300) else: openai_config = config["openai"] - llm = ChatOpenAI( + return ChatOpenAI( model_name=openai_config["model"], temperature=config["temperature"], request_timeout=300) - return llm -def ask_agent(user_prompt, docs, templates): - agent = PipedreamOpenAPIAgent(docs, templates) +def ask_agent(user_prompt, docs, templates, auth_example): + agent = PipedreamOpenAPIAgent(docs, templates, auth_example) result = agent.run(user_prompt) return result -def no_docs(app, prompt, templates): +def no_docs(app, prompt, templates, auth_example): + user_prompt = f"{prompt}.The app is {app}." + system_instructions = format_template(templates.system_instructions(auth_example)) + result = get_llm()(messages=[ - SystemMessage(content=format_template( - templates.no_docs_system_instructions)), - HumanMessage(content=templates.no_docs_user_prompt % (prompt, app)), + SystemMessage(content=system_instructions), + HumanMessage(content=user_prompt), ]) return format_result(result.content) diff --git a/packages/component_code_gen/templates/actions/additional_rules.py b/packages/component_code_gen/templates/actions/additional_rules.py new file mode 100644 index 0000000000000..9b3a8e1d2ece1 --- /dev/null +++ b/packages/component_code_gen/templates/actions/additional_rules.py @@ -0,0 +1,32 @@ +additional_rules = """## Additional rules for actions + +1. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it. + +2. Always use this signature for the run method: + +async run({steps, $}) { + // your code here +} + +Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. + +9. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs. + +For example, do this: + +const response = await axios(this, { + url: `https://api.stability.ai/v1/engines/list`, + headers: { + Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, + }, +}); +// process the response data. response.data is undefined + +not this: + +const { data } = await axios(this, { + url: `https://api.stability.ai/v1/engines/list`, + headers: { + Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, + }, +});""" diff --git a/packages/component_code_gen/templates/actions/export_summary.py b/packages/component_code_gen/templates/actions/export_summary.py new file mode 100644 index 0000000000000..f73c4fa5c9d05 --- /dev/null +++ b/packages/component_code_gen/templates/actions/export_summary.py @@ -0,0 +1 @@ +export_summary = """A short summary should be exported before the end so that the user can quickly read what has happened. This is done by calling `$.export("$summary", "Your summary here")`. This is not optional.""" diff --git a/packages/component_code_gen/templates/actions/introduction.py b/packages/component_code_gen/templates/actions/introduction.py new file mode 100644 index 0000000000000..e50056635d2f5 --- /dev/null +++ b/packages/component_code_gen/templates/actions/introduction.py @@ -0,0 +1,12 @@ +# Experimenting with prompt from an AI researcher: https://twitter.com/jeremyphoward/status/1689464589191454720 +introduction = """## Instructions + +You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. + +Your goal is to create Pipedream Action Components. Your code should solve the requirements provided below. + +Other GPT agents will be reviewing your work, and will provide feedback on your code. You will be rewarded for code that is accurate, factual, thoughtful, nuanced, and solves the requirements provided in the instructions. + +## Pipedream components + +All Pipedream components are Node.js modules that have a default export: `defineComponent`. `defineComponent` is provided to the environment as a global — you do not need to import `defineComponent`. `defineComponent` is a function that takes an object — a Pipedream component — as its single argument.""" diff --git a/packages/component_code_gen/templates/actions/main_example.py b/packages/component_code_gen/templates/actions/main_example.py new file mode 100644 index 0000000000000..aa1b59b6c7c02 --- /dev/null +++ b/packages/component_code_gen/templates/actions/main_example.py @@ -0,0 +1,28 @@ +main_example = """Here's an example component: + +```javascript +import { axios } from "@pipedream/platform" +export default defineComponent({ + key: "openai-list-models", + name: "List Models", + description: "Lists all models available to the user.", + version: "0.0.{{ts}}", + type: "action", + props: { + openai: { + type: "app", + app: "openai", + } + }, + async run({steps, $}) { + const response = await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, + }) + $.export("$summary", "Successfully listed models") + return response + }, +}) +```""" diff --git a/packages/component_code_gen/templates/actions/other_example.py b/packages/component_code_gen/templates/actions/other_example.py new file mode 100644 index 0000000000000..a1b4f92874901 --- /dev/null +++ b/packages/component_code_gen/templates/actions/other_example.py @@ -0,0 +1,69 @@ +other_example = """Here's an example Pipedream component that makes a test request against the Slack API: + +export default defineComponent({ + key: "slack-send-message", + name: "Send Message", + version: "0.0.{{ts}}", + description: "Sends a message to a channel. [See docs here]()", + type: "action", + props: { + slack: { + type: "app", + app: "slack", + }, + channel: { + type: "string", + label: "Channel", + description: "The channel to post the message to", + }, + text: { + type: "string", + label: "Message Text", + description: "The text of the message to post", + }, + }, + async run({ steps, $ }) { + const response = await axios($, { + method: "POST", + url: `https://slack.com/api/chat.postMessage`, + headers: { + Authorization: `Bearer ${this.slack.$auth.oauth_access_token}`, + }, + data: { + channel: this.channel, + text: this.text, + }, + }) + $.export("$summary", "Sent message successfully") + return response + }, +}); + +Notice this section: + +data: { + channel: this.channel, + text: this.text, +}, + +This shows you how to pass the values of props (e.g. this.channel and this.text) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See rule #2 below for more detail. + +The code you generate should be placed within the `run` method of the Pipedream component: + +import { axios } from "@pipedream/platform"; + +export default defineComponent({ + props: { + the_app_name_slug: { + type: "app", + app: "the_app_name_slug", + }, + }, + async run({ steps, $ }) { + const response = await axios($, { + // Add the axios configuration object to make the HTTP request here + }) + $.export("$summary", "Your summary here") + return response + }, +});""" diff --git a/packages/component_code_gen/templates/apps/additional_rules.py b/packages/component_code_gen/templates/apps/additional_rules.py new file mode 100644 index 0000000000000..d34ace201e68d --- /dev/null +++ b/packages/component_code_gen/templates/apps/additional_rules.py @@ -0,0 +1 @@ +additional_rules = """""" diff --git a/packages/component_code_gen/templates/apps/auth.py b/packages/component_code_gen/templates/apps/auth.py new file mode 100644 index 0000000000000..09614f248dba6 --- /dev/null +++ b/packages/component_code_gen/templates/apps/auth.py @@ -0,0 +1,7 @@ +auth = """This lets the user connect their app account to the step, authorizing requests to the app API. + +`this` exposes the user's app credentials in the object `this.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header.""" diff --git a/packages/component_code_gen/templates/apps/introduction.py b/packages/component_code_gen/templates/apps/introduction.py new file mode 100644 index 0000000000000..68f72ae8fb1f4 --- /dev/null +++ b/packages/component_code_gen/templates/apps/introduction.py @@ -0,0 +1,6 @@ +introduction = """You are an agent designed to create Pipedream App Code. + +## Pipedream Apps + +All Pipedream apps are Node.js modules that have a default export: a javascript object - a Pipedream app - as its single argument. +It is essentially a wrapper on an API that requires authentication. Pipedream facades the authentication data in an object accessed by `this.$auth`. All app objects have three four keys: type, app, propDefinitions, and methods. The app object contains a `type` property, which is always set to "app". The `app` property is the name of the app, e.g. "google_sheets". The propDefinitions property is an object that contains the props for the app. The methods property is an object that contains the methods for the app.""" diff --git a/packages/component_code_gen/templates/apps/main_example.py b/packages/component_code_gen/templates/apps/main_example.py new file mode 100644 index 0000000000000..cc51767db49c3 --- /dev/null +++ b/packages/component_code_gen/templates/apps/main_example.py @@ -0,0 +1,136 @@ +main_example = """Here's an example Pipedream app for Raindrop: + +```javascript +import { axios } from "@pipedream/platform"; + +export default { + type: "app", + app: "raindrop", + propDefinitions: { + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + raindropId: { + type: "string", + label: "Bookmark ID", + description: "Existing Bookmark ID", + async options({ + prevContext, collectionId, + }) { + const page = prevContext.page + ? prevContext.page + : 0; + const { items } = await this.getRaindrops(this, collectionId, { + page, + }); + return { + options: items.map((e) => ({ + value: e._id, + label: e.title, + })), + context: { + page: page + 1, + }, + }; + }, + }, + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + method, + url: `https://api.raindrop.io/rest/v1${path}`, + headers: { + ...opts.headers, + "user-agent": "@PipedreamHQ/pipedream v0.1", + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, + }, + data, + params, + }); + }, + async getCollections($) { + return this._makeRequest($, { + path: "/collections", + }); + }, + async getRaindrops($, collectionId, params) { + return this._makeRequest($, { + path: `/raindrops/${collectionId}`, + params, + }); + }, + }, +}; +``` + +This object contains a `propDefinitions` property, which contains the definitions for the props of the app. + +The propDefinitions object contains two props: collectionId and raindropId. The collectionId prop is a string prop. The raindropId prop is also a string prop. The propDefinitions object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options. + +This object contains a `props` property, which defines a single prop of type "app": + +```javascript +import { axios } from "@pipedream/platform"; +export default { + type: "app", + app: "the_app_name", + propDefinitions: { + prop_key: { + type: "string", + label: "Prop Label", + description: "A description of the prop", + async options() { + const static_options = ["option 1", "option 2"]; // a static array of options + const dynamic_options = await this.getOptions(); // a Promise that resolves to an array of options. + return dynamic_options; // return the options + }, + } + }, + methods: { + async _makeRequest($ = this, opts) { + const { + method = "get", + path, + data, + params, + ...otherOpts + } = opts; + return await axios($, { + ...otherOpts, + method, + url: `https://api.the_app_name.com${path}`, // the base URL of the app API + headers: { + ...opts.headers, + "Authorization": `Bearer ${this.$auth.oauth_access_token}`, // the authentication type depends on the app + }, + params, + data, + }) + }, + async getOptions() { + // the code to get the options + return await this._makeRequest({ + ...opts, + }) + }, + }, +} +```""" diff --git a/packages/component_code_gen/templates/apps/methods.py b/packages/component_code_gen/templates/apps/methods.py new file mode 100644 index 0000000000000..7dadb17a811be --- /dev/null +++ b/packages/component_code_gen/templates/apps/methods.py @@ -0,0 +1,9 @@ +methods = """## Methods + +The `methods` property contains auxiliary methods. A `async _makeRequest` method is always required. It contains the code that makes the API request. It takes two arguments: `$ = this` and `opts`. `$` is the context passed by the Pipedream runtime. It should default to `this`. `opts` is an object that contains the parameters of the API request. The `opts` object may contain the following fields: `method`, `path`, `data`, `params`, and `headers`. The `method` field is the HTTP method of the request. The `path` field is the path of the request. The `data` field is the body of the request. The `params` field is the query parameters of the request. The `headers` field is the headers of the request. The `opts` object also contains any other fields that are passed to the `_makeRequest` method. The `_makeRequest` method returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. + +The axios request uses the authentication method defined by the app. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. The axios request should use the format from the app docs. + +Auxiliary methods, usually for CRUD operations call `_makeRequest` with the appropriate parameters. Please add a few methods for common operations, e.g. get and list. You can also add other methods that you think are useful. + +For listing operations, verify how the pagination is done in the API. Also add a method for pagination. This method should be named `paginate`, and the arguments are `fn`, the listing method that will be called, and `...opts`, the parameters of the HTTP request. The method starts with an empty array, and calls the listing method with the parameters. It then checks the response and verifies if there is more data. If it does, it calls itself with the listing method and the parameters for fetching the next set of data. If it doesn't, it returns the array of results.""" diff --git a/packages/component_code_gen/templates/apps/prop_definitions.py b/packages/component_code_gen/templates/apps/prop_definitions.py new file mode 100644 index 0000000000000..2ecac5b83466e --- /dev/null +++ b/packages/component_code_gen/templates/apps/prop_definitions.py @@ -0,0 +1,3 @@ +prop_definitions = """## Prop Definitions + +The app code should contain a `propDefinitions` property, which are the definitions for the props. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method.""" diff --git a/packages/component_code_gen/templates/common/app_prop.py b/packages/component_code_gen/templates/common/app_prop.py new file mode 100644 index 0000000000000..d839a4724b155 --- /dev/null +++ b/packages/component_code_gen/templates/common/app_prop.py @@ -0,0 +1,15 @@ +app_prop = """This object contains a `props` property, which defines a single prop of type "app": + +```javascript +export default { + props: { + the_app_name: { + type: "app", + app: "the_app_name", + }, + } + // the rest of the component ... +}) +``` + +This lets the user connect their app account to the step, authorizing requests to the app API.""" diff --git a/packages/component_code_gen/templates/common/async_options.py b/packages/component_code_gen/templates/common/async_options.py new file mode 100644 index 0000000000000..d77c91857fc5b --- /dev/null +++ b/packages/component_code_gen/templates/common/async_options.py @@ -0,0 +1,63 @@ +async_options = """## Async options + +The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: + +``` +[ + { + label: "Human-readable option 1", + value: "unique identifier 1", + }, + { + label: "Human-readable option 2", + value: "unique identifier 2", + }, +] +``` + +The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. + +If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. + +Example async options methods: + +``` +msg: { + type: "string", + label: "Message", + description: "Select a message to `console.log()`", + async options() { + // write any node code that returns a string[] (with label/value keys) + return ["This is option 1", "This is option 2"]; + }, +}, +``` + +``` +board: { + type: "string", + label: "Board", + async options(opts) { + const boards = await this.getBoards(this.$auth.oauth_uid); + const activeBoards = boards.filter((board) => board.closed === false); + return activeBoards.map((board) => { + return { label: board.name, value: board.id }; + }); + }, +}, +``` + +``` +async options(opts) { + const response = await axios(this, { + method: "GET", + url: `https://api.spotify.com/v1/me/playlists`, + headers: { + Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, + }, + }); + return response.items.map((playlist) => { + return { label: playlist.name, value: playlist.id }; + }); +}, +```""" diff --git a/packages/component_code_gen/templates/common/auth.py b/packages/component_code_gen/templates/common/auth.py new file mode 100644 index 0000000000000..af7d3ed864a6f --- /dev/null +++ b/packages/component_code_gen/templates/common/auth.py @@ -0,0 +1,7 @@ +auth = """## Authentication + +Within the `run` method, `this` exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the `$auth` object contains properties for each key / token the user enters. For OAuth integrations, `this` object exposes the OAuth access token in the `oauth_access_token` property of the `$auth` object. + +The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. + +The app can also be an OAuth app. For OAuth integrations, `this` object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header.""" diff --git a/packages/component_code_gen/templates/common/component_metadata.py b/packages/component_code_gen/templates/common/component_metadata.py new file mode 100644 index 0000000000000..638b3201d89f7 --- /dev/null +++ b/packages/component_code_gen/templates/common/component_metadata.py @@ -0,0 +1,49 @@ +# ---------------------------- action example ---------------------------- # +action_example = """``` +export default { + key: "google_drive-list-all-drives", + name: "List All Drives", + description: "Lists all drives in an account.", + version: "0.0.{{ts}}", + type: "action", +}; +```""" + + +# ---------------------------- source example ---------------------------- # +source_example = """``` +export default { + key: "google_drive-new-shared-drive-created", + name: "New Shared Drive Created", + description: "Emits a new event any time a shared drive is created.", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", +}; +```""" + + +# ---------------------------- component metadata ---------------------------- # +component_metadata = """## Component Metadata + +Registry components require a unique key and version, and a friendly name and description. E.g. + +{example} + +Component keys are in the format app_name_slug-slugified-component-name. +You should come up with a name and a description for the component you are generating. +In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). +Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). +Always add version "0.0.{{ts}}". +Always put {component_type}. +You MUST add metadata to the component code you generate.""" + + +# ---------------------------- action metadata ---------------------------- # +action_metadata = component_metadata.format( + example=action_example, component_type='"type": "action"') + + +# ---------------------------- source metadata ---------------------------- # +source_metadata = component_metadata.format( + example=source_example, component_type='"type": "source" and "dedupe": "unique"') diff --git a/packages/component_code_gen/templates/common/docs_system_instructions.py b/packages/component_code_gen/templates/common/docs_system_instructions.py new file mode 100644 index 0000000000000..fdbf30cbd2356 --- /dev/null +++ b/packages/component_code_gen/templates/common/docs_system_instructions.py @@ -0,0 +1,19 @@ +docs_system_instructions = """You are an agent designed to interact with an OpenAPI JSON specification. +You have access to the following tools which help you learn more about the JSON you are interacting with. +Only use the below tools. Only use the information returned by the below tools to construct your final answer. +Do not make up any information that is not contained in the JSON. +Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. +You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. +If you have not seen a key in one of those responses, you cannot use it. +You should only add one key at a time to the path. You cannot add multiple keys at once. +If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. + +Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. +Then you should proceed to search for the rest of the information to build your answer. + +If the question does not seem to be related to the JSON, just return "I don't know" as the answer. +Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. + +Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". +In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. +Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" diff --git a/packages/component_code_gen/templates/common/end.py b/packages/component_code_gen/templates/common/end.py new file mode 100644 index 0000000000000..1b5eff2d308c3 --- /dev/null +++ b/packages/component_code_gen/templates/common/end.py @@ -0,0 +1,6 @@ +end = """## Remember, return ONLY code + +Only return Node.js code. You produce Pipedream component code and ONLY Pipedream component code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. + +Consider all the instructions and rules above. +---""" diff --git a/packages/component_code_gen/templates/common/format_instructions.py b/packages/component_code_gen/templates/common/format_instructions.py new file mode 100644 index 0000000000000..5ed6d18405c45 --- /dev/null +++ b/packages/component_code_gen/templates/common/format_instructions.py @@ -0,0 +1,10 @@ +format_instructions = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do. always escape curly brackets +Action: the action to take, should be one of [{tool_names}] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can repeat N times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question. do not include any other text than the code itself""" diff --git a/packages/component_code_gen/templates/common/platform_axios.py b/packages/component_code_gen/templates/common/platform_axios.py new file mode 100644 index 0000000000000..7595d06a673ea --- /dev/null +++ b/packages/component_code_gen/templates/common/platform_axios.py @@ -0,0 +1,24 @@ +platform_axios = """## Pipedream Platform Axios + +If you need to make an HTTP request use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: + +import { axios } from "@pipedream/platform"; + +You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. + +The `axios` constructor takes two arguments: + +1. `this` - the context passed by the run method of the component. + +2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. + +For example: + +return await axios($, { + url: `https://api.openai.com/v1/models`, + headers: { + Authorization: `Bearer ${this.openai.$auth.api_key}`, + }, +}) + +`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property.""" diff --git a/packages/component_code_gen/templates/common/props.py b/packages/component_code_gen/templates/common/props.py new file mode 100644 index 0000000000000..c2c315dc89308 --- /dev/null +++ b/packages/component_code_gen/templates/common/props.py @@ -0,0 +1,5 @@ +props = """## Props + +The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any parameters of the API as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. + +Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties.""" diff --git a/packages/component_code_gen/templates/common/rules.py b/packages/component_code_gen/templates/common/rules.py new file mode 100644 index 0000000000000..b599e90538876 --- /dev/null +++ b/packages/component_code_gen/templates/common/rules.py @@ -0,0 +1,65 @@ +rules = """## Rules + +When you generate code, you must follow all of the rules above. Review the rules and think through them step-by-step before you generate code. Look at how these map to the example code and component API described above. + +Once you generate your code, you must review each of these rules again, one-by-one, and ensure you've followed them. Accuracy is critical, and we can wait for you to review your code. If you notice you haven't followed a particular rule, you can regenerate your code and start over. If you do make any edits, you'll need to again review each rule one-by-one to make sure your edits didn't conflict with another rule. I cannot stress how critical it is to follow all of the rules below. Consider it your constitution. + +1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. + +2. Include all parameters of the API request as props. DO NOT use example values from any API docs, OpenAPI specs, or example code above or that you've been trained on. + +For example, do this: + +``` +data: { + text_prompts: [ + { + text: this.textPrompt, + weight: this.weight, + }, + ], + cfg_scale: this.cfg_scale, + height: this.height, + width: this.width, + samples: this.samples, + steps: this.steps, +} +``` + +But never do this: + +``` +data: { + text_prompts: [ + { + text: this.textPrompt, + weight: 1, + }, + ], + cfg_scale: 7, + height: 512, + width: 512, + samples: 1, + steps: 75, +} +``` + +You can see that there's no static value present in the first example. In the second example, the values are hardcoded, so the user can't enter their own values. + +I need to reiterate: you MUST NOT use static, example values in the code. You MUST use the value of the prop (this.) instead. Think about it: if you hardcode values in the code, the user can't enter their own value. + +2b. Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. + +2c. You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. + +2d. Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. + +3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. + +4. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. + +5. Double-check the code against known Node.js examples, from GitHub and any other real code you find. + +6. You must pass a value of `0.0.{ts}` to the `version` property. This is the only valid version value. "{ts}" is expanded by the Pipedream platform to the current epoch ms timestamp. Think about it: if you pass a different value, the developer won't be able to republish the component with a dynamic version, and publishing will fail, which will waste their time. + +7. Remember, please do not pass example values from the API docs or OpenAPI spec. You must pass the value of the prop to all params instead. This is the only way the user can enter their own values.""" diff --git a/packages/component_code_gen/templates/common/suffix.py b/packages/component_code_gen/templates/common/suffix.py new file mode 100644 index 0000000000000..ce1e0d45f1bec --- /dev/null +++ b/packages/component_code_gen/templates/common/suffix.py @@ -0,0 +1,3 @@ +suffix = """Begin! +Question: {input} +{agent_scratchpad}""" diff --git a/packages/component_code_gen/templates/common/typescript_definitions.py b/packages/component_code_gen/templates/common/typescript_definitions.py new file mode 100644 index 0000000000000..12e5a45f12aa9 --- /dev/null +++ b/packages/component_code_gen/templates/common/typescript_definitions.py @@ -0,0 +1,152 @@ +typescript_definitions = """## TypeScript Definitions + +export interface Methods { + [key: string]: (...args: any) => unknown; +} + +// $.flow.exit() and $.flow.delay() +export interface FlowFunctions { + exit: (reason: string) => void; + delay: (ms: number) => { + resume_url: string; + cancel_url: string; + }; +} + +export interface Pipedream { + export: (key: string, value: JSONValue) => void; + send: SendFunctionsWrapper; + /** + * Respond to an HTTP interface. + * @param response Define the status and body of the request. + * @returns A promise that is fulfilled when the body is read or an immediate response is issued + */ + respond: (response: HTTPResponse) => Promise | void; + flow: FlowFunctions; +} + +// Arguments to the options method for props +export interface OptionsMethodArgs { + page?: number; + prevContext?: any; + [key: string]: any; +} + +// You can reference the values of previously-configured props! +export interface OptionalOptsFn { + (configuredProps: { [key: string]: any; }): object; +} + +export type PropDefinition = + [App, string] | + [App, string, OptionalOptsFn]; + +// You can reference props defined in app methods, referencing the propDefintion directly in props +export interface PropDefinitionReference { + propDefinition: PropDefinition; +} + +export interface App< + Methods, + AppPropDefinitions +> { + type: "app"; + app: string; + propDefinitions?: AppPropDefinitions; + methods?: Methods & ThisType; +} + +export function defineApp< + Methods, + AppPropDefinitions, +> +(app: App): App { + return app; +} + +// Props + +export interface DefaultConfig { + intervalSeconds?: number; + cron?: string; +} + +export interface Field { + name: string; + value: string; +} + +export interface BasePropInterface { + label?: string; + description?: string; +} + +export type PropOptions = any[] | Array<{ [key: string]: string; }>; + +export interface UserProp extends BasePropInterface { + type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; + options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); + optional?: boolean; + default?: JSONValue; + secret?: boolean; + min?: number; + max?: number; +} + +export interface InterfaceProp extends BasePropInterface { + type: "$.interface.http" | "$.interface.timer"; + default?: string | DefaultConfig; +} + +// When users ask about data stores, remember to include a prop of type "data_store" in the props object +export interface DataStoreProp extends BasePropInterface { + type: "data_store"; +} + +export interface HttpRequestProp extends BasePropInterface { + type: "http_request"; + default?: DefaultHttpRequestPropConfig; +} + +export interface ActionPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; +} + +export interface AppPropDefinitions { + [name: string]: PropDefinitionReference | App | UserProp; +} + +export interface ActionRunOptions { + $: Pipedream; + steps: JSONValue; +} + +type PropThis = { + [Prop in keyof Props]: Props[Prop] extends App ? any : any +}; + +export interface Action< + Methods, + ActionPropDefinitions +> { + key: string; + name?: string; + description?: string; + version: string; + type: "action"; + methods?: Methods & ThisType & Methods>; + props?: ActionPropDefinitions; + additionalProps?: ( + previousPropDefs: ActionPropDefinitions + ) => Promise; + run: (this: PropThis & Methods, options?: ActionRunOptions) => any; +} + +export function defineAction< + Methods, + ActionPropDefinitions, +> +(component: Action): Action { + return component; +} +""" diff --git a/packages/component_code_gen/templates/generate_actions.py b/packages/component_code_gen/templates/generate_actions.py index ab8736d71f404..00388206c8070 100644 --- a/packages/component_code_gen/templates/generate_actions.py +++ b/packages/component_code_gen/templates/generate_actions.py @@ -1,552 +1,45 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.actions.additional_rules import additional_rules +from templates.actions.export_summary import export_summary +from templates.actions.introduction import introduction +from templates.actions.main_example import main_example +from templates.actions.other_example import other_example +from templates.common.app_prop import app_prop +from templates.common.auth import auth +from templates.common.component_metadata import action_metadata +from templates.common.platform_axios import platform_axios +from templates.common.props import props +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -# Experimenting with prompt from an AI researcher: https://twitter.com/jeremyphoward/status/1689464589191454720 -no_docs_system_instructions = """## Instructions +def system_instructions(auth_example=""): + return f"""{introduction} -You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF. You carefully provide accurate, factual, thoughtful, nuanced code, and are brilliant at reasoning. +{main_example} -Your goal is to create Pipedream Action Components. Your code should solve the requirements provided below. +{app_prop} -Other GPT agents will be reviewing your work, and will provide feedback on your code. You will be rewarded for code that is accurate, factual, thoughtful, nuanced, and solves the requirements provided in the instructions. +{auth} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{auth_example} -## Pipedream components +{props} -All Pipedream components are Node.js modules that have a default export: `defineComponent`. `defineComponent` is provided to the environment as a global — you do not need to import `defineComponent`. `defineComponent` is a function that takes an object — a Pipedream component — as its single argument. +{export_summary} -Here's an example component: +{platform_axios} -```javascript -import { axios } from "@pipedream/platform" -export default defineComponent({ - key: "openai-list-models", - name: "List Models", - description: "Lists all models available to the user.", - version: "0.0.{{ts}}", - type: "action", - props: { - openai: { - type: "app", - app: "openai", - } - }, - async run({steps, $}) { - const response = await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) - $.export("$summary", "Successfully listed models") - return response - }, -}) -``` +{other_example} -This object contains a `props` property, which defines a single prop of type "app": +{async_options} -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` +{action_metadata} -This lets the user connect their app account to the step, authorizing requests to the app API. +{typescript_definitions} -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. +{rules} -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. +{additional_rules} -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any parameters of the API as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. - -A short summary should be exported before the end so that the user can quickly read what has happened. This is done by calling `$.export("$summary", "Your summary here")`. This is not optional. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `$` - the `$` context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -Here's an example Pipedream component that makes a test request against the Slack API: - -export default defineComponent({ - key: "slack-send-message", - name: "Send Message", - version: "0.0.{{ts}}", - description: "Sends a message to a channel. [See docs here]()", - type: "action", - props: { - slack: { - type: "app", - app: "slack", - }, - channel: { - type: "string", - label: "Channel", - description: "The channel to post the message to", - }, - text: { - type: "string", - label: "Message Text", - description: "The text of the message to post", - }, - }, - async run({ steps, $ }) { - const response = await axios($, { - method: "POST", - url: `https://slack.com/api/chat.postMessage`, - headers: { - Authorization: `Bearer ${this.slack.$auth.oauth_access_token}`, - }, - data: { - channel: this.channel, - text: this.text, - }, - }) - $.export("$summary", "Sent message successfully") - return response - }, -}); - -Notice this section: - -data: { - channel: this.channel, - text: this.text, -}, - -This shows you how to pass the values of props (e.g. this.channel and this.text) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See rule #2 below for more detail. - -The code you generate should be placed within the `run` method of the Pipedream component: - -import { axios } from "@pipedream/platform"; - -export default defineComponent({ - props: { - the_app_name_slug: { - type: "app", - app: "the_app_name_slug", - }, - }, - async run({ steps, $ }) { - const response = await axios($, { - // Add the axios configuration object to make the HTTP request here - }) - $.export("$summary", "Your summary here") - return response - }, -}); - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-list-all-drives", - name: "List All Drives", - description: "Lists all drives in an account.", - version: "0.0.{{ts}}", - type: "action", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Action keys should use active verbs to describe the action that will occur, (e.g., linear_app-create-issue). -Always add version "0.0.{{ts}}" and type "action". -You MUST add metadata to the component code you generate. - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -When you generate code, you must follow all of the rules above. Review the rules and think through them step-by-step before you generate code. Look at how these map to the example code and component API described above. - -Once you generate your code, you must review each of these rules again, one-by-one, and ensure you've followed them. Accuracy is critical, and we can wait for you to review your code. If you notice you haven't followed a particular rule, you can regenerate your code and start over. If you do make any edits, you'll need to again review each rule one-by-one to make sure your edits didn't conflict with another rule. I cannot stress how critical it is to follow all of the rules below. Consider it your constitution. - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above the `defineComponent` call. - -2. Include all parameters of the API request as props. DO NOT use example values from any API docs, OpenAPI specs, or example code above or that you've been trained on. - -For example, do this: - -``` -data: { - text_prompts: [ - { - text: this.textPrompt, - weight: this.weight, - }, - ], - cfg_scale: this.cfg_scale, - height: this.height, - width: this.width, - samples: this.samples, - steps: this.steps, -} -``` - -But never do this: - -``` -data: { - text_prompts: [ - { - text: this.textPrompt, - weight: 1, - }, - ], - cfg_scale: 7, - height: 512, - width: 512, - samples: 1, - steps: 75, -} -``` - -You can see that there's no static value present in the first example. In the second example, the values are hardcoded, so the user can't enter their own values. - -I need to reiterate: you MUST NOT use static, example values in the code. You MUST use the value of the prop (this.) instead. Think about it: if you hardcode values in the code, the user can't enter their own value. - -2b. Optional inputs should include `"optional": true` in the prop declaration. The default is `"optional": false`, so please do not include this for required inputs. The API docs and OpenAPI spec should specify what inputs are required. - -2c. You should understand what props map to the request path, headers, query string params, and the request body. Pass the value of the prop (this.) in the appropriate place in the request: the path, `headers`, `params`, or `data` (respectively) properties of the `axios` request. - -2d. Map the types of inputs in the API spec to the correct prop types. Look closely at each param of the API docs, double-checking the final code to make sure each param is included as a prop and not passed as a static value to the API like you may have seen as examples. Values of props should _always_ reference this.. Think about it — the user will need to enter these values as props, so they can't be hardcoded. - -3. If you produce output files, or if a library produces output files, you MUST write files to the /tmp directory. You MUST NOT write files to `./` or any relative directory. `/tmp` is the only writable directory you have access to. - -4. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -5. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -6. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it. - -7. Always use this signature for the run method: - -async run({steps, $}) { - // your code here -} - -Always pass {steps, $}, even if you don't use them in the code. Think about it: the user needs to access the steps and $ context when they edit the code. - -8. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs. - -For example, do this: - -const response = await axios(this, { - url: `https://api.stability.ai/v1/engines/list`, - headers: { - Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, - }, -}); -// process the response data. response.data is undefined - -not this: - -const { data } = await axios(this, { - url: `https://api.stability.ai/v1/engines/list`, - headers: { - Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`, - }, -}); - -9. You must pass a value of `0.0.{{ts}}` to the `version` property. This is the only valid version value. "{{ts}}" is expanded by the Pipedream platform to the current epoch ms timestamp. Think about it: if you pass a different value, the developer won't be able to republish the component with a dynamic version, and publishing will fail, which will waste their time. - -10. Remember, please do not pass example values from the API docs or OpenAPI spec. You must pass the value of the prop to all params instead. This is the only way the user can enter their own values. - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} - -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/generate_apps.py b/packages/component_code_gen/templates/generate_apps.py index 65b29c4392da5..306ffb856935c 100644 --- a/packages/component_code_gen/templates/generate_apps.py +++ b/packages/component_code_gen/templates/generate_apps.py @@ -1,466 +1,36 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.apps.additional_rules import additional_rules +from templates.apps.auth import auth +from templates.apps.introduction import introduction +from templates.apps.main_example import main_example +from templates.apps.methods import methods +from templates.apps.prop_definitions import prop_definitions +from templates.common.platform_axios import platform_axios +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -no_docs_system_instructions = """You are an agent designed to create Pipedream App Code. +def system_instructions(auth_example=""): + return f"""{introduction} -You will receive a prompt from an user. You should create a code in Node.js using Pipedream axios for HTTP requests. Your goal is to create a Pipedream App Code. -You should not return any text other than the code. +{main_example} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{auth} -## Pipedream Apps +{auth_example} -All Pipedream apps are Node.js modules that have a default export: an javascript object - a Pipedream app - as its single argument. -It is essentially a wrapper on an API that requires authentication. Pipedream facades the authentication data in an object accessed by `this.$auth`. All app objects have three four keys: type, app, propDefinitions, and methods. The app object contains a `type` property, which is always set to "app". The `app` property is the name of the app, e.g. "google_sheets". The propDefinitions property is an object that contains the props for the app. The methods property is an object that contains the methods for the app. +{prop_definitions} -Here's an example Pipedream app for Raindrop: +{methods} -```javascript -import { axios } from "@pipedream/platform"; +{platform_axios} -export default { - type: "app", - app: "raindrop", - propDefinitions: { - collectionId: { - type: "string", - label: "Collection ID", - description: "The collection ID", - async options() { - const { items } = await this.getCollections(); - return items.map((e) => ({ - value: e._id, - label: e.title, - })); - }, - }, - raindropId: { - type: "string", - label: "Bookmark ID", - description: "Existing Bookmark ID", - async options({ - prevContext, collectionId, - }) { - const page = prevContext.page - ? prevContext.page - : 0; - const { items } = await this.getRaindrops(this, collectionId, { - page, - }); - return { - options: items.map((e) => ({ - value: e._id, - label: e.title, - })), - context: { - page: page + 1, - }, - }; - }, - }, - }, - methods: { - async _makeRequest($ = this, opts) { - const { - method = "get", - path, - data, - params, - ...otherOpts - } = opts; - return axios($, { - ...otherOpts, - method, - url: `https://api.raindrop.io/rest/v1${path}`, - headers: { - ...opts.headers, - "user-agent": "@PipedreamHQ/pipedream v0.1", - "Authorization": `Bearer ${this.$auth.oauth_access_token}`, - }, - data, - params, - }); - }, - async getCollections($) { - return this._makeRequest($, { - path: "/collections", - }); - }, - async getRaindrops($, collectionId, params) { - return this._makeRequest($, { - path: `/raindrops/${collectionId}`, - params, - }); - }, - }, -}; -``` +{async_options} -This object contains a `propDefinitions` property, which contains the definitions for the props of the app. +{typescript_definitions} -The propDefinitions object contains two props: collectionId and raindropId. The collectionId prop is a string prop. The raindropId prop is also a string prop. The propDefinitions object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options. +{rules} -This object contains a `props` property, which defines a single prop of type "app": +{additional_rules} -```javascript -import { axios } from "@pipedream/platform"; -export default { - type: "app", - app: "the_app_name", - propDefinitions: { - prop_key: { - type: "string", - label: "Prop Label", - description: "A description of the prop", - async options() { - const static_options = ["option 1", "option 2"]; // a static array of options - const dynamic_options = await this.getOptions(); // a Promise that resolves to an array of options. - return dynamic_options; // return the options - }, - } - }, - methods: { - async _makeRequest($ = this, opts) { - const { - method = "get", - path, - data, - params, - ...otherOpts - } = opts; - return await axios($, { - ...otherOpts, - method, - url: `https://api.the_app_name.com${path}`, // the base URL of the app API - headers: { - ...opts.headers, - "Authorization": `Bearer ${this.$auth.oauth_access_token}`, // the authentication type depends on the app - }, - params, - data, - }) - }, - async getOptions() { - // the code to get the options - return await this._makeRequest({ - ...opts, - }) - }, - }, -} -``` - -This lets the user connect their app account to the step, authorizing requests to the app API. - -`this` exposes the user's app credentials in the object `this.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. - -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. - -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The app code should contain a `propDefinitions` property, which are the definitions for the props. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -The `methods` property contains auxiliary methods. A `async _makeRequest` method is always required. It contains the code that makes the API request. It takes two arguments: `$ = this` and `opts`. `$` is the context passed by the Pipedream runtime. It should default to `this`. `opts` is an object that contains the parameters of the API request. The `opts` object may contain the following fields: `method`, `path`, `data`, `params`, and `headers`. The `method` field is the HTTP method of the request. The `path` field is the path of the request. The `data` field is the body of the request. The `params` field is the query parameters of the request. The `headers` field is the headers of the request. The `opts` object also contains any other fields that are passed to the `_makeRequest` method. The `_makeRequest` method returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -The axios request uses the authentication method defined by the app. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. The axios request should use the format from the app docs. - -Auxiliary methods, usually for CRUD operations call `_makeRequest` with the appropriate parameters. Please add a few methods for common operations, e.g. get and list. You can also add other methods that you think are useful. - -For listing operations, verify how the pagination is done in the API. Also add a method for pagination. This method should be named `paginate`, and the arguments are `fn`, the listing method that will be called, and `...opts`, the parameters of the HTTP request. The method starts with an empty array, and calls the listing method with the parameters. It then checks the response and verifies if there is more data. If it does, it calls itself with the listing method and the parameters for fetching the next set of data. If it doesn't, it returns the array of results. - -## Pipedream Platform Axios - -Always use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `this` - the context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, -}) - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -## Async options - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/generate_polling_sources.py b/packages/component_code_gen/templates/generate_polling_sources.py index 7f368c31a44c0..a58463ce85aee 100644 --- a/packages/component_code_gen/templates/generate_polling_sources.py +++ b/packages/component_code_gen/templates/generate_polling_sources.py @@ -1,482 +1,45 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.sources.db import db +from templates.sources.polling.additional_rules import additional_rules +from templates.sources.polling.hooks import hooks +from templates.sources.polling.introduction import introduction +from templates.sources.polling.main_example import main_example +from templates.common.app_prop import app_prop +from templates.common.auth import auth +from templates.common.component_metadata import source_metadata +from templates.common.platform_axios import platform_axios +from templates.common.props import props +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -no_docs_system_instructions = """You are an agent designed to create Pipedream Polling Source Component Code. +def system_instructions(auth_example=""): + return f"""{introduction} -You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Polling Source Component Code. -You should not return any text other than the code. +{main_example} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{app_prop} -## Pipedream Source Components +{auth} -All Pipedream polling source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument. +{auth_example} -Here's an example Pipedream source component that fetches all bookmarks from Raindrop.io and emits each bookmark as an event: +{props} -```javascript -import { axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform" -export default { - key: "raindrop-bookmark-created", - name: "New Bookmark Created", - description: `Emit new event when a bookmark is created. [See the documentation](${docsLink})`, - version: "0.0.1", - type: "source", - dedupe: "unique", - props: { - raindrop: { - type: "app", - app: "raindrop", - }, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - }, - collectionId: { - type: "string", - label: "Collection ID", - description: "The collection ID", - async options() { - // fetch collections from the API - const { items } = await this.getCollections(); - return items.map((e) => ({ - value: e._id, - label: e.title, - })); - }, - }, - }, - methods: { - _getPage() { - return this.db.get("page") ?? 0; - }, - _setPage(page) { - this.db.set("page", page); - }, - }, - hooks: { - async deploy() { - let page = 0; - const all_bookmarks = []; +{db} - while (true) { - // fetch bookmarks from the API - const bookmarks = await this.raindrop.getRaindrops(this, this.collectionId, { - page, - perpage: MAX_PER_PAGE, - }); - all_bookmarks.unshift(...bookmarks.reverse()); +{hooks} - if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; - page++; - } +{platform_axios} - for (const bookmark of all_bookmarks.slice(0, 50)) { - this.$emit(bookmark, { - id: bookmark._id, - summary: `New Raindrop: ${bookmark.title}`, - ts: Date.parse(bookmark.created), - }); - } +{async_options} - this._setPage(page); - }, - }, - async run() { - let page = this._getPage(); +{source_metadata} - while (true) { - // fetch bookmarks from the API - const { items: bookmarks } = await this.raindrop.getRaindrops(this, this.collectionId, { - page, - perpage: 50, - }); +{typescript_definitions} - for (const bookmark of bookmarks) { - this.$emit(bookmark, { - id: bookmark._id, - summary: `New Raindrop: ${bookmark.title}`, - ts: Date.parse(bookmark.created), - }); - }; +{rules} - if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; +{additional_rules} - page++; - } - - this._setPage(page); - }, -}; -``` - -This object contains a `props` property, which defines a single prop of type "app": - -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` - -This lets the user connect their app account to the step, authorizing requests to the app API. - -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. - -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. - -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. - -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. - -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. - -The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event. - -There are also two other props in sources: `http` and `db`. `http` is an interface that lets you receive and respond to HTTP requests. `db` is a data store that lets you store data between runs of the component. You should always include both. - -The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". - -Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. - -The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. - -## Source Hooks - -Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `this` - the context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-new-shared-drive-created", - name: "New Shared Drive Created", - description: "Emits a new event any time a shared drive is created.", - version: "0.0.1", - type: "source", - dedupe: "unique", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). -Always add version "0.0.1", type "source", and dedupe "unique". - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -5. Always emit relevant data. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. - -6. Always use this signature for the run method: - -async run() { - // your code here -} - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/generate_webhook_sources.py b/packages/component_code_gen/templates/generate_webhook_sources.py index fdb4d16d44417..dbd1d6f4c86e8 100644 --- a/packages/component_code_gen/templates/generate_webhook_sources.py +++ b/packages/component_code_gen/templates/generate_webhook_sources.py @@ -1,508 +1,54 @@ -no_docs_user_prompt = """%s. The app is %s.""" +from templates.sources.db import db +from templates.sources.webhooks.async_run import async_run +from templates.sources.webhooks.additional_rules import additional_rules +from templates.sources.webhooks.hooks import hooks +from templates.sources.webhooks.http import http +from templates.sources.webhooks.introduction import introduction +from templates.sources.webhooks.main_example import main_example +from templates.sources.webhooks.other_example import other_example +from templates.common.app_prop import app_prop +from templates.common.auth import auth +from templates.common.component_metadata import source_metadata +from templates.common.platform_axios import platform_axios +from templates.common.props import props +from templates.common.rules import rules +from templates.common.async_options import async_options +from templates.common.typescript_definitions import typescript_definitions +from templates.common.end import end -no_docs_system_instructions = """You are an agent designed to create Pipedream Webhooks Source Component Code. +def system_instructions(auth_example=""): + return f"""{introduction} -You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Webhooks Source Component Code, also called Pipedream Webhooks Trigger Code. -You should not return any text other than the code. +{main_example} -output: Node.js code and ONLY Node.js code. You produce Pipedream component code and ONLY Pipedream component code. You MUST NOT include English before or after code, and MUST NOT include Markdown (like ```javascript) surrounding the code. I just want the code! +{app_prop} -## Pipedream Source Components +{auth} -All Pipedream webhook source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument. +{auth_example} -Here's an example component: +{props} -```javascript -import crypto from "crypto" -import { axios } from "@pipedream/platform" -export default { - key: "github-new-notification-received", - name: "New Notification Received", - description: "Emit new event when a notification is received.", - version: "0.0.1", - type: "source", - dedupe: "unique", - props: { - github: { - type: "app", - app: "github", - }, - http: { - type: ""$.interface.http"", - customResponse: true, // optional: defaults to false - }, - db: "$.service.db", - }, - methods: { - _getWebhookId() { - return this.db.get("webhookId") - }, - _setWebhookId(id) { - this.db.set("webhookId", id) - }, - }, - hooks: { - async deploy() { - const events = await this.github.listMostRecentNotifications({paginate: true, max: 50}); - for (const event of events) { - this.$emit(event, { - id: event.id, - summary: `New event: ${event.name}`, - ts: Date.parse(event.ts), - }) - } - }, - async activate() { - const hookId = await this.createWebhook(opts) - this._setWebhookId(hookId) - }, - async deactivate() { - const id = this._getWebhookId() - await this.deleteWebhook(id) - }, - }, - async run(event) { - const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") - if (computedSignature !== webhookSignature) { - this.http.respond({ status: 401, body: "Unauthorized" }) - return - } - console.log(`Emitting event...`) - this.$emit(event, { - id: event.id, - summary: `New event: ${event.name}`, - ts: Date.parse(event.ts), - }) - }, -}; -``` +{async_run} -This object contains a `props` property, which defines a single prop of type "app": +{http} -```javascript -export default defineComponent({ - props: { - the_app_name: { - type: "app", - app: "the_app_name", - }, - } - // the rest of the component ... -}) -``` +{db} -This lets the user connect their app account to the step, authorizing requests to the app API. +{hooks} -Within the run method, this exposes the user's app credentials in the object `this.the_app_name_slug.$auth`. For integrations where users provide static API keys / tokens, the $auth object contains properties for each key / token the user enters. For OAuth integrations, this object exposes the OAuth access token in the oauth_access_token property of the $auth object. +{platform_axios} -The app can be a key-based app. For integrations where users provide static API keys / tokens, `this.the_app_name_slug.$auth` contains properties for each key / token the user enters. Users are asked to enter custom fields. They are each exposed as properties in the object `this.the_app_name_slug.$auth`. When you make the API request, use the format from the app docs. Different apps pass credentials in different places in the HTTP request, e.g. headers, url params, etc. +{async_options} -The app can also be an OAuth app. For OAuth integrations, this object exposes the OAuth access token in the variable `this.the_app_name_slug.$auth.oauth_access_token`. When you make the API request, make sure to use the format from the app docs, e.g. you may need to pass the OAuth access token as a Bearer token in the Authorization header. +{source_metadata} -The object _may_ contain an optional a `props` property, which in the example below defines a string prop. The props object is not required. Include it only if the code connects to a Pipedream integration, or the code in the run method requires input. Props lets the user pass data to the step via a form in the Pipedream UI, so they can fill in the values of the variables. Include any required parameters as properties of the `props` object. Props must include a human-readable `label` and a `type` (one of string|boolean|integer|object) that corresponds to the Node.js type of the required param. string, boolean, and integer props allow for arrays of input, and the array types are "string[]", "boolean[]", and "integer[]" respectively. Complex props (like arrays of objects) can be passed as string[] props, and each item of the array can be parsed as JSON. If the user asks you to provide an array of object, ALWAYS provide a `type` of string[]. Optionally, props can have a human-readable `description` describing the param. Optional parameters that correspond to the test code should be declared with `optional: true`. Recall that props may contain an `options` method. +{typescript_definitions} -Within the component's run method, the `this` variable refers to properties of the component. All props are exposed at `this.`. e.g. `this.input`. `this` doesn't contain any other properties. +{other_example} -The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event. +{rules} -There are also two other props in sources: `http` and `db`. `http` is an interface that lets you receive and respond to HTTP requests. `db` is a data store that lets you store data between runs of the component. You should always include both. +{additional_rules} -The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". - -Always add computing signature validation, and please use the the crypto package HMAC-SHA256 method unless specified otherwise. - -The `db` prop is a simple key-value pair database that stores JSON-serializable data. It is used to maintain state across executions of the component. It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored. - -## Source Hooks - -Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks. - -## Pipedream Platform Axios - -If you need to make an HTTP request, use the `axios` constructor from the `@pipedream/platform` package, and include the following import at the top of your Node.js code, above the component, in this exact format: - -import { axios } from "@pipedream/platform"; - -You MUST use that import format when importing axios. Do NOT attempt to import any other package like `import axios from "@pipedream/platform/axios"`. - -The `axios` constructor takes two arguments: - -1. `this` - the context passed by the run method of the component. - -2. `config` - the same as the `config` object passed to the `axios` constructor in the standard `axios` package, with some extra properties. - -For example: - -async run({steps, $}) { - return await axios($, { - url: `https://api.openai.com/v1/models`, - headers: { - Authorization: `Bearer ${this.openai.$auth.api_key}`, - }, - }) -}, - -`@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. - -Here's an example Pipedream source component that receives a webhook from Tally for every new form response and processes the incoming event data: - -export default { - key: "tally-new-response", - name: "New Response", - version: "0.0.1", - description: "Emit new event on each form message. [See docs here]()", - type: "source", - dedupe: "unique", - props: { - tally: { - type: "app", - app: "tally", - }, - db: "$.service.db", - http: { - type: "$.interface.http", - customResponse: false, - }, - formId: { - type: "string", - label: "Form", - description: "Select a form", - async options() { - const forms = await this.getForms(); - return forms.map((form) => ({ - label: form.name, - value: form.id, - })); - }, - }, - }, - async run(event) { - const { data: response } = event; - this.$emit(response, { - id: response.responseId, - summary: `New response for ${response.formName} form`, - ts: Date.parse(response.createdAt), - }); - }, -}; - -The code you generate should be placed within the `run` method of the Pipedream component: - -import { axios } from "@pipedream/platform"; - -export default defineComponent({ - props: { - the_app_name_slug: { - type: "app", - app: "the_app_name_slug", - }, - http: "$.interface.http", - db: "$.service.db", - }, - async run(event) { - // your code here - }, -}); - -## Async options props - -The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options: - -``` -[ - { - label: "Human-readable option 1", - value: "unique identifier 1", - }, - { - label: "Human-readable option 2", - value: "unique identifier 2", - }, -] -``` - -The `label` MUST BE a human-readable name of the option presented to the user in the UI, and the `value` is the value of the prop in the `run` method. The `label` MUST be set to the property that defines the name of the object, and the `value` should be the property that defines the unique identifier of the object. - -If an API endpoint exists that can be used to fetch the options for the prop, you MUST define an `async` options method. This allows Pipedream to make an API call to fetch the options for the prop when the user is configuring the component, rather than forcing the user to enter values for the option manually. Think about it: this is so much easier for the user. - -Example async options methods: - -``` -msg: { - type: "string", - label: "Message", - description: "Select a message to `console.log()`", - async options() { - // write any node code that returns a string[] (with label/value keys) - return ["This is option 1", "This is option 2"]; - }, -}, -``` - -``` -board: { - type: "string", - label: "Board", - async options(opts) { - const boards = await this.getBoards(this.$auth.oauth_uid); - const activeBoards = boards.filter((board) => board.closed === false); - return activeBoards.map((board) => { - return { label: board.name, value: board.id }; - }); - }, -}, -``` - -``` -async options(opts) { - const response = await axios(this, { - method: "GET", - url: `https://api.spotify.com/v1/me/playlists`, - headers: { - Authorization: `Bearer \${this.spotify.$auth.oauth_access_token}`, - }, - }); - return response.items.map((playlist) => { - return { label: playlist.name, value: playlist.id }; - }); -}, -``` - -## Component Metadata - -Registry components require a unique key and version, and a friendly name and description. E.g. - -``` -export default { - key: "google_drive-new-shared-drive-created", - name: "New Shared Drive Created", - description: "Emits a new event any time a shared drive is created.", - version: "0.0.1", - type: "source", - dedupe: "unique", -}; -``` - -Component keys are in the format app_name_slug-slugified-component-name. -You should come up with a name and a description for the component you are generating. -In the description, you should include a link to the app docs, if they exist. Or add this as a placeholder: [See docs here](). -Source keys should use past tense verbs that describe the event that occurred (e.g., linear_app-issue-created-instant). -Always add version "0.0.1", type "source", and dedupe "unique". - -## TypeScript Definitions - -export interface Methods { - [key: string]: (...args: any) => unknown; -} - -// $.flow.exit() and $.flow.delay() -export interface FlowFunctions { - exit: (reason: string) => void; - delay: (ms: number) => { - resume_url: string; - cancel_url: string; - }; -} - -export interface Pipedream { - export: (key: string, value: JSONValue) => void; - send: SendFunctionsWrapper; - /** - * Respond to an HTTP interface. - * @param response Define the status and body of the request. - * @returns A promise that is fulfilled when the body is read or an immediate response is issued - */ - respond: (response: HTTPResponse) => Promise | void; - flow: FlowFunctions; -} - -// Arguments to the options method for props -export interface OptionsMethodArgs { - page?: number; - prevContext?: any; - [key: string]: any; -} - -// You can reference the values of previously-configured props! -export interface OptionalOptsFn { - (configuredProps: { [key: string]: any; }): object; -} - -export type PropDefinition = - [App, string] | - [App, string, OptionalOptsFn]; - -// You can reference props defined in app methods, referencing the propDefintion directly in props -export interface PropDefinitionReference { - propDefinition: PropDefinition; -} - -export interface App< - Methods, - AppPropDefinitions -> { - type: "app"; - app: string; - propDefinitions?: AppPropDefinitions; - methods?: Methods & ThisType; -} - -export function defineApp< - Methods, - AppPropDefinitions, -> -(app: App): App { - return app; -} - -// Props - -export interface DefaultConfig { - intervalSeconds?: number; - cron?: string; -} - -export interface Field { - name: string; - value: string; -} - -export interface BasePropInterface { - label?: string; - description?: string; -} - -export type PropOptions = any[] | Array<{ [key: string]: string; }>; - -export interface UserProp extends BasePropInterface { - type: "boolean" | "boolean[]" | "integer" | "integer[]" | "string" | "string[]" | "object" | "any"; - options?: PropOptions | ((this: any, opts: OptionsMethodArgs) => Promise); - optional?: boolean; - default?: JSONValue; - secret?: boolean; - min?: number; - max?: number; -} - -export interface InterfaceProp extends BasePropInterface { - type: "$.interface.http" | "$.interface.timer"; - default?: string | DefaultConfig; -} - -// When users ask about data stores, remember to include a prop of type "data_store" in the props object -export interface DataStoreProp extends BasePropInterface { - type: "data_store"; -} - -export interface HttpRequestProp extends BasePropInterface { - type: "http_request"; - default?: DefaultHttpRequestPropConfig; -} - -export interface ActionPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp | DataStoreProp | HttpRequestProp; -} - -export interface AppPropDefinitions { - [name: string]: PropDefinitionReference | App | UserProp; -} - -export interface ActionRunOptions { - $: Pipedream; - steps: JSONValue; -} - -type PropThis = { - [Prop in keyof Props]: Props[Prop] extends App ? any : any -}; - -export interface Action< - Methods, - ActionPropDefinitions -> { - key: string; - name?: string; - description?: string; - version: string; - type: "action"; - methods?: Methods & ThisType & Methods>; - props?: ActionPropDefinitions; - additionalProps?: ( - previousPropDefs: ActionPropDefinitions - ) => Promise; - run: (this: PropThis & Methods, options?: ActionRunOptions) => any; -} - -export function defineAction< - Methods, - ActionPropDefinitions, -> -(component: Action): Action { - return component; -} - -## Additional rules - -1. Use ESM for all imports, not CommonJS. Place all imports at the top of the file, above `export default`. - -2. Include all required headers and parameters in the API request. Please pass literal values as the values of all required params. Use the proper types of values, e.g. "test" for strings and true for booleans. - -3. Always use the correct HTTP method in the `axios` request. Compare this to other code examples you've been trained on. - -4. Double-check the code against known Node.js examples, from GitHub and any other real code you find. - -5. Always emit relevant data incoming from the webhook. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. - -6. Always use this signature for the run method: - -async run(event) { - // your code here -} - -## Remember, return ONLY code - -Only return Node.js code. DO NOT include any English text before or after the Node.js code. DO NOT say something like "Here's an example..." to preface the code. DO NOT include the code in Markdown code blocks, or format it in any fancy way. Just show me the code. - -Consider all the instructions and rules above, and use the following code as a template for your code: %s -""" - -with_docs_system_instructions = f"""{no_docs_system_instructions} -You are an agent designed to interact with an OpenAPI JSON specification. -You have access to the following tools which help you learn more about the JSON you are interacting with. -Only use the below tools. Only use the information returned by the below tools to construct your final answer. -Do not make up any information that is not contained in the JSON. -Your input to the tools should be in the form of `data["key"][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Python. -You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `json_spec_list_keys`. -If you have not seen a key in one of those responses, you cannot use it. -You should only add one key at a time to the path. You cannot add multiple keys at once. -If you encounter a "KeyError", go back to the previous key, look at the available keys, and try again. - -Before you build your answer, you should first look for the the base endpoint and authentication method in the JSON values. -Then you should proceed to search for the rest of the information to build your answer. - -If the question does not seem to be related to the JSON, just return "I don't know" as the answer. -Always begin your interaction with the `json_spec_list_keys` tool with input "data" to see what keys exist in the JSON. - -Note that sometimes the value at a given path is large. In this case, you will get an error "Value is a large dictionary, should explore its keys directly". -In this case, you should ALWAYS follow up by using the `json_spec_list_keys` tool to see what keys exist at that path. -Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.""" - -suffix = """--- -Begin! -Remember, DO NOT include any other text in your response other than the code. -DO NOT return ``` or any other code formatting characters in your response. - -Question: {input} -{agent_scratchpad}""" - -format_instructions = """Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do. always escape curly brackets -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question. do not include any other text than the code itself""" +{end}""" diff --git a/packages/component_code_gen/templates/sources/db.py b/packages/component_code_gen/templates/sources/db.py new file mode 100644 index 0000000000000..b0b9a0cbf6285 --- /dev/null +++ b/packages/component_code_gen/templates/sources/db.py @@ -0,0 +1,5 @@ +db = """There is also another prop in sources: `db`. It is a data store of type `$.service.db`. You should always include it. + +It is a simple key-value pair database that stores JSON-serializable data that lets you store data between runs of the component. It is used to maintain state across executions of the component. + +It contains two methods, `get` and `set`. The `get` method has one parameter - the key of the data to retrieve. The `set` method has two parameters - the key of the data to store, and the value to store. Both methods return a Promise that resolves when the data is read or stored.""" diff --git a/packages/component_code_gen/templates/sources/polling/additional_rules.py b/packages/component_code_gen/templates/sources/polling/additional_rules.py new file mode 100644 index 0000000000000..82638a49247d0 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/additional_rules.py @@ -0,0 +1,9 @@ +additional_rules = """## Additional rules for polling sources + +1. Always emit relevant data. The data being emitted must be JSON-serializable. The emitted data is displayed in Pipedream and used in the next steps. + +2. Always use this signature for the run method: + +async run() { + // your code here +}""" diff --git a/packages/component_code_gen/templates/sources/polling/hooks.py b/packages/component_code_gen/templates/sources/polling/hooks.py new file mode 100644 index 0000000000000..989940f1f62f9 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/hooks.py @@ -0,0 +1,3 @@ +hooks = """## Source Hooks + +Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks.""" diff --git a/packages/component_code_gen/templates/sources/polling/introduction.py b/packages/component_code_gen/templates/sources/polling/introduction.py new file mode 100644 index 0000000000000..c1cf4b8c24125 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/introduction.py @@ -0,0 +1,7 @@ +introduction = """You are an agent designed to create Pipedream Polling Source Component Code. + +You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Polling Source Component Code. + +## Pipedream Source Components + +All Pipedream polling source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument.""" diff --git a/packages/component_code_gen/templates/sources/polling/main_example.py b/packages/component_code_gen/templates/sources/polling/main_example.py new file mode 100644 index 0000000000000..a5156877c0372 --- /dev/null +++ b/packages/component_code_gen/templates/sources/polling/main_example.py @@ -0,0 +1,100 @@ +main_example = """Here's an example Pipedream source component that fetches all bookmarks from Raindrop.io and emits each bookmark as an event: + +```javascript +import { axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform" +export default { + key: "raindrop-bookmark-created", + name: "New Bookmark Created", + description: `Emit new event when a bookmark is created. [See the documentation](${docsLink})`, + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + raindrop: { + type: "app", + app: "raindrop", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + collectionId: { + type: "string", + label: "Collection ID", + description: "The collection ID", + async options() { + // fetch collections from the API + const { items } = await this.getCollections(); + return items.map((e) => ({ + value: e._id, + label: e.title, + })); + }, + }, + }, + methods: { + _getPage() { + return this.db.get("page") ?? 0; + }, + _setPage(page) { + this.db.set("page", page); + }, + }, + hooks: { + async deploy() { + let page = 0; + const all_bookmarks = []; + + while (true) { + // fetch bookmarks from the API + const bookmarks = await this.raindrop.getRaindrops(this, this.collectionId, { + page, + perpage: MAX_PER_PAGE, + }); + all_bookmarks.unshift(...bookmarks.reverse()); + + if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; + page++; + } + + for (const bookmark of all_bookmarks.slice(0, 50)) { + this.$emit(bookmark, { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }); + } + + this._setPage(page); + }, + }, + async run() { + let page = this._getPage(); + + while (true) { + // fetch bookmarks from the API + const { items: bookmarks } = await this.raindrop.getRaindrops(this, this.collectionId, { + page, + perpage: 50, + }); + + for (const bookmark of bookmarks) { + this.$emit(bookmark, { + id: bookmark._id, + summary: `New Raindrop: ${bookmark.title}`, + ts: Date.parse(bookmark.created), + }); + }; + + if (bookmarks.length < constants.DEFAULT_PER_PAGE) break; + + page++; + } + + this._setPage(page); + }, +}; +```""" diff --git a/packages/component_code_gen/templates/sources/webhooks/additional_rules.py b/packages/component_code_gen/templates/sources/webhooks/additional_rules.py new file mode 100644 index 0000000000000..bd82e8100fd68 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/additional_rules.py @@ -0,0 +1,7 @@ +additional_rules = """## Additional rules for webhook sources + +1. Always use this signature for the run method: + +async run(event) { + // your code here +}""" diff --git a/packages/component_code_gen/templates/sources/webhooks/async_run.py b/packages/component_code_gen/templates/sources/webhooks/async_run.py new file mode 100644 index 0000000000000..c4504d27f064a --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/async_run.py @@ -0,0 +1 @@ +async_run = """The run method is called when the component receives an event. The event is passed as the first and only argument to the run method. The event is a JSON object that contains the data from the webhook. The event is emitted by calling `this.$emit`. The first argument to `$emit` is the data to emit. You should only pass relevant data. For example, usually only the event.body is relevant. Headers and others are used to validate the webhook, but shouldn't be emitted. The second argument is an object that contains three fields: `id`, `summary`, and `ts`. The `id` field is a unique identifier for the event. The `summary` field is a human-readable summary of the event. The `ts` field is a timestamp of the event.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/hooks.py b/packages/component_code_gen/templates/sources/webhooks/hooks.py new file mode 100644 index 0000000000000..989940f1f62f9 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/hooks.py @@ -0,0 +1,3 @@ +hooks = """## Source Hooks + +Pipedream sources support the following hooks: deploy, activate and deactivate. The deploy() hook is automatically invoked by Pipedream when a source is deployed. It is usually used to fetch historical data from the API and emit events for each item. The max number of historical events is 50. They should be the most recent ones. Please paginate through all until the last 50 events are reached, unless sorting events by most recent is available. The activate() hook is automatically invoked by Pipedream when a source is activated. It is usually used to create a webhook subscription. The deactivate() hook is automatically invoked by Pipedream when a source is deactivated. It is usually used to delete a webhook subscription. Always include code for all three hooks.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/http.py b/packages/component_code_gen/templates/sources/webhooks/http.py new file mode 100644 index 0000000000000..0b0b9f01c3770 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/http.py @@ -0,0 +1,5 @@ +http = """There is also another prop in sources: `http`. It is an interface of type `$.interface.http` that lets you receive and respond to HTTP requests. You should always include it. + +The `http` prop has a field called `customResponse`, which is used when a signature validation is needed to be done before responding the request. If the `customResponse` is set to `true`, the `respond` method will be called with the response object as the argument. The response object has three fields: `status`, `headers` and `body`. The `status` field is the HTTP status code of the response, the `headers` is a key-value object of the response and the `body` field is the body of the response. The `respond` method should return a promise that resolves when the body is read or an immediate response is issued. If the `customResponse` is set to `false`, an immediate response will be transparently issued with a status code of 200 and a body of "OK". + +Always add computing signature validation when the app supports it, and please use the the crypto package HMAC-SHA256 method unless specified otherwise.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/introduction.py b/packages/component_code_gen/templates/sources/webhooks/introduction.py new file mode 100644 index 0000000000000..a7b82a423858f --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/introduction.py @@ -0,0 +1,8 @@ +introduction = """You are an agent designed to create Pipedream Webhooks Source Component Code. + +You will receive a prompt from an user. You should create a code in Node.js using axios for a HTTP request if needed. Your goal is to create a Pipedream Webhooks Source Component Code, also called Pipedream Webhooks Trigger Code. +You should not return any text other than the code. + +## Pipedream Source Components + +All Pipedream webhook source components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument.""" diff --git a/packages/component_code_gen/templates/sources/webhooks/main_example.py b/packages/component_code_gen/templates/sources/webhooks/main_example.py new file mode 100644 index 0000000000000..493e73f96aca1 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/main_example.py @@ -0,0 +1,66 @@ +main_example = """Here's an example component: + +```javascript +import crypto from "crypto" +import { axios } from "@pipedream/platform" +export default { + key: "github-new-notification-received", + name: "New Notification Received", + description: "Emit new event when a notification is received.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + github: { + type: "app", + app: "github", + }, + http: { + type: ""$.interface.http"", + customResponse: true, // optional: defaults to false + }, + db: "$.service.db", + }, + methods: { + _getWebhookId() { + return this.db.get("webhookId") + }, + _setWebhookId(id) { + this.db.set("webhookId", id) + }, + }, + hooks: { + async deploy() { + const events = await this.github.listMostRecentNotifications({paginate: true, max: 50}); + for (const event of events) { + this.$emit(event, { + id: event.id, + summary: `New event: ${event.name}`, + ts: Date.parse(event.ts), + }) + } + }, + async activate() { + const hookId = await this.createWebhook(opts) + this._setWebhookId(hookId) + }, + async deactivate() { + const id = this._getWebhookId() + await this.deleteWebhook(id) + }, + }, + async run(event) { + const computedSignature = crypto.createHmac(sha256, secretKey).update(rawBody).digest("base64") + if (computedSignature !== webhookSignature) { + this.http.respond({ status: 401, body: "Unauthorized" }) + return + } + console.log(`Emitting event...`) + this.$emit(event, { + id: event.id, + summary: `New event: ${event.name}`, + ts: Date.parse(event.ts), + }) + }, +}; +```""" diff --git a/packages/component_code_gen/templates/sources/webhooks/other_example.py b/packages/component_code_gen/templates/sources/webhooks/other_example.py new file mode 100644 index 0000000000000..7e474e3366527 --- /dev/null +++ b/packages/component_code_gen/templates/sources/webhooks/other_example.py @@ -0,0 +1,59 @@ +other_example = """Here's an example Pipedream source component that receives a webhook from Tally for every new form response and processes the incoming event data: + +export default { + key: "tally-new-response", + name: "New Response", + version: "0.0.1", + description: "Emit new event on each form message. [See docs here]()", + type: "source", + dedupe: "unique", + props: { + tally: { + type: "app", + app: "tally", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: false, + }, + formId: { + type: "string", + label: "Form", + description: "Select a form", + async options() { + const forms = await this.getForms(); + return forms.map((form) => ({ + label: form.name, + value: form.id, + })); + }, + }, + }, + async run(event) { + const { data: response } = event; + this.$emit(response, { + id: response.responseId, + summary: `New response for ${response.formName} form`, + ts: Date.parse(response.createdAt), + }); + }, +}; + +The code you generate should be placed within the `run` method of the Pipedream component: + +import { axios } from "@pipedream/platform"; + +export default defineComponent({ + props: { + the_app_name_slug: { + type: "app", + app: "the_app_name_slug", + }, + http: "$.interface.http", + db: "$.service.db", + }, + async run(event) { + // your code here + }, +});"""