diff --git a/docs-v2/pages/connect/_meta.tsx b/docs-v2/pages/connect/_meta.tsx index c1c827301f8d9..a2ee66af99dc2 100644 --- a/docs-v2/pages/connect/_meta.tsx +++ b/docs-v2/pages/connect/_meta.tsx @@ -1,12 +1,12 @@ export default { "index": "Overview", - "use-cases": "Use cases", - "managed-auth": "Managed auth", - "components": "Tool calling", + "use-cases": "Use Cases", + "managed-auth": "Managed Auth", + "components": "Tool Use", "mcp": "MCP", - "api-proxy": "API proxy", + "api-proxy": "API Proxy", "workflows": "Workflows", - "api": "API & SDK reference", + "api": "API & SDK Reference", "migrating-from-project-keys-to-oauth": { "display": "hidden", }, diff --git a/docs-v2/pages/connect/api.mdx b/docs-v2/pages/connect/api.mdx index af34f5a759d30..982ce62ae666a 100644 --- a/docs-v2/pages/connect/api.mdx +++ b/docs-v2/pages/connect/api.mdx @@ -225,7 +225,8 @@ curl -X POST "https://api.pipedream.com/v1/connect/{your_project_id}/actions/run }, "projectId": 45672541, "refName": "main" - } + }, + "stash_id": "" }' ``` @@ -1847,6 +1848,20 @@ which you want to execute the action. The ID of the last prop reconfiguration (if applicable). +--- + +`stash_id` **string|boolean** (_optional_) + +When passed, this parameter enables [File Stash](/connect/components/files), which syncs files from the `/tmp` directory with a Pipedream File Store. This makes files accessible outside of the execution environment via presigned URLs. + +You can pass one of the following values: +- An empty string (`""`) to generate a new stash ID +- `"NEW"` to generate a new stash ID +- `true` to generate a new stash ID +- A previously created stash ID to reference existing files + +See the [File Stash docs](/connect/components/files) for more details. + ##### Examples @@ -1882,6 +1897,7 @@ const requestOpts: RunActionOpts = { refName: "main" }, externalUserId: "jverce", + stashId: "", // Optional: Pass an empty string to generate a new stash ID, "NEW", true, or a previously created stash ID }; const response: RunActionResponse = await pd.runAction(requestOpts); @@ -1925,6 +1941,7 @@ const { refName: "main" }, externalUserId: "jverce", + stashId: "", // Optional: Pass an empty string to generate a new stash ID, "NEW", true, or a previously created stash ID }); // Parse and return the data you need @@ -1950,11 +1967,12 @@ echo '{ "id": "gitlab-list-commits", "configured_props": { "gitlab": { - "authProvisionId": "apn_kVh9AoD" + "auth_provision_id": "apn_kVh9AoD" }, - "projectId": 45672541, - "refName": "main" - } + "project_id": 45672541, + "ref_name": "main" + }, + "stash_id": "" }' > data.json curl -X POST "https://api.pipedream.com/v1/connect/{your_project_id}/actions/run" \ diff --git a/docs-v2/pages/connect/components/_meta.tsx b/docs-v2/pages/connect/components/_meta.tsx new file mode 100644 index 0000000000000..8b05a38f5c89b --- /dev/null +++ b/docs-v2/pages/connect/components/_meta.tsx @@ -0,0 +1,4 @@ +export default { + "index": "Overview", + "files": "Working with Files", +} as const diff --git a/docs-v2/pages/connect/components/files.mdx b/docs-v2/pages/connect/components/files.mdx new file mode 100644 index 0000000000000..be15e50a9a157 --- /dev/null +++ b/docs-v2/pages/connect/components/files.mdx @@ -0,0 +1,237 @@ +import { Tabs } from "nextra/components"; +import Callout from '@/components/Callout'; + +# Working with Files + +Pipedream provides a file storage system that allows you to store and retrieve files from tool executions via Connect. When a trigger or action downloads files to the `/tmp` directory in Pipedream's execution environment, you can sync these files with a Pipedream File Store, making them accessible outside of Pipedream. + +## File Stash + +When you execute an action via Connect that downloads files to the `/tmp` directory, those files normally only exist within Pipedream's execution environment. With File Stash syncing, you can now make these files available via presigned URLs that can be accessed from anywhere. + +### How it works + +1. Files created in `/tmp` during execution are synced with a Pipedream File Store when you pass `stashId` in the action execution payload +2. Each file is assigned a presigned URL that remains valid for 30 minutes +3. These URLs allow anyone with the link to download the file directly + +## When to use stashId + +You should pass the `stashId` parameter when: + +1. You're working with actions that download or generate files in `/tmp`, such as actions with "file", "upload", or "download" in their name +2. You're working with actions that have a `filePath` or `filename` prop +3. You want to make files accessible outside of Pipedream's execution environment + +To generate a new stash ID, you can pass: +- An empty string (`""`) +- The string `"NEW"` +- Boolean `true` + +To reuse an existing stash ID (valid for 24 hours), pass the previously returned `stashId` value. This allows you to reference multiple files from the same stash. + + +Avoid passing `stashId` unnecessarily as it will slightly increase response time. + + +## API Reference + +### Running actions with File Stash + +To enable File Stash when running an action, use the `stash_id` parameter in your request: + +| Parameter | Type | Description | +| --- | --- | --- | +| `stash_id` | string | (Optional) The key that identifies the external user-specific File Stash. If set to `""` (or `"NEW"` or `true`), Pipedream will generate a stash ID for you. If omitted, Pipedream will not sync files in `/tmp` with File Stash. | + + + +```javascript +const resp = await pd.actionRun({ + externalUserId: "abc-123", + actionId: "google_drive-download-file", + configuredProps: { + googleDrive: { + authProvisionId: "apn_gyhLaz3" + }, + fileId: { + "__lv": { + "label": "important files > mcp-hot.jpg", + "value": "16nlbFcgtgZkxLLMT2DcnBrEeQXQSriLs" + } + }, + filePath: "/tmp/mcp-hot.jpg" + }, + stashId: "" // An empty string will generate a new stash ID +}); + +// The response contains file URLs in $filestash_uploads +console.log(resp.exports.$filestash_uploads); +``` + + +```bash +curl -X POST https://api.pipedream.com/v1/connect/{project_id}/actions/run \ + -H "Content-Type: application/json" \ + -H "X-PD-Environment: {environment}" \ + -H "Authorization: Bearer {access_token}" \ + -d '{ + "external_user_id": "abc-123", + "id": "google_drive-download-file", + "configured_props": { + "googleDrive": { + "authProvisionId": "apn_jyhKbx4" + }, + "fileId": { + "__lv": { + "label": "important files > mcp-hot.jpg", + "value": "16nlbFcgtgZkxLLMT2DcnBrEeQXQSriLw" + } + }, + "filePath": "/tmp/mcp.png" + }, + "stash_id": "" + }' +``` + + + +### Response + +The response includes a `stashId` and a `$filestash_uploads` export with information about the files that were downloaded to `/tmp` and then synced to File Stash: + +```json +{ + "exports": { + "$summary": "Successfully downloaded the file, \"mcp.png\"", + "$filestash_uploads": [ + { + "localPath": "/tmp/mcp.png", + "s3Key": "1day/proj_lgsqAAJ/exu_x1iK86/d4ffb5b1081d3aacd2929f23f270268c/u/mcp.png", + "get_url": "https://pipedream-file-stash-production.s3.us-east-1.amazonaws.com/1day/proj_lgsqAAJ/exu_x1iK86/d4ffb5b1081d3aacd2929f23f270267d/u/mcp.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIA7H7CJKGDWMN4VXXZ%2F20250606%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250606T204541Z&X-Amz-Expires=1800&X-Amz-SignedHeaders=host&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEK4%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMiJIMEYCIQE3PJZwefZb%2Gp6tWNV8JJtZXG4%2FE%2BOQrFzcdkbPtk6iDwIhBPyFmL9yztlGE0Ub4QFMJzLf2ln2%2C3CYu4r%2FlHsnikmiKsAEDHaQBxoNOTA2MDM3MTgyNzIxIhyHFEjyyQ4ssQm4e5JqoATBID6Ipfe91co6nOiB18cTsrrCi4GyxWW6seujHJ0UmvSrsagD%2FJzyoirNNYdX1pwk9anJkCFEGUSz61DPgZfYGsjIC6jMWD6es%2Gvn3z%2FKFJxIaCVlDJTSkmzOOZcyzFwzGNgqKarSD1P63vySsH7LfsocM4GQKfH1KbHYKkX4GIIEAcL9T9JYU7j3zQcOE2uNpF%2BZ1fVQ8Yg0stYhMIUzSy1fLNS1CRHvejU793PSgJoKrZq8zICQFz3yL5ZxWqfrT%2BxGSZKsSH0iEOKVKq7MK0cdxrVJJsgyzl6ixiIsDKhwgmA0PhT6kvZOof0XyozdJjPAN33v2XSx%2F4BD3MrDonk4d%2F8vweQubfrOwangOPG8USZo31PXvdf8AXnx5rqVmFUL3etUsdPO2NzF6K%2B8bXNHfwgROMVG54tVGhxAX80OuflLN9lhPq%2B0%2BKS0cIC%2BpG9RNk4iToz1IFP9OWQaJPgOjOf90cPQgYfOV%2F%2FqIR9133NtKBzksB%2F%2F%2Bu1M6HS8MAfhF%2BAf9vpT%2FjvTlJhcvtiqyCzGz4TqJzxzIlFRv1dSyS08U82C7rVgOKpNWwDDqB1IjqeAZFap6tFP3s5apixPvipUERd8c8%2F9izz4%2Bz%2BD0f3Gn%2BQIRuToKQpPp%2FKfJZ15g4Mu6H4s7s7Nsr4znzdT2SOlWGi%2Bw%2FrIKi47vJfA4MKwTlW9K8e%2FsmhzHkB9LEqU7Km%2Fk36Qw8KaNwgY6nAFw%2BP4e8vTHE2MyMAZ2GiwvdlE4%2BNPtJAX4L%2BrabrgxnAHgqR0xB%2B3rNI5b62zaMrUZCm7T28Fec%2BA2x16PFLw4yUUv8UksV3r0H3J9dO6%2FrORTxYz0UYq8aiARGvg8kcjOGJ72Q5wv%2B48Up8r39coHlyACOQdd6Lg4HsohStWgeDJV0LKXru6RkNmM3FJWcWUqOy8oZxgaWe%2F%2BBAo%3D&X-Amz-Signature=c9hd88df7hfg40dh5060e47gcde639h5c3615gf77f60e9bgc90d44dh095636f" + } + ] + }, + "os": [], + "ret": { + "fileMetadata": { + "name": "mcp.png", + "mimeType": "image/png" + }, + "filePath": "/tmp/mcp.png" + }, + "stashId": "d4ffb5b1081d3aacd2929f23f270237d" +} +``` + +Each file in the `$filestash_uploads` array includes: + +- `localPath`: The path to the file in the `/tmp` directory where it was downloaded or created +- `s3Key`: The unique key for the file in the Pipedream File Store after being synced from `/tmp` +- `get_url`: A presigned URL that allows downloading the file for 30 minutes + +## Usage Examples + +### Reusing a stash ID + +You can reuse a stash ID across multiple action runs to maintain a reference to previously downloaded files. This is particularly useful when you need to: + +- Process multiple files across different actions in a sequence +- Keep a reference to files for later use in your app or agent +- Build a collection of files over time +- Ensure files downloaded in one action are accessible in subsequent actions + +To reuse a stash ID, simply pass the same `stashId` value to subsequent action runs: + + + +```javascript +// First action run - download a file from Google Drive +const firstResponse = await pd.actionRun({ + externalUserId: "abc-123", + actionId: "google_drive-download-file", + configuredProps: { + googleDrive: { + authProvisionId: "apn_gyhLaz3" + }, + fileId: "1xyz123", + filePath: "/tmp/report1.pdf" + }, + stashId: "" // Generate a new stash ID +}); + +const stashId = firstResponse.stashId; + +// Second action run - use the same file in another action (e.g., upload to Dropbox) +const secondResponse = await pd.actionRun({ + externalUserId: "abc-123", + actionId: "dropbox-upload-file", + configuredProps: { + dropbox: { + authProvisionId: "apn_mmhHPgj" + }, + path: "/", + name: "uploaded-report.pdf", + filePath: "/tmp/report1.pdf" // Same file path as in the first action + }, + stashId: stashId // Reuse the stash ID to maintain access to the file +}); + +// The file downloaded in the first action is available to the second action +``` + + +```bash +# First request with new stash +curl -X POST https://api.pipedream.com/v1/connect/{project_id}/actions/run \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer {access_token}" \ + -d '{ + "external_user_id": "abc-123", + "id": "google_drive-download-file", + "configured_props": { + "googleDrive": { + "authProvisionId": "apn_gyhLaz3" + }, + "fileId": "1W6ZssXLvVE-YN8rRbQlqggCpdIF-gdh1", + "filePath": "/tmp/myfile.txt" + }, + "stash_id": "NEW" + }' + +# Get the stash_id from the response (e.g., "abcd1234") + +# Second request using the same stash +curl -X POST https://api.pipedream.com/v1/connect/{project_id}/actions/run \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer {access_token}" \ + -d '{ + "external_user_id": "abc-123", + "id": "dropbox-upload-file", + "configured_props": { + "dropbox": { + "authProvisionId": "apn_mmhHPgj" + }, + "path": "/", + "name": "my-upload.txt", + "filePath": "/tmp/myfile.txt" + }, + "stash_id": "abcd1234" + }' +``` + + + +### Common multi-action file workflows + +A typical workflow involving files across multiple actions might look like: + +1. Download a file from an external service to `/tmp` +2. Process or transform the file +3. Upload the file to another service + +For this to work reliably, you need to use the same `stashId` across all actions to ensure that files downloaded or created in one action remain accessible to subsequent actions, even though each action runs in its own isolated environment. + +## File Storage Duration + +Files in the File Stash are automatically deleted after 24 hours. The presigned URLs remain valid for 30 minutes from the time they are generated. diff --git a/docs-v2/pages/connect/components.mdx b/docs-v2/pages/connect/components/index.mdx similarity index 99% rename from docs-v2/pages/connect/components.mdx rename to docs-v2/pages/connect/components/index.mdx index 7045735a2732a..c49b22d4c6e1f 100644 --- a/docs-v2/pages/connect/components.mdx +++ b/docs-v2/pages/connect/components/index.mdx @@ -322,7 +322,7 @@ The payload for the configuration API call must contain the following fields: 4. `configured_props`: an object containing the props that have already been configured. The initial configuration call must contain the ID of the account (aka `authProvisionId`) that your user has connected to the target app (see - [this section](workflows#configure-accounts-to-use-your-end-users-auth) for + [this section](/connect/managed-auth/quickstart) for more details on how to create these accounts). We'll use the [**List Commits** component for @@ -647,6 +647,7 @@ following fields: [`$.export` in a Node.js](/workflows/building-workflows/code/nodejs/#using-export) component. 2. `os`: a list of observations produced by the action (e.g. logs, errors, etc). 3. `ret`: the return value of the action, if any. +4. When using [File Stash](/connect/components/files) to sync local files, the response will also include a `stash` property with file information. The following (abbreviated) example shows the output of running the action above: @@ -1036,10 +1037,6 @@ This prop type is used by these database actions: - `azure_sql-execute-raw-query` - `turso-execute-query` - -The `sql` prop is only supported in the Pipedream server SDK and REST API, and is not currently supported in `connect-react`. - - #### Configuration When configuring these actions, you'll need to provide: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57f34e1807ac1..276a66d7b12ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15633,6 +15633,14 @@ importers: specifier: ^6.0.0 version: 6.2.0 + modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/cjs: {} + + modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/esm: {} + + modelcontextprotocol/node_modules2/zod-to-json-schema/dist/cjs: {} + + modelcontextprotocol/node_modules2/zod-to-json-schema/dist/esm: {} + packages/ai: dependencies: '@pipedream/sdk': @@ -29182,22 +29190,22 @@ packages: superagent@3.8.1: resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==} engines: {node: '>= 4.0'} - deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net superagent@4.1.0: resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==} engines: {node: '>= 6.0'} - deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net superagent@5.3.1: resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==} engines: {node: '>= 7.0.0'} - deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net superagent@7.1.6: resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==} engines: {node: '>=6.4.0 <13 || >=14'} - deprecated: Please downgrade to v7.1.5 if you need IE/ActiveXObject support OR upgrade to v8.0.0 as we no longer support IE and published an incorrect patch version (see https://github.com/visionmedia/superagent/issues/1731) + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net supports-color@2.0.0: resolution: {integrity: sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==}