From 3bc7129c4a4ce5e688a357678b0cc2a787eae896 Mon Sep 17 00:00:00 2001 From: Paul Cornell Date: Mon, 21 Oct 2024 16:13:10 -0700 Subject: [PATCH] Astra DB destination connector for Platform --- mint.json | 1 + platform/connectors.mdx | 1 + platform/destinations/astradb.mdx | 24 ++++ platform/overview.mdx | 110 +++--------------- .../general-shared-text/astradb-platform.mdx | 7 ++ 5 files changed, 50 insertions(+), 93 deletions(-) create mode 100644 platform/destinations/astradb.mdx create mode 100644 snippets/general-shared-text/astradb-platform.mdx diff --git a/mint.json b/mint.json index 477d82d6..c65d8f83 100644 --- a/mint.json +++ b/mint.json @@ -445,6 +445,7 @@ "group": "Destinations", "pages": [ "platform/destinations/overview", + "platform/destinations/astradb", "platform/destinations/azure-cognitive-search", "platform/destinations/milvus", "platform/destinations/mongodb", diff --git a/platform/connectors.mdx b/platform/connectors.mdx index d351a621..e8239014 100644 --- a/platform/connectors.mdx +++ b/platform/connectors.mdx @@ -21,6 +21,7 @@ If your source is not listed here, you might still be able to connect Unstructur ## Destinations +- [Astra DB](/platform/destinations/astradb) - [Azure Cognitive Search](/platform/destinations/azure-cognitive-search) - [Milvus](/platform/destinations/milvus) - [MongoDB](/platform/destinations/mongodb) diff --git a/platform/destinations/astradb.mdx b/platform/destinations/astradb.mdx new file mode 100644 index 00000000..c9bd7a6d --- /dev/null +++ b/platform/destinations/astradb.mdx @@ -0,0 +1,24 @@ +--- +title: Astra DB +--- + +Send processed data from Unstructured to Astra DB. + +You'll need: + +import AstraDBPrerequisites from '/snippets/general-shared-text/astradb.mdx'; + + + +To create the destination connector: + +1. On the sidebar, click **Destinations**. +2. Click **New Destination**. +3. In the **Type** drop-down list, select **Astra DB**. +4. Fill in the fields as described later on this page. +5. Click **Save and Test**. +6. Click **Close**. + +import AstraDBFields from '/snippets/general-shared-text/astradb-platform.mdx'; + + \ No newline at end of file diff --git a/platform/overview.mdx b/platform/overview.mdx index 45000298..0933ad9d 100644 --- a/platform/overview.mdx +++ b/platform/overview.mdx @@ -1,101 +1,25 @@ --- -title: Unstructured Platform -sidebarTitle: Overview +title: Overview +description: Destination connectors in the Unstructured Platform are designed to specify the endpoint for data processed within the platform. These connectors ensure that the transformed and analyzed data is securely and efficiently transferred to a storage system for future use, often to a vector database for tasks that involve high-speed retrieval and advanced data analytics operations. --- -To start using the Unstructured Platform right away, skip ahead to the [quickstart](/platform/quickstart). +![Destinations in the sidebar](/img/platform/Destinations-Sidebar.png) -## What is the Unstructured Platform? +To see your existing destination connectors, on the sidebar, click **Destinations**. - - -The Unstructured Platform is a no-code user interface, pay-as-you-go platform for transforming your unstructured data into data that is ready for Retrieval Augmented Generation (RAG). - -## How does it work? - -To get your data RAG-ready, the Unstructured Platform moves it through the following process: - -```mermaid - flowchart LR - Connect[1. Connect]-->Route[2. Route]-->Transform[3. Transform]-->Chunk[4. Chunk]-->Enrich[5. Enrich]-->Embed[6. Embed]-->Persist[7. Persist] -``` - - - The Unstructured Platform offers multiple [source connectors](/platform/sources/overview) to connect to your data in its existing location. - - - Routing determines which strategy Unstructured Platform uses to transforming your documents into Unstructured's canonical JSON schema. The Unstructured Platform provides these [partitioning](/platform/partitioning) strategies for document transformation: - - - **Fast** is great for when there is extractable text available, like in HTML files or in the Microsoft Office Document format. - - **Hi Res** is best for PDFs and tables and where accurate classification of document elements is critical. - - If you're unsure which strategy to use, choose **Auto**, and the Unstructured Platform will handle the decision for you. - - - - Your source document is transformed into Unstructured's canonical JSON schema. Regardless of the input document, this JSON schema we gives you a [standardized output](/platform/document-elements). It contains more than 20 elements, such as `Header`, `Footer`, `Title`, `NarrativeText`, `Table`, `Image`, and many more. Each document is wrapped in extensive metadata so you can understand languages, file types, sources, hierarchies, and much more. - - - The Unstructured Platform provides these [chunking](/platform/chunking) strategies: - - - **Basic** combines sequential elements up to specified size limits. Oversized elements are split, while tables are isolated and divided if necessary. Overlap between chunks is optional. - - **By Title** uses semantic chunking, understands the layout of the document, and makes intelligent splits. - - **By Page** attempts to preserve page boundaries when determining the chunks' contents. - - **By Similarity** uses an embedding model to identify topically similar sequential elements and combines them into chunks. - - - - Images and tables can be optionally summarized. This generates enriched content around the images or tables that were parsed during the transformation process. - - - The Unstructured Platform uses optional third-party [embedding](/platform/embedding) providers such as OpenAI. - - - The Unstructured Platform offers multiple [destination connectors](/platform/destinations/overview), including all major vector databases. - - - -To simplify this process and provide it as a no-code solution, the Unstructured Platform brings together four key concepts: - -```mermaid - flowchart LR - subgraph Workflow[3. Workflow] - direction LR - Source[1. Source Connector] --> Destination[2. Destination Connector] - end - Jobs - Workflow[3. Workflow] --> Jobs[4. Jobs] -``` - - - - [Source connectors](/platform/sources/overview) to ingest your data into the Unstructured Platform for transformation. - - - [Destination connectors](/platform/destinations/overview) tell the Unstructured Platform where to write your transformed data to. - - - [Workflows](/platform/workflows) connect sources to destinations and provide chunking, embedding, and scheduling options. - - - [Jobs](/platform/jobs) enable you to monitor data transformation progress. - - - -## What support is there for compliance? - -The platform is designed for global reach with SOC2 Type 1, SOC2 Type 2, and HIPAA compliance. It has support for over 50 languages. - -## How do I get started? - -Skip ahead to the [quickstart](/platform/quickstart). +To create a destination connector: +1. On the sidebar, click **Destinations**. +2. Click **New Destination**. +3. In the **Type** drop-down list, select the connector type that matches your destination. +4. Fill in the fields according to your connector type. To learn how, click your connector type in the following list: + - [Astra DB](/platform/destinations/astradb) + - [Azure Cognitive Search](/platform/destinations/azure-cognitive-search) + - [Milvus](/platform/destinations/milvus) + - [MongoDB](/platform/destinations/mongodb) + - [Pinecone](/platform/destinations/pinecone) + - [S3](/platform/destinations/s3) +5. Click **Save and Test**. +6. Click **Close**. \ No newline at end of file diff --git a/snippets/general-shared-text/astradb-platform.mdx b/snippets/general-shared-text/astradb-platform.mdx new file mode 100644 index 00000000..fbfb6cad --- /dev/null +++ b/snippets/general-shared-text/astradb-platform.mdx @@ -0,0 +1,7 @@ +Fill in the following fields: + +- **Name** (_required_): A unique name for this connector. +- **Token** (_required_): The application token for the database. +- **API Endpoint** (_required_): The database's associated API endpoint. +- **Collection Name** (_required_): The name of the collection in the namespace. +- **Embedding Dimension** (_required_): The number of dimensions in the collection. \ No newline at end of file