From b0aaf98222cb311a5beccc1db7886a9a00ad4f35 Mon Sep 17 00:00:00 2001 From: Kevin Cheung Date: Tue, 12 Nov 2024 12:03:54 -0800 Subject: [PATCH 1/2] docs: some 0.9 updates --- docs/evaluation.md | 157 +++++++++++++++--------- docs/get-started.md | 239 +++++++++++------------------------- docs/rag.md | 281 ++++++++++++++++++++++--------------------- docs/tool-calling.md | 241 ++++++++++++++++++++++++------------- 4 files changed, 472 insertions(+), 446 deletions(-) diff --git a/docs/evaluation.md b/docs/evaluation.md index 613a9f77f9..990955cf40 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -1,37 +1,49 @@ # Evaluation -Evaluations are a form of testing which helps you validate your LLM’s responses and ensure they meet your quality bar. - -Firebase Genkit supports third-party evaluation tools through plugins, paired with powerful observability features which provide insight into the runtime state -of your LLM-powered applications. Genkit tooling helps you automatically extract data including inputs, outputs, and information from intermediate steps to evaluate the end-to-end quality of LLM responses as well as understand the performance of your system’s building blocks. - -For example, if you have a RAG flow, Genkit will extract the set -of documents that was returned by the retriever so that you can evaluate the -quality of your retriever while it runs in the context of the flow as shown below with the Genkit faithfulness and answer relevancy metrics: - -```js -import { GenkitMetric, genkitEval } from '@genkit-ai/evaluator'; -import { textEmbeddingGecko } from '@genkit-ai/vertexai'; - -export default configureGenkit({ +Evaluations are a form of testing that helps you validate your LLM's responses +and ensure they meet your quality bar. + +Firebase Genkit supports third-party evaluation tools through plugins, paired +with powerful observability features that provide insight into the runtime state +of your LLM-powered applications. Genkit tooling helps you automatically extract +data including inputs, outputs, and information from intermediate steps to +evaluate the end-to-end quality of LLM responses as well as understand the +performance of your system's building blocks. + +For example, if you have a RAG flow, Genkit will extract the set of documents +that was returned by the retriever so that you can evaluate the quality of your +retriever while it runs in the context of the flow as shown below with the +Genkit faithfulness and answer relevancy metrics: + +```ts +import { genkit } from 'genkit'; +import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator'; +import { vertexAI, textEmbedding004, gemini15Flash } from '@genkit-ai/vertexai'; + +const ai = genkit({ plugins: [ + vertexAI(), genkitEval({ judge: gemini15Flash, metrics: [GenkitMetric.FAITHFULNESS, GenkitMetric.ANSWER_RELEVANCY], - embedder: textEmbeddingGecko, // GenkitMetric.ANSWER_RELEVANCY requires an embedder + embedder: textEmbedding004, // GenkitMetric.ANSWER_RELEVANCY requires an embedder }), ], // ... }); ``` -Note: The configuration above requires installing the `@genkit-ai/evaluator` and `@genkit-ai/vertexai` packages. +**Note:** The configuration above requires installing the `genkit`, +`@genkit-ai/google-ai`, `@genkit-ai/evaluator` and `@genkit-ai/vertexai` +packages. ```posix-terminal npm install @genkit-ai/evaluator @genkit-ai/vertexai ``` -Start by defining a set of inputs that you want to use as an input dataset called `testInputs.json`. This input dataset represents the test cases you will use to generate output for evaluation. +Start by defining a set of inputs that you want to use as an input dataset +called `testInputs.json`. This input dataset represents the test cases you will +use to generate output for evaluation. ```json ["Cheese", "Broccoli", "Spinach and Kale"] @@ -52,41 +64,47 @@ genkit start Then navigate to `localhost:4000/evaluate`. -Alternatively, you can provide an output file to inspect the output in a json file. +Alternatively, you can provide an output file to inspect the output in a JSON +file. ```posix-terminal genkit eval:flow menuSuggestionFlow --input testInputs.json --output eval-result.json ``` -Note: Below you can see an example of how an LLM can help you generate the test -cases. +**Note:** Below you can see an example of how an LLM can help you generate the +test cases. ## Supported evaluators ### Genkit evaluators -Genkit includes a small number of native evaluators, inspired by RAGAS, to help you get started: +Genkit includes a small number of native evaluators, inspired by RAGAS, to help +you get started: -- Faithfulness -- Answer Relevancy -- Maliciousness +* Faithfulness +* Answer Relevancy +* Maliciousness ### Evaluator plugins Genkit supports additional evaluators through plugins: -- VertexAI Rapid Evaluators via the [VertexAI Plugin](plugins/vertex-ai#evaluation). -- [LangChain Criteria Evaluation](https://python.langchain.com/docs/guides/productionization/evaluation/string/criteria_eval_chain/) via the [LangChain plugin](plugins/langchain.md). +* VertexAI Rapid Evaluators via the [VertexAI + Plugin](http://plugins/vertex-ai#evaluation). +* [LangChain Criteria + Evaluation](https://python.langchain.com/docs/guides/productionization/evaluation/string/criteria_eval_chain/) + via the [LangChain plugin](http://plugins/langchain.md). ## Advanced use `eval:flow` is a convenient way to quickly evaluate the flow, but sometimes you -might need more control over evaluation steps. This may occur if you are using a different -framework and already have some output you would like to evaluate. You can perform all -the step that `eval:flow` performs semi-manually. +might need more control over evaluation steps. This may occur if you are using a +different framework and already have some output you would like to evaluate. You +can perform all the steps that `eval:flow` performs semi-manually. -You can batch run your Genkit flow and add a unique label to the run which -then will be used to extract an evaluation dataset (a set of inputs, outputs, and contexts). +You can batch run your Genkit flow and add a unique label to the run which then +will be used to extract an evaluation dataset (a set of inputs, outputs, and +contexts). Run the flow over your test inputs: @@ -100,7 +118,8 @@ Extract the evaluation data: genkit eval:extractData myRagFlow --label customLabel --output customLabel_dataset.json ``` -The exported data will be output as a json file with each testCase in the following format: +The exported data will be output as a JSON file with each testCase in the +following format: ```json [ @@ -114,13 +133,19 @@ The exported data will be output as a json file with each testCase in the follow ] ``` -The data extractor will automatically locate retrievers and add the produced docs to the context array. By default, `eval:run` will run against all configured evaluators, and like `eval:flow`, results for `eval:run` will appear in the evaluation page of Developer UI, located at `localhost:4000/evaluate`. +The data extractor will automatically locate retrievers and add the produced +docs to the context array. By default, `eval:run` will run against all +configured evaluators, and like `eval:flow`, results for `eval:run` will appear +in the evaluation page of Developer UI, located at `localhost:4000/evaluate`. ### Custom extractors -You can also provide custom extractors to be used in `eval:extractData` and `eval:flow` commands. Custom extractors allow you to override the default extraction logic giving you more power in creating datasets and evaluating them. +You can also provide custom extractors to be used in `eval:extractData` and +`eval:flow` commands. Custom extractors allow you to override the default +extraction logic giving you more power in creating datasets and evaluating them. -To configure custom extractors, add a tools config file named `genkit-tools.conf.js` to your project root, if you don't have one already. +To configure custom extractors, add a tools config file named +`genkit-tools.conf.js` to your project root if you don't have one already. ```posix-terminal cd $GENKIT_PROJECT_HOME @@ -134,7 +159,7 @@ In the tools config file, add the following code: module.exports = { evaluators: [ { - flowName: 'myFlow', + actionRef: '/flow/myFlow', extractors: { context: { outputOf: 'foo-step' }, output: 'bar-step', @@ -144,17 +169,34 @@ module.exports = { }; ``` -In this sample, you configure an extractor for `myFlow` flow. The config overrides the extractors for `context` and `output` fields, and used the default logic for the `input` field. +In this sample, you configure an extractor for `myFlow` flow. The config +overrides the extractors for `context` and `output` fields and uses the default +logic for the `input` field. The specification of the evaluation extractors is as follows: -- `evaluators` field accepts an array of EvaluatorConfig objects, which are scoped by `flowName` -- `extractors` is an object that specifies the extractor overrides. The current supported keys in `extractors` are `[input, output, context]`. The acceptable value types are: - - `string` - this should be a step name, specified as a stirng. The output of this step is extracted for this key. - - `{ inputOf: string }` or `{ outputOf: string }` - These objects represent specific channels (input or output) of a step. For example, `{ inputOf: 'foo-step' }` would extract the input of step `foo-step` for this key. - - `(trace) => string;` - For further flexibility, you can provide a function that accepts a Genkit trace and returns a `string`, and specify the extraction logic inside this function. Refer to `genkit/genkit-tools/common/src/types/trace.ts` for the exact TraceData schema. - -Note: The extracted data for all these steps will be a JSON string. The tooling will parse this JSON string at the time of evaluation automatically. If providing a function extractor, make sure that the output is a valid JSON string. For eg: `"Hello, world!"` is not valid JSON; `"\"Hello, world!\""` is valid. +* `evaluators` field accepts an array of EvaluatorConfig objects, which are + scoped by `flowName` +* `extractors` is an object that specifies the extractor overrides. The + current supported keys in `extractors` are `[input, output, context]`. The + acceptable value types are: + * `string` - this should be a step name, specified as a string. The output + of this step is extracted for this key. + * `{ inputOf: string }` or `{ outputOf: string }` - These objects + represent specific channels (input or output) of a step. For example, `{ + inputOf: 'foo-step' }` would extract the input of step `foo-step` for + this key. + * `(trace) => string;` - For further flexibility, you can provide a + function that accepts a Genkit trace and returns a `string`, and specify + the extraction logic inside this function. Refer to + `genkit/genkit-tools/common/src/types/trace.ts` for the exact TraceData + schema. + +**Note:** The extracted data for all these steps will be a JSON string. The +tooling will parse this JSON string at the time of evaluation automatically. If +providing a function extractor, make sure that the output is a valid JSON +string. For example: `"Hello, world!"` is not valid JSON; `"\"Hello, world!\""` +is valid. ### Running on existing datasets @@ -170,7 +212,8 @@ To output to a different location, use the `--output` flag. genkit eval:flow menuSuggestionFlow --input testInputs.json --output customLabel_evalresult.json ``` -To run on a subset of the configured evaluators, use the `--evaluators` flag and provide a comma separated list of evaluators by name: +To run on a subset of the configured evaluators, use the `--evaluators` flag and +provide a comma-separated list of evaluators by name: ```posix-terminal genkit eval:run customLabel_dataset.json --evaluators=genkit/faithfulness,genkit/answer_relevancy @@ -178,27 +221,33 @@ genkit eval:run customLabel_dataset.json --evaluators=genkit/faithfulness,genkit ### Synthesizing test data using an LLM -Here's an example flow that uses a PDF file to generate possible questions -users might be asking about it. +Here's an example flow that uses a PDF file to generate possible questions users +might be asking about it. -```js -export const synthesizeQuestions = defineFlow( +```ts +import { genkit, run, z } from "genkit"; +import { googleAI, gemini15Flash } from "@genkit-ai/googleai"; +import { chunk } from "llm-chunk"; + +const ai = genkit({ plugins: [googleAI()] }); + +export const synthesizeQuestions = ai.defineFlow( { - name: 'synthesizeQuestions', - inputSchema: z.string().describe('PDF file path'), + name: "synthesizeQuestions", + inputSchema: z.string().describe("PDF file path"), outputSchema: z.array(z.string()), }, async (filePath) => { filePath = path.resolve(filePath); - const pdfTxt = await run('extract-text', () => extractText(filePath)); + const pdfTxt = await run("extract-text", () => extractText(filePath)); - const chunks = await run('chunk-it', async () => + const chunks = await run("chunk-it", async () => chunk(pdfTxt, chunkingConfig) ); const questions: string[] = []; for (var i = 0; i < chunks.length; i++) { - const qResponse = await generate({ + const qResponse = await ai.generate({ model: gemini15Flash, prompt: { text: `Generate one question about the text below: ${chunks[i]}`, diff --git a/docs/get-started.md b/docs/get-started.md index 694f89c029..35e12eeb26 100644 --- a/docs/get-started.md +++ b/docs/get-started.md @@ -1,189 +1,88 @@ # Get started -To get started with Firebase Genkit, install the Genkit CLI and run -`genkit init` in a Node.js project. The rest of this page shows you how. +This guide shows you how to get started with Genkit in a Node.js app. -## Requirements +## Prerequisites -Node.js 20 or later. +This guide assumes that you're familiar with building applications with Node.js. -Recommendation: The [`nvm`](https://github.com/nvm-sh/nvm) and -[`nvm-windows`](https://github.com/coreybutler/nvm-windows) tools are a -convenient way to install Node. +To complete this quickstart, make sure that your development environment meets +the following requirements: -## Install Genkit {:#install} +* Node.js v20+ +* npm -Install the Genkit CLI by running the following command: +## Install Genkit dependencies -```posix-terminal -npm i -g genkit -``` - -This command installs the Genkit CLI into your Node installation directory -so that it can be used outside of a Node project. - -## Create and explore a sample project {:#explore} - -1. Create a new Node project: - - ```posix-terminal - mkdir genkit-intro && cd genkit-intro - - npm init -y - ``` - - Look at package.json and make sure the `main` field is set to - `lib/index.js`. - -1. Initialize a Genkit project: +Install the following Genkit dependencies to use Genkit in your project: - ```posix-terminal - genkit init - ``` - - 1. Select your model: - - - {Gemini (Google AI)} - - The simplest way to get started is with Google AI Gemini API. Make sure - it's - [available in your region](https://ai.google.dev/available_regions). - - [Generate an API key](https://aistudio.google.com/app/apikey) for the - Gemini API using Google AI Studio. Then, set the `GOOGLE_GENAI_API_KEY` - environment variable to your key: - - ```posix-terminal - export GOOGLE_GENAI_API_KEY= - ``` +* `genkit` provides Genkit core capabilities. +* `@genkit-ai/googleai` provides access to the Google AI Gemini models. - - {Gemini (Vertex AI)} - - If the Google AI Gemini API is not available in your region, consider - using the Vertex AI API which also offers Gemini and other models. You - will need to have a billing-enabled Google Cloud project, enable AI - Platform API, and set some additional environment variables: - - ```posix-terminal - gcloud services enable aiplatform.googleapis.com - - export GCLOUD_PROJECT= - - export GCLOUD_LOCATION=us-central1 - ``` - - See https://cloud.google.com/vertex-ai/generative-ai/pricing for Vertex AI pricing. - - 1. Choose default answers to the rest of the questions, which will - initialize your project folder with some sample code. - - The `genkit init` command creates a sample source file, `index.ts`, which - defines a single flow, `menuSuggestionFlow`, that prompts an LLM to suggest - an item for a restaurant with a given theme. +```posix-terminal +npm install genkit @genkit-ai/googleai +``` - This file looks something like the following (the plugin configuration steps - might look different if you selected Vertex AI): +## Configure your model API key - ```ts - import * as z from 'zod'; +For this guide, we’ll show you how to use the Gemini API which provides a +generous free tier and does not require a credit card to get started. To use the +Gemini API, you'll need an API key. If you don't already have one, create a key +in Google AI Studio. - // Import the Genkit core libraries and plugins. - import { generate } from '@genkit-ai/ai'; - import { configureGenkit } from '@genkit-ai/core'; - import { defineFlow, startFlowsServer } from '@genkit-ai/flow'; - import { googleAI } from '@genkit-ai/googleai'; +[Get an API key from Google AI Studio](https://makersuite.google.com/app/apikey) - // Import models from the Google AI plugin. The Google AI API provides access to - // several generative models. Here, we import Gemini 1.5 Flash. - import { gemini15Flash } from '@genkit-ai/googleai'; +After you’ve created an API key, set the `GOOGLE_GENAI_API_KEY` environment +variable to your key with the following command: - configureGenkit({ - plugins: [ - // Load the Google AI plugin. You can optionally specify your API key - // by passing in a config object; if you don't, the Google AI plugin uses - // the value from the GOOGLE_GENAI_API_KEY environment variable, which is - // the recommended practice. - googleAI(), - ], - // Log debug output to tbe console. - logLevel: 'debug', - // Perform OpenTelemetry instrumentation and enable trace collection. - enableTracingAndMetrics: true, - }); +```posix-terminal +export GOOGLE_GENAI_API_KEY= +``` - // Define a simple flow that prompts an LLM to generate menu suggestions. - export const menuSuggestionFlow = defineFlow( - { - name: 'menuSuggestionFlow', - inputSchema: z.string(), - outputSchema: z.string(), - }, - async (subject) => { - // Construct a request and send it to the model API. - const llmResponse = await generate({ - prompt: `Suggest an item for the menu of a ${subject} themed restaurant`, - model: gemini15Flash, - config: { - temperature: 1, - }, - }); - - // Handle the response from the model API. In this sample, we just convert - // it to a string, but more complicated flows might coerce the response into - // structured output or chain the response into another LLM call, etc. - return llmResponse.text; - } - ); - - // Start a flow server, which exposes your flows as HTTP endpoints. This call - // must come last, after all of your plug-in configuration and flow definitions. - // You can optionally specify a subset of flows to serve, and configure some - // HTTP server options, but by default, the flow server serves all defined flows. - startFlowsServer(); - ``` - - As you build out your app's AI features with Genkit, you will likely - create flows with multiple steps such as input preprocessing, more - sophisticated prompt construction, integrating external information - sources for retrieval-augmented generation (RAG), and more. - -1. Now you can run and explore Genkit features and the sample project locally - on your machine. Download and start the Genkit Developer UI: - - ```posix-terminal - genkit start - ``` - - Welcome to Genkit Developer UI - - The Genkit Developer UI is now running on your machine. When you run models - or flows in the next step, your machine will perform the orchestration tasks - needed to get the steps of your flow working together; calls to external - services such as the Gemini API will continue to be made against live - servers. - - Also, because you are in a dev environment, Genkit will store traces and - flow state in local files. - -1. The Genkit Developer UI downloads and opens automatically when you run the - `genkit start` command. - - The Developer UI lets you see which flows you have defined and models you - configured, run them, and examine traces of previous runs. Try out some of - these features: - - - On the **Run** tab, you will see a list of all of the flows that you have - defined and any models that have been configured by plugins. - - Click **menuSuggestionFlow** and try running it with some input text (for example, - `"cat"`). If all goes well, you'll be rewarded with a menu suggestion for a cat - themed restaurant. - - - On the **Inspect** tab, you'll see a history of flow executions. For each - flow, you can see the parameters that were passed to the flow and a - trace of each step as they ran. +Note: While this tutorial uses the Gemini API from AI Studio, Genkit supports a +wide variety of model providers including +[Gemini from Vertex AI](/docs/genkit/plugins/vertex-ai#generative_ai_models), +Anthropic’s Claude 3 models and Llama 3.1 through the +[Vertex AI Model Garden](/docs/genkit/plugins/vertex-ai#anthropic_claude_3_on_vertex_ai_model_garden), +open source models through +[Ollama](/docs/genkit/plugins/ollama), and several other +[community-supported providers](/docs/genkit/models#models-supported) like +OpenAI and Cohere. + +## Make your first request + +Get started with Genkit in just a few lines of simple code. + +```ts +// import the Genkit and Google AI plugin libraries +import { genkit } from 'genkit'; +import { googleAI, gemini15Flash } from '@genkit-ai/googleai'; + +// configure a Genkit instance +const ai = genkit({ + plugins: [googleAI()], + model: gemini15Flash // set default model +}); + +// make a generation request +const response = await ai.generate('Hello, Gemini!'); +console.log(response.text); +``` ## Next steps -Check out how to build and deploy your Genkit app with [Firebase](firebase.md), -[Cloud Run](cloud-run.md), or any [Node.js platform](deploy-node.md). +Now that you’re set up to make model requests with Genkit, learn how to use more +Genkit capabilities to build your AI-powered apps and workflows. To get started +with additional Genkit capabilities, see the following guides: + +* [Developer tools](/docs/genkit/devtools): Learn how to set up and use + Genkit’s CLI and developer UI to help you locally test and debug your app. +* [Generating content](/docs/genkit/models): Learn how to use Genkit’s unified + generation API to generate text and structured data from any supported + model. +* [Creating flows](/docs/genkit/flows): Learn how to use special Genkit + functions, called flows, that provide end-to-end observability for workflows + and rich debugging from Genkit tooling. +* [Prompting models](/docs/genkit/prompts): Learn how Genkit lets you treat + prompt templates as functions, encapsulating model configurations and + input/output schema. diff --git a/docs/rag.md b/docs/rag.md index 23b459ed01..838edf9ed2 100644 --- a/docs/rag.md +++ b/docs/rag.md @@ -1,16 +1,17 @@ # Retrieval-augmented generation (RAG) -Firebase Genkit provides abstractions that help you build retrieval-augmented generation -(RAG) flows, as well as plugins that provide integrations with related tools. +Firebase Genkit provides abstractions that help you build retrieval-augmented +generation (RAG) flows, as well as plugins that provide integrations with +related tools. ## What is RAG? Retrieval-augmented generation is a technique used to incorporate external sources of information into an LLM’s responses. It's important to be able to do -so because, while LLMs are typically trained on a broad body of -material, practical use of LLMs often requires specific domain knowledge (for -example, you might want to use an LLM to answer customers' questions about your -company’s products). +so because, while LLMs are typically trained on a broad body of material, +practical use of LLMs often requires specific domain knowledge (for example, you +might want to use an LLM to answer customers' questions about your company’s +products). One solution is to fine-tune the model using more specific data. However, this can be expensive both in terms of compute cost and in terms of the effort needed @@ -25,22 +26,22 @@ to Lisa?" This approach has several advantages: -- It can be more cost effective because you don't have to retrain the model. -- You can continuously update your data source and the LLM can immediately make - use of the updated information. -- You now have the potential to cite references in your LLM's responses. +* It can be more cost-effective because you don't have to retrain the model. +* You can continuously update your data source and the LLM can immediately + make use of the updated information. +* You now have the potential to cite references in your LLM's responses. On the other hand, using RAG naturally means longer prompts, and some LLM API services charge for each input token you send. Ultimately, you must evaluate the cost tradeoffs for your applications. RAG is a very broad area and there are many different techniques used to achieve -the best quality RAG. The core Genkit framework offers two main abstractions to -help you do RAG: +the best quality RAG. The core Genkit framework offers three main abstractions +to help you do RAG: -- Indexers: add documents to an "index". -- Embedders: transforms documents into a vector representation -- Retrievers: retrieve documents from an "index", given a query. +* Indexers: add documents to an "index". +* Embedders: transforms documents into a vector representation +* Retrievers: retrieve documents from an "index", given a query. These definitions are broad on purpose because Genkit is un-opinionated about what an "index" is or how exactly documents are retrieved from it. Genkit only @@ -62,19 +63,18 @@ Before you can retrieve documents for the purpose of generation, you need to ingest them into your document index. A typical ingestion flow does the following: -1. Split up large documents into smaller documents so that only relevant - portions are used to augment your prompts – "chunking". This is necessary - because many LLMs have a limited context window, making it impractical to - include entire documents with a prompt. +1. Split up large documents into smaller documents so that only relevant + portions are used to augment your prompts – "chunking". This is necessary + because many LLMs have a limited context window, making it impractical to + include entire documents with a prompt. Genkit doesn't provide built-in chunking libraries; however, there are open source libraries available that are compatible with Genkit. -1. Generate embeddings for each chunk. Depending on the database you're using, - you might explicitly do this with an embedding generation model, or you - might use the embedding generator provided by the database. - -1. Add the text chunk and its index to the database. +2. Generate embeddings for each chunk. Depending on the database you're using, + you might explicitly do this with an embedding generation model, or you might + use the embedding generator provided by the database. +3. Add the text chunk and its index to the database. You might run your ingestion flow infrequently or only once if you are working with a stable source of data. On the other hand, if you are working with data @@ -83,30 +83,34 @@ example, in a Cloud Firestore trigger, whenever a document is updated). ### Embedders -An embedder is a function that takes content (text, images, audio, etc.) and creates a numeric vector that encodes the semantic meaning of the original content. As mentioned above, embedders are leveraged as part of the process of indexing, however, they can also be used independently to create embeddings without an index. +An embedder is a function that takes content (text, images, audio, etc.) and +creates a numeric vector that encodes the semantic meaning of the original +content. As mentioned above, embedders are leveraged as part of the process of +indexing, however, they can also be used independently to create embeddings +without an index. ### Retrievers A retriever is a concept that encapsulates logic related to any kind of document retrieval. The most popular retrieval cases typically include retrieval from -vector stores, however, in Genkit a retriever can be any function that returns data. +vector stores, however, in Genkit a retriever can be any function that returns +data. -To create a retriever, you can use one of the provided implementations or -create your own. +To create a retriever, you can use one of the provided implementations or create +your own. ## Supported indexers, retrievers, and embedders Genkit provides indexer and retriever support through its plugin system. The following plugins are officially supported: -- [Cloud Firestore vector store](plugins/firebase.md) -- [Vertex AI Vector Search](plugins/vertex-ai.md) -- [Chroma DB](plugins/chroma.md) vector database -- [Pinecone](plugins/pinecone.md) cloud vector database +* [Cloud Firestore vector store](plugins/firebase.md) +* [Vertex AI Vector Search](plugins/vertex-ai.md) +* [Chroma DB](plugins/chroma.md) vector database +* [Pinecone](plugins/pinecone.md) cloud vector database -In addition, Genkit supports the following vector stores through predefined -code templates, which you can customize for your database configuration and -schema: +In addition, Genkit supports the following vector stores through predefined code +templates, which you can customize for your database configuration and schema: - PostgreSQL with [`pgvector`](templates/pgvector.md) @@ -122,8 +126,9 @@ Embedding model support is provided through the following plugins: ## Defining a RAG Flow -The following examples show how you could ingest a collection of restaurant menu PDF documents -into a vector database and retrieve them for use in a flow that determines what food items are available. +The following examples show how you could ingest a collection of restaurant menu +PDF documents into a vector database and retrieve them for use in a flow that +determines what food items are available. ### Install dependencies for processing PDFs @@ -140,18 +145,19 @@ import { devLocalIndexerRef, devLocalVectorstore, } from '@genkit-ai/dev-local-vectorstore'; -import { textEmbeddingGecko, vertexAI } from '@genkit-ai/vertexai'; +import { textEmbedding004, vertexAI } from '@genkit-ai/vertexai'; +import { z, genkit } from 'genkit'; -configureGenkit({ +const ai = genkit({ plugins: [ - // vertexAI provides the textEmbeddingGecko embedder + // vertexAI provides the textEmbedding004 embedder vertexAI(), // the local vector store requires an embedder to translate from text to vector devLocalVectorstore([ { indexName: 'menuQA', - embedder: textEmbeddingGecko, + embedder: textEmbedding004, }, ]), ], @@ -160,26 +166,26 @@ configureGenkit({ ### Define an Indexer -The following example shows how to create an indexer to ingest a collection of PDF documents -and store them in a local vector database. +The following example shows how to create an indexer to ingest a collection of +PDF documents and store them in a local vector database. -It uses the local file-based vector similarity retriever -that Genkit provides out-of-the box for simple testing and prototyping (_do not -use in production_) +It uses the local file-based vector similarity retriever that Genkit provides +out-of-the-box for simple testing and prototyping (_do not use in production_) #### Create the indexer ```ts -import { devLocalIndexerRef } from '@genkit-ai/dev-local-vectorstore'; - export const menuPdfIndexer = devLocalIndexerRef('menuQA'); ``` #### Create chunking config -This example uses the `llm-chunk` library which provides a simple text splitter to break up documents into segments that can be vectorized. +This example uses the `llm-chunk` library which provides a simple text splitter +to break up documents into segments that can be vectorized. -The following definition configures the chunking function to gaurantee a document segment of between 1000 and 2000 characters, broken at the end of a sentence, with an overlap between chunks of 100 characters. +The following definition configures the chunking function to guarantee a +document segment of between 1000 and 2000 characters, broken at the end of a +sentence, with an overlap between chunks of 100 characters. ```ts const chunkingConfig = { @@ -191,21 +197,26 @@ const chunkingConfig = { } as any; ``` -More chunking options for this library can be found in the [llm-chunk documentation](https://www.npmjs.com/package/llm-chunk). +More chunking options for this library can be found in the [llm-chunk +documentation](https://www.npmjs.com/package/llm-chunk). #### Define your indexer flow ```ts -import { index } from '@genkit-ai/ai'; -import { Document } from '@genkit-ai/ai/retriever'; -import { defineFlow, run } from '@genkit-ai/flow'; -import { readFile } from 'fs/promises'; +import { Document } from 'genkit/retriever'; import { chunk } from 'llm-chunk'; +import { readFile } from 'fs/promises'; import path from 'path'; import pdf from 'pdf-parse'; -import * as z from 'zod'; -export const indexMenu = defineFlow( +async function extractTextFromPdf(filePath: string) { + const pdfFile = path.resolve(filePath); + const dataBuffer = await readFile(pdfFile); + const data = await pdf(dataBuffer); + return data.text; +} + +export const indexMenu = ai.defineFlow( { name: 'indexMenu', inputSchema: z.string().describe('PDF file path'), @@ -230,28 +241,22 @@ export const indexMenu = defineFlow( }); // Add documents to the index. - await index({ + await ai.index({ indexer: menuPdfIndexer, documents, }); } ); - -async function extractTextFromPdf(filePath: string) { - const pdfFile = path.resolve(filePath); - const dataBuffer = await readFile(pdfFile); - const data = await pdf(dataBuffer); - return data.text; -} ``` #### Run the indexer flow ```posix-terminal -genkit flow:run indexMenu "'../pdfs'" +genkit flow:run indexMenu "'menu.pdf'" ``` -After running the `indexMenu` flow, the vector database will be seeded with documents and ready to be used in Genkit flows with retrieval steps. +After running the `indexMenu` flow, the vector database will be seeded with +documents and ready to be used in Genkit flows with retrieval steps. ### Define a flow with retrieval @@ -260,44 +265,36 @@ the indexer example, this example uses Genkit's file-based vector retriever, which you should not use in production. ```ts -import { generate } from '@genkit-ai/ai'; -import { retrieve } from '@genkit-ai/ai/retriever'; import { devLocalRetrieverRef } from '@genkit-ai/dev-local-vectorstore'; -import { defineFlow } from '@genkit-ai/flow'; -import { gemini15Flash } from '@genkit-ai/vertexai'; -import * as z from 'zod'; // Define the retriever reference export const menuRetriever = devLocalRetrieverRef('menuQA'); -export const menuQAFlow = defineFlow( +export const menuQAFlow = ai.defineFlow( { name: 'menuQA', inputSchema: z.string(), outputSchema: z.string() }, async (input: string) => { // retrieve relevant documents - const docs = await retrieve({ + const docs = await ai.retrieve({ retriever: menuRetriever, query: input, options: { k: 3 }, }); // generate a response - const llmResponse = await generate({ - model: gemini15Flash, + const { text } = await ai.generate({ prompt: ` - You are acting as a helpful AI assistant that can answer - questions about the food available on the menu at Genkit Grub Pub. - - Use only the context provided to answer the question. - If you don't know, do not make up an answer. - Do not add or change items on the menu. - - Question: ${input} - `, - context: docs, +You are acting as a helpful AI assistant that can answer +questions about the food available on the menu at Genkit Grub Pub. + +Use only the context provided to answer the question. +If you don't know, do not make up an answer. +Do not add or change items on the menu. + +Question: ${input}`, + docs, }); - const output = llmResponse.text; - return output; + return text; } ); ``` @@ -315,38 +312,37 @@ RAG techniques (such as reranking or prompt extensions) on top. Simple retrievers let you easily convert existing code into retrievers: -```javascript -import { - defineSimpleRetriever, - retrieve -} from '@genkit-ai/ai/retriever'; -import { searchEmails } from './db'; -import { z } from 'zod'; - -defineSimpleRetriever({ - name: 'myDatabase', - configSchema: z.object({ - limit: z.number().optional() - }).optional(), - // we'll extract "message" from the returned email item - content: 'message', - // and several keys to use as metadata - metadata: ['from', 'to', 'subject'], -} async (query, config) => { - const result = await searchEmails(query.text, {limit: config.limit}); - return result.data.emails; -}); +```ts +import { z } from "genkit"; +import { searchEmails } from "./db"; + +ai.defineSimpleRetriever( + { + name: "myDatabase", + configSchema: z + .object({ + limit: z.number().optional(), + }) + .optional(), + // we'll extract "message" from the returned email item + content: "message", + // and several keys to use as metadata + metadata: ["from", "to", "subject"], + }, + async (query, config) => { + const result = await searchEmails(query.text, { limit: config.limit }); + return result.data.emails; + } +); ``` ### Custom Retrievers -```javascript +```ts import { CommonRetrieverOptionsSchema, - defineRetriever, - retrieve, -} from '@genkit-ai/ai/retriever'; -import * as z from 'zod'; +} from 'genkit/retriever'; +import { z } from 'genkit'; export const menuRetriever = devLocalRetrieverRef('menuQA'); @@ -354,14 +350,14 @@ const advancedMenuRetrieverOptionsSchema = CommonRetrieverOptionsSchema.extend({ preRerankK: z.number().max(1000), }); -const advancedMenuRetriever = defineRetriever( +const advancedMenuRetriever = ai.defineRetriever( { name: `custom/advancedMenuRetriever`, configSchema: advancedMenuRetrieverOptionsSchema, }, async (input, options) => { const extendedPrompt = await extendPrompt(input); - const docs = await retrieve({ + const docs = await ai.retrieve({ retriever: menuRetriever, query: extendedPrompt, options: { k: options.preRerankK || 10 }, @@ -377,8 +373,8 @@ not provided by the framework) And then you can just swap out your retriever: -```javascript -const docs = await retrieve({ +```ts +const docs = await ai.retrieve({ retriever: advancedRetriever, query: input, options: { preRerankK: 7, k: 3 }, @@ -387,19 +383,22 @@ const docs = await retrieve({ ### Rerankers and Two-Stage Retrieval -A reranking model — also known as a cross-encoder — is a type of model that, given a query and document, will output a similarity score. We use this score to reorder the documents by relevance to our query. Reranker APIs take a list of documents (for example the output of a retriever) and reorders the documents based on their relevance to the query. This step can be useful for fine-tuning the results and ensuring that the most pertinent information is used in the prompt provided to a generative model. - +A reranking model — also known as a cross-encoder — is a type of model that, +given a query and document, will output a similarity score. We use this score to +reorder the documents by relevance to our query. Reranker APIs take a list of +documents (for example the output of a retriever) and reorders the documents +based on their relevance to the query. This step can be useful for fine-tuning +the results and ensuring that the most pertinent information is used in the +prompt provided to a generative model. #### Reranker Example -A reranker in Genkit is defined in a similar syntax to retrievers and indexers. Here is an example using a reranker in Genkit. This flow reranks a set of documents based on their relevance to the provided query using a predefined Vertex AI reranker. +A reranker in Genkit is defined in a similar syntax to retrievers and indexers. +Here is an example using a reranker in Genkit. This flow reranks a set of +documents based on their relevance to the provided query using a predefined +Vertex AI reranker. ```ts -import { rerank } from '@genkit-ai/ai/reranker'; -import { Document } from '@genkit-ai/ai/retriever'; -import { defineFlow } from '@genkit-ai/flow'; -import * as z from 'zod'; - const FAKE_DOCUMENT_CONTENT = [ 'pythagorean theorem', 'e=mc^2', @@ -410,7 +409,7 @@ const FAKE_DOCUMENT_CONTENT = [ 'harry potter', ]; -export const rerankFlow = defineFlow( +export const rerankFlow = ai.defineFlow( { name: 'rerankFlow', inputSchema: z.object({ query: z.string() }), @@ -423,32 +422,35 @@ export const rerankFlow = defineFlow( }, async ({ query }) => { const documents = FAKE_DOCUMENT_CONTENT.map((text) => - Document.fromText(text) + ({ content: text }) ); - const rerankedDocuments = await rerank({ + const rerankedDocuments = await ai.rerank({ reranker: 'vertexai/semantic-ranker-512', - query: Document.fromText(query), + query: ({ content: query }), documents, }); return rerankedDocuments.map((doc) => ({ - text: doc.text, + text: doc.content, score: doc.metadata.score, })); } ); ``` -This reranker uses the Vertex AI genkit plugin with `semantic-ranker-512` to score and rank documents. The higher the score, the more relevant the document is to the query. + +This reranker uses the Vertex AI genkit plugin with `semantic-ranker-512` to +score and rank documents. The higher the score, the more relevant the document +is to the query. #### Custom Rerankers -You can also define custom rerankers to suit your specific use case. This is helpful when you need to rerank documents using your own custom logic or a custom model. Here’s a simple example of defining a custom reranker: -```typescript -import { defineReranker } from '@genkit-ai/ai/reranker'; -import * as z from 'zod'; +You can also define custom rerankers to suit your specific use case. This is +helpful when you need to rerank documents using your own custom logic or a +custom model. Here’s a simple example of defining a custom reranker: -export const customReranker = defineReranker( +```ts +export const customReranker = ai.defineReranker( { name: 'custom/reranker', configSchema: z.object({ @@ -469,4 +471,7 @@ export const customReranker = defineReranker( } ); ``` -Once defined, this custom reranker can be used just like any other reranker in your RAG flows, giving you flexibility to implement advanced reranking strategies. \ No newline at end of file + +Once defined, this custom reranker can be used just like any other reranker in +your RAG flows, giving you flexibility to implement advanced reranking +strategies. diff --git a/docs/tool-calling.md b/docs/tool-calling.md index 0a846718cb..70aaa1dbbb 100644 --- a/docs/tool-calling.md +++ b/docs/tool-calling.md @@ -9,131 +9,194 @@ The use cases of tool calling generally fall into a few themes: **Giving an LLM access to information it wasn't trained with** -- Frequently changing information, such as a restaurant's daily menu or a - store's inventory status. -- Information specific to your app domain, such as product information. +* Frequently changing information, such as a stock price or the current + weather. +* Information specific to your app domain, such as product information or user + profiles. Note the overlap with [retrieval augmented generation](rag) (RAG), which is also a way to let an LLM integrate factual information into its generations. RAG is a -heavier solution that is most suited when you have a large amount of -information or the information that's most relevant to a prompt is ambiguous. On -the other hand, if retrieving the information the LLM needs is a simple function -call or database lookup, tool calling is more appropriate. +heavier solution that is most suited when you have a large amount of information +or the information that's most relevant to a prompt is ambiguous. On the other +hand, if retrieving the information the LLM needs is a simple function call or +database lookup, tool calling is more appropriate. **Introducing a degree of determinism into an LLM workflow** -- Performing calculations that the LLM cannot reliably complete itself. -- Forcing an LLM to generate verbatim text under certain circumstances, such as - when responding to a question about an app's terms of service. +* Performing calculations that the LLM cannot reliably complete itself. +* Forcing an LLM to generate verbatim text under certain circumstances, such + as when responding to a question about an app's terms of service. **Performing an action when initiated by an LLM** -- Turning on and off lights in an LLM-powered home assistant -- Reserving table reservations in an LLM-powered restaurant agent +* Turning on and off lights in an LLM-powered home assistant +* Reserving table reservations in an LLM-powered restaurant agent -### Before you begin {:#you-begin} +## Before you begin {:#you-begin} If you want to run the code examples on this page, first complete the steps in the [Getting started](get-started) guide. All of the examples assume that you have already set up a project with Genkit dependencies installed. -This page discusses one of the advanced features of Genkit model abstraction and -generate() function, so before you dive too deeply, you should be familiar with -the content on the [Generating content with AI models](models) page. You should -also be familiar with Genkit's system for defining input and output schemas, -which is discussed on the [Flows](flows) page. +This page discusses one of the advanced features of Genkit model abstraction, so +before you dive too deeply, you should be familiar with the content on the +[Generating content with AI models](models) page. You should also be familiar +with Genkit's system for defining input and output schemas, which is discussed +on the [Flows](flows) page. -### Overview of tool calling {:#overview-tool} +## Overview of tool calling {:#overview-tool} At a high level, this is what a typical tool-calling interaction with an LLM looks like: -1. The calling application prompts the LLM with a request and also includes in - the prompt a list of tools the LLM can use to generate a response. -1. The LLM either generates a complete response or generates a tool call - request in a specific format. -1. If the caller receives a complete response, the request is fulfilled and the - interaction ends; but if the caller receives a tool call, it performs - whatever logic is appropriate and sends a new request to the LLM containing - the original prompt or some variation of it as well as the result of the - tool call. -1. The LLM handles the new prompt as in Step 2. +1. The calling application prompts the LLM with a request and also includes in + the prompt a list of tools the LLM can use to generate a response. +2. The LLM either generates a complete response or generates a tool call request + in a specific format. +3. If the caller receives a complete response, the request is fulfilled and the + interaction ends; but if the caller receives a tool call, it performs + whatever logic is appropriate and sends a new request to the LLM containing + the original prompt or some variation of it as well as the result of the tool + call. +4. The LLM handles the new prompt as in Step 2. For this to work, several requirements must be met: -- The model must be trained to make tool requests when it's needed to complete a - prompt. Most of the larger models provided through web APIs, such as Gemini - and Claude, can do this, but smaller and more specialized models often cannot. - Genkit will throw an error if you try to provide tools to a model that doesn't - support it. -- The calling application must provide tool definitions to the model in the - format it expects. -- The calling application must prompt the model to generate tool calling - requests in the format the application expects. +* The model must be trained to make tool requests when it's needed to complete + a prompt. Most of the larger models provided through web APIs, such as + Gemini and Claude, can do this, but smaller and more specialized models + often cannot. Genkit will throw an error if you try to provide tools to a + model that doesn't support it. +* The calling application must provide tool definitions to the model in the + format it expects. +* The calling application must prompt the model to generate tool calling + requests in the format the application expects. -### Tool calling with Genkit {:#tool-calling} +## Tool calling with Genkit {:#tool-calling} Genkit provides a single interface for tool calling with models that support it. Each model plugin ensures that the last two of the above criteria are met, and -the `generate()` function automatically carries out the tool calling loop -described earlier. +the Genkit instance's `generate()` function automatically carries out the tool +calling loop described earlier. -#### Model support +### Model support Tool calling support depends on the model, the model API, and the Genkit plugin. Consult the relevant documentation to determine if tool calling is likely to be supported. In addition: -- Genkit will throw an error if you try to provide tools to a model that doesn't - support it. -- If the plugin exports model references, the `info.supports.tools` property - will indicate if it supports tool calling. +* Genkit will throw an error if you try to provide tools to a model that + doesn't support it. +* If the plugin exports model references, the `info.supports.tools` property + will indicate if it supports tool calling. -#### Defining tools +### Defining tools -Use the `defineTool()` function to write tool definitions: +Use the Genkit instance's `defineTool()` function to write tool definitions: ```ts -const specialToolInputSchema = z.object({ meal: z.enum(["breakfast", "lunch", "dinner"]) }); -const specialTool = defineTool( +import { genkit, z } from 'genkit'; +import { googleAI, gemini15Flash } from '@genkit-ai/google-ai'; + +const ai = genkit({ + plugins: [googleAI()], + model: gemini15Flash, +}); + +const getWeather = ai.defineTool( { - name: "specialTool", - description: "Retrieves today's special for the given meal", - inputSchema: specialToolInputSchema, + name: "getWeather", + description: "Gets the current weather in a given location", + inputSchema: z.object({ + location: z.string().describe('The location to get the current weather for') + }), outputSchema: z.string(), }, - async ({ meal }): Promise => { - // Retrieve up-to-date information and return it. Here, we just return a - // fixed value. - return "Baked beans on toast"; + async (input) => { + // Here, we would typically make an API call or database query. For this + // example, we just return a fixed value. + return "The current weather in ${input.location} is 63°F and sunny."; } ); ``` The syntax here looks just like the `defineFlow()` syntax; however, all four of the `name`, `description`, `inputSchema`, and `outputSchema` parameters are -required. -When writing a tool definition, take special care with the wording and +required. When writing a tool definition, take special care with the wording and descriptiveness of these parameters, as they are vital for the LLM to effectively make use of the available tools. -#### Including tools with your prompts +### Using tools -After you've defined your tools, specify them in the tools parameter of -`generate()`: +Include defined tools in your prompts to generate content. -```ts -const llmResponse = await generate({ - model: gemini15Flash, - prompt, - tools: [specialTool], -}); -``` +* {Generate} + + ```ts + const response = await ai.generate({ + prompt: "What is the weather in Baltimore?", + tools: [getWeather], + }); + ``` + +* {definePrompt} + + ```ts + const weatherPrompt = ai.definePrompt( + { + name: "weatherPrompt", + tools: [getWeather], + }, + "What is the weather in {{location}}?" + ); + + const response = await weatherPrompt({ location: "Baltimore" }); + ``` + +* {Prompt file} + + ```none + --- + system: "Answer questions using the tools you have." + tools: [getWeather] + input: + schema: + location: string + --- + + "What is the weather in {{location}}?" + ``` -You can make multiple tools available; the LLM will call the tools as necessary -to complete the prompt. + Then you can execute the prompt in your code as follows: -#### Explicitly handling tool calls + ```ts + // assuming prompt file is named weatherPrompt.prompt + const weatherPrompt = ai.prompt("weatherPrompt"); + + const response = await weatherPrompt({ location: "Baltimore" }); + ``` + +* {Chat} + + ```ts + const chat = ai.chat({ + system: "Answer questions using the tools you have.", + tools: [getWeather], + }); + + const response = await chat.send("What is the weather in Baltimore?"); + + // Or, specify tools that are message-specific + const response = await chat.send({ + prompt: "What is the weather in Baltimore?", + tools: [getWeather], + }); + ``` + +Genkit will automatically handle the tool call if the LLM needs to use the +`getWeather` tool to answer the prompt. + +### Explicitly handling tool calls By default, Genkit repeatedly calls the LLM until every tool call has been resolved. If you want more control over this tool calling loop, for example to @@ -141,16 +204,25 @@ apply more complicated logic, set the `returnToolRequests` parameter to `true`. Now it's your responsibility to ensure all of the tool requests are fulfilled: ```ts -let generateOptions: GenerateOptions = { - model: gemini15Flash, - prompt, - tools: [specialTool], +const getWeather = ai.defineTool( + { + // ... tool definition ... + }, + async ({ location }) => { + // ... tool implementation ... + }, +); + +const generateOptions: GenerateOptions = { + prompt: "What's the weather like in Baltimore?", + tools: [getWeather], returnToolRequests: true, }; + let llmResponse; while (true) { - llmResponse = await generate(generateOptions); - const toolRequests = llmResponse.toolRequests(); + llmResponse = await ai.generate(generateOptions); + const toolRequests = llmResponse.toolRequests; if (toolRequests.length < 1) { break; } @@ -162,14 +234,15 @@ while (true) { toolResponse: { name: part.toolRequest.name, ref: part.toolRequest.ref, - output: await specialTool(specialToolInputSchema.parse(part.toolRequest?.input)), + output: await getWeather(part.toolRequest.input), }, }; default: - throw Error('Tool not found'); - } - })); - generateOptions.history = llmResponse.messages; - generateOptions.prompt = toolResponses; + throw Error("Tool not found"); + } + }) + ); + generateOptions.messages = llmResponse.messages; + generateOptions.prompt = toolResponses; } ``` From 497e808622071148f942554a39fde5c0f98ecc84 Mon Sep 17 00:00:00 2001 From: Kevin Cheung Date: Tue, 12 Nov 2024 14:02:03 -0800 Subject: [PATCH 2/2] ' --- docs/tool-calling.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/tool-calling.md b/docs/tool-calling.md index 70aaa1dbbb..93598e1ad7 100644 --- a/docs/tool-calling.md +++ b/docs/tool-calling.md @@ -105,8 +105,8 @@ const ai = genkit({ const getWeather = ai.defineTool( { - name: "getWeather", - description: "Gets the current weather in a given location", + name: 'getWeather', + description: 'Gets the current weather in a given location', inputSchema: z.object({ location: z.string().describe('The location to get the current weather for') }), @@ -115,7 +115,7 @@ const getWeather = ai.defineTool( async (input) => { // Here, we would typically make an API call or database query. For this // example, we just return a fixed value. - return "The current weather in ${input.location} is 63°F and sunny."; + return 'The current weather in ${input.location} is 63°F and sunny.'; } ); ``` @@ -134,7 +134,7 @@ Include defined tools in your prompts to generate content. ```ts const response = await ai.generate({ - prompt: "What is the weather in Baltimore?", + prompt: 'What is the weather in Baltimore?', tools: [getWeather], }); ``` @@ -144,13 +144,13 @@ Include defined tools in your prompts to generate content. ```ts const weatherPrompt = ai.definePrompt( { - name: "weatherPrompt", + name: 'weatherPrompt', tools: [getWeather], }, - "What is the weather in {{location}}?" + 'What is the weather in {{location}}?' ); - const response = await weatherPrompt({ location: "Baltimore" }); + const response = await weatherPrompt({ location: 'Baltimore' }); ``` * {Prompt file} @@ -164,31 +164,31 @@ Include defined tools in your prompts to generate content. location: string --- - "What is the weather in {{location}}?" + What is the weather in {{location}}? ``` Then you can execute the prompt in your code as follows: ```ts // assuming prompt file is named weatherPrompt.prompt - const weatherPrompt = ai.prompt("weatherPrompt"); + const weatherPrompt = ai.prompt('weatherPrompt'); - const response = await weatherPrompt({ location: "Baltimore" }); + const response = await weatherPrompt({ location: 'Baltimore' }); ``` * {Chat} ```ts const chat = ai.chat({ - system: "Answer questions using the tools you have.", + system: 'Answer questions using the tools you have.', tools: [getWeather], }); - const response = await chat.send("What is the weather in Baltimore?"); + const response = await chat.send('What is the weather in Baltimore?'); // Or, specify tools that are message-specific const response = await chat.send({ - prompt: "What is the weather in Baltimore?", + prompt: 'What is the weather in Baltimore?', tools: [getWeather], }); ``` @@ -214,7 +214,7 @@ const getWeather = ai.defineTool( ); const generateOptions: GenerateOptions = { - prompt: "What's the weather like in Baltimore?", + prompt: 'What's the weather like in Baltimore?', tools: [getWeather], returnToolRequests: true, }; @@ -229,7 +229,7 @@ while (true) { const toolResponses: ToolResponsePart[] = await Promise.all( toolRequests.map(async (part) => { switch (part.toolRequest.name) { - case "specialTool": + case 'specialTool': return { toolResponse: { name: part.toolRequest.name, @@ -238,7 +238,7 @@ while (true) { }, }; default: - throw Error("Tool not found"); + throw Error('Tool not found'); } }) );