diff --git a/js/samples/coffee-shop/package.json b/js/samples/coffee-shop/package.json deleted file mode 100644 index 46031216a8..0000000000 --- a/js/samples/coffee-shop/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "coffee-shop", - "version": "1.0.0", - "description": "Samples for a coffeeshop", - "main": "lib/index.js", - "scripts": { - "start": "node lib/index.js", - "compile": "tsc", - "build": "pnpm build:clean && pnpm compile", - "build:clean": "rm -rf ./lib", - "build:watch": "tsc --watch", - "build-and-run": "pnpm build && node lib/index.js" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "@genkit-ai/ai": "workspace:*", - "@genkit-ai/core": "workspace:*", - "@genkit-ai/dotprompt": "workspace:*", - "@genkit-ai/flow": "workspace:*", - "genkitx-chromadb": "workspace:*", - "@genkit-ai/dev-local-vectorstore": "workspace:*", - "@genkit-ai/firebase": "workspace:*", - "@genkit-ai/googleai": "workspace:*", - "genkitx-ollama": "workspace:*", - "genkitx-pinecone": "workspace:*", - "@genkit-ai/evaluator": "workspace:*", - "@genkit-ai/vertexai": "workspace:*", - "zod": "^3.22.4" - }, - "devDependencies": { - "typescript": "^5.3.3" - } -} diff --git a/samples/js-coffee-shop/README.md b/samples/js-coffee-shop/README.md new file mode 100644 index 0000000000..618a80f5e8 --- /dev/null +++ b/samples/js-coffee-shop/README.md @@ -0,0 +1,6 @@ +## Running the sample + +```bash +npm i +genkit start +``` diff --git a/samples/js-coffee-shop/package.json b/samples/js-coffee-shop/package.json new file mode 100644 index 0000000000..42ff52dc3f --- /dev/null +++ b/samples/js-coffee-shop/package.json @@ -0,0 +1,35 @@ +{ + "name": "coffee-shop", + "version": "1.0.0", + "description": "Samples for a coffeeshop", + "main": "lib/index.js", + "scripts": { + "start": "node lib/index.js", + "compile": "tsc", + "build": "npm run build:clean && npm run compile", + "build:clean": "rm -rf ./lib", + "build:watch": "tsc --watch", + "build-and-run": "npm run build && node lib/index.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@genkit-ai/ai": "~0.5.0-rc.11", + "@genkit-ai/core": "~0.5.0-rc.11", + "@genkit-ai/dotprompt": "~0.5.0-rc.11", + "@genkit-ai/flow": "~0.5.0-rc.11", + "genkitx-chromadb": "~0.5.0-rc.11", + "@genkit-ai/dev-local-vectorstore": "~0.5.0-rc.11", + "@genkit-ai/firebase": "~0.5.0-rc.11", + "@genkit-ai/googleai": "~0.5.0-rc.11", + "genkitx-ollama": "~0.5.0-rc.11", + "genkitx-pinecone": "~0.5.0-rc.11", + "@genkit-ai/evaluator": "~0.5.0-rc.11", + "@genkit-ai/vertexai": "~0.5.0-rc.11", + "zod": "^3.22.4" + }, + "devDependencies": { + "typescript": "^5.3.3" + } +} diff --git a/js/samples/coffee-shop/src/index.ts b/samples/js-coffee-shop/src/index.ts similarity index 100% rename from js/samples/coffee-shop/src/index.ts rename to samples/js-coffee-shop/src/index.ts diff --git a/js/samples/coffee-shop/src/input.json b/samples/js-coffee-shop/src/input.json similarity index 100% rename from js/samples/coffee-shop/src/input.json rename to samples/js-coffee-shop/src/input.json diff --git a/js/samples/coffee-shop/tsconfig.json b/samples/js-coffee-shop/tsconfig.json similarity index 100% rename from js/samples/coffee-shop/tsconfig.json rename to samples/js-coffee-shop/tsconfig.json diff --git a/samples/js-menu/README.md b/samples/js-menu/README.md new file mode 100644 index 0000000000..8200d81b6f --- /dev/null +++ b/samples/js-menu/README.md @@ -0,0 +1,27 @@ +## Menu Understanding Sample Application + +This sample demonstrates an application that can understand a restaurant menu and answer relevant questions about the items on the menu. + +There are 5 iterations of this sample application, growing in complexity and demonstrating utilization of many different Genkit features. + +To test each one out, open the Developer UI and exercise the prompts and flows. Each step contains one or more `example.json` files which you can use as inputs. + +### Prerequisites + +This example uses Vertex AI for language models and embeddings. + +### Prompts and Flows + +1. This step shows how to define prompts in code that can accept user input to their templates. +2. This step illustrates how to wrap your llm calls and other application code into flows with strong input and output schemas. + It also adds an example of tool usage to load the menu from a data file. +3. This step adds session history and supports a multi-turn chat with the model. +4. This step ingests the menu items into a vector database and uses retrieval to include releveant menu items in the prompt. +5. This step illustrates how to combine models with different modalities. It uses a vision model to ingest the menu items from a photograph. + +## Running the sample + +```bash +npm i +genkit start +``` diff --git a/samples/js-menu/data/menu.jpeg b/samples/js-menu/data/menu.jpeg new file mode 100644 index 0000000000..ae096cd7bd Binary files /dev/null and b/samples/js-menu/data/menu.jpeg differ diff --git a/samples/js-menu/data/menu.json b/samples/js-menu/data/menu.json new file mode 100644 index 0000000000..d46739205d --- /dev/null +++ b/samples/js-menu/data/menu.json @@ -0,0 +1,97 @@ +[ + { + "title": "Mozzarella Sticks", + "price": 8, + "description": "Crispy fried mozzarella sticks served with marinara sauce." + }, + { + "title": "Chicken Wings", + "price": 10, + "description": "Crispy fried chicken wings tossed in your choice of sauce." + }, + { + "title": "Nachos", + "price": 12, + "description": "Crispy tortilla chips topped with melted cheese, chili, sour cream, and salsa." + }, + { + "title": "Onion Rings", + "price": 7, + "description": "Crispy fried onion rings served with ranch dressing." + }, + { + "title": "French Fries", + "price": 5, + "description": "Crispy fried french fries." + }, + { + "title": "Mashed Potatoes", + "price": 6, + "description": "Creamy mashed potatoes." + }, + { + "title": "Coleslaw", + "price": 4, + "description": "Homemade coleslaw." + }, + { + "title": "Classic Cheeseburger", + "price": 12, + "description": "A juicy beef patty topped with melted American cheese, lettuce, tomato, and onion on a toasted bun." + }, + { + "title": "Bacon Cheeseburger", + "price": 14, + "description": "A classic cheeseburger with the addition of crispy bacon." + }, + { + "title": "Mushroom Swiss Burger", + "price": 15, + "description": "A beef patty topped with sautéed mushrooms, melted Swiss cheese, and a creamy horseradish sauce." + }, + { + "title": "Chicken Sandwich", + "price": 13, + "description": "A crispy chicken breast on a toasted bun with lettuce, tomato, and your choice of sauce." + }, + { + "title": "Pulled Pork Sandwich", + "price": 14, + "description": "Slow-cooked pulled pork on a toasted bun with coleslaw and barbecue sauce." + }, + { + "title": "Reuben Sandwich", + "price": 15, + "description": "Thinly sliced corned beef, Swiss cheese, sauerkraut, and Thousand Island dressing on rye bread." + }, + { + "title": "House Salad", + "price": 8, + "description": "Mixed greens with your choice of dressing." + }, + { + "title": "Caesar Salad", + "price": 9, + "description": "Romaine lettuce with croutons, Parmesan cheese, and Caesar dressing." + }, + { + "title": "Greek Salad", + "price": 10, + "description": "Mixed greens with feta cheese, olives, tomatoes, cucumbers, and red onions." + }, + { + "title": "Chocolate Lava Cake", + "price": 8, + "description": "A warm, gooey chocolate cake with a molten chocolate center." + }, + { + "title": "Apple Pie", + "price": 7, + "description": "A classic apple pie with a flaky crust and warm apple filling." + }, + { + "title": "Cheesecake", + "price": 8, + "description": "A creamy cheesecake with a graham cracker crust." + } +] diff --git a/samples/js-menu/package.json b/samples/js-menu/package.json new file mode 100644 index 0000000000..c91369e48b --- /dev/null +++ b/samples/js-menu/package.json @@ -0,0 +1,31 @@ +{ + "name": "menu", + "version": "1.0.0", + "description": "Samples for a menu understanding app", + "main": "lib/index.js", + "scripts": { + "start": "node lib/index.js", + "compile": "tsc", + "build": "npm run build:clean && npm run compile", + "build:clean": "rm -rf ./lib", + "build:watch": "tsc --watch", + "build-and-run": "npm run build && node lib/index.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@genkit-ai/ai": "~0.5.0-rc.11", + "@genkit-ai/core": "~0.5.0-rc.11", + "@genkit-ai/dotprompt": "~0.5.0-rc.11", + "@genkit-ai/flow": "~0.5.0-rc.11", + "@genkit-ai/dev-local-vectorstore": "~0.5.0-rc.11", + "@genkit-ai/firebase": "~0.5.0-rc.11", + "@genkit-ai/evaluator": "~0.5.0-rc.11", + "@genkit-ai/vertexai": "~0.5.0-rc.11", + "zod": "^3.22.4" + }, + "devDependencies": { + "typescript": "^5.3.3" + } +} diff --git a/samples/js-menu/src/01/example.json b/samples/js-menu/src/01/example.json new file mode 100644 index 0000000000..10c76a2ea9 --- /dev/null +++ b/samples/js-menu/src/01/example.json @@ -0,0 +1,5 @@ +{ + "input": { + "question": "Which of your burgers would you recommend for someone like me who loves bacon?" + } +} diff --git a/samples/js-menu/src/01/prompts.ts b/samples/js-menu/src/01/prompts.ts new file mode 100644 index 0000000000..f632dc7c17 --- /dev/null +++ b/samples/js-menu/src/01/prompts.ts @@ -0,0 +1,88 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { definePrompt } from '@genkit-ai/ai'; +import { GenerateRequest } from '@genkit-ai/ai/model'; +import { defineDotprompt } from '@genkit-ai/dotprompt'; +import { geminiPro } from '@genkit-ai/vertexai'; +import { MenuQuestionInput, MenuQuestionInputSchema } from '../types'; + +// Define a prompt to handle a customer question about the menu. +// This prompt uses definePrompt directly. + +export const s01_vanillaPrompt = definePrompt( + { + name: 's01_vanillaPrompt', + inputSchema: MenuQuestionInputSchema, + }, + async (input: MenuQuestionInput): Promise => { + const promptText = ` + You are acting as a helpful AI assistant named "Walt" that can answer + questions about the food available on the menu at Walt's Burgers. + Customer says: ${input.question} + `; + + return { + messages: [{ role: 'user', content: [{ text: promptText }] }], + config: { temperature: 0.3 }, + }; + } +); + +// Define another prompt which uses the Dotprompt library +// that also gives us a type-safe handlebars template system, +// and well-defined output schemas. + +export const s01_staticMenuDotPrompt = defineDotprompt( + { + name: 's01_staticMenuDotPrompt', + model: geminiPro, + input: { schema: MenuQuestionInputSchema }, + output: { format: 'text' }, + }, + ` +You are acting as a helpful AI assistant named "Walt" that can answer +questions about the food available on the menu at Walt's Burgers. +Here is today's menu: + +- The Regular Burger $12 + The classic charbroiled to perfection with your choice of cheese + +- The Fancy Burger $13 + Classic burger topped with bacon & Blue Cheese + +- The Bacon Burger $13 + Bacon cheeseburger with your choice of cheese. + +- Everything Burger $14 + Heinz 57 sauce, American cheese, bacon, fried egg & crispy onion bits + +- Chicken Breast Sandwich $12 + Tender juicy chicken breast on a brioche roll. + Grilled, blackened, or fried + +Our fresh 1/2 lb. beef patties are made using choice cut +brisket, short rib & sirloin. Served on a toasted +brioche roll with chips. Served with lettuce, tomato & pickles. +Onions upon request. Substitute veggie patty $2 + +Answer this customer's question, in a concise and helpful manner, +as long as it is about food. + +Question: +{{question}} ? +` +); diff --git a/samples/js-menu/src/02/example.json b/samples/js-menu/src/02/example.json new file mode 100644 index 0000000000..57a5977691 --- /dev/null +++ b/samples/js-menu/src/02/example.json @@ -0,0 +1,3 @@ +{ + "question": "I'd like to try something spicy. What do you recommend?" +} diff --git a/samples/js-menu/src/02/flows.ts b/samples/js-menu/src/02/flows.ts new file mode 100644 index 0000000000..ef2bc5761c --- /dev/null +++ b/samples/js-menu/src/02/flows.ts @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineFlow } from '@genkit-ai/flow'; +import { AnswerOutputSchema, MenuQuestionInputSchema } from '../types'; +import { s02_dataMenuPrompt } from './prompts'; + +// Define a flow which generates a response from the prompt. + +export const s02_menuQuestionFlow = defineFlow( + { + name: 's02_menuQuestion', + inputSchema: MenuQuestionInputSchema, + outputSchema: AnswerOutputSchema, + }, + async (input) => { + return s02_dataMenuPrompt + .generate({ + input: { question: input.question }, + }) + .then((response) => { + return { answer: response.text() }; + }); + } +); diff --git a/samples/js-menu/src/02/prompts.ts b/samples/js-menu/src/02/prompts.ts new file mode 100644 index 0000000000..c9350bd901 --- /dev/null +++ b/samples/js-menu/src/02/prompts.ts @@ -0,0 +1,45 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineDotprompt } from '@genkit-ai/dotprompt'; +import { geminiPro } from '@genkit-ai/vertexai'; +import { MenuQuestionInputSchema } from '../types'; +import { menuTool } from './tools'; + +// The prompt uses a tool which will load the menu data, +// if the user asks a reasonable question about the menu. + +export const s02_dataMenuPrompt = defineDotprompt( + { + name: 's02_dataMenu', + model: geminiPro, + input: { schema: MenuQuestionInputSchema }, + output: { format: 'text' }, + tools: [menuTool], + }, + ` +You are acting as a helpful AI assistant named Walt that can answer +questions about the food available on the menu at Walt's Burgers. + +Answer this customer's question, in a concise and helpful manner, +as long as it is about food on the menu or something harmless like sports. +Use the tools available to answer menu questions. +DO NOT INVENT ITEMS NOT ON THE MENU. + +Question: +{{question}} ? +` +); diff --git a/samples/js-menu/src/02/tools.ts b/samples/js-menu/src/02/tools.ts new file mode 100644 index 0000000000..d9a60290da --- /dev/null +++ b/samples/js-menu/src/02/tools.ts @@ -0,0 +1,35 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineTool } from '@genkit-ai/ai'; +import * as z from 'zod'; +import { MenuItem, MenuItemSchema } from '../types'; + +const menuData: Array = require('../../data/menu.json'); + +export const menuTool = defineTool( + { + name: 'todaysMenu', + description: "Use this tool to retrieve all the items on today's menu", + inputSchema: z.object({}), + outputSchema: z.object({ + menuData: z + .array(MenuItemSchema) + .describe('A list of all the items on the menu'), + }), + }, + async () => Promise.resolve({ menuData: menuData }) +); diff --git a/samples/js-menu/src/03/chats.ts b/samples/js-menu/src/03/chats.ts new file mode 100644 index 0000000000..2456d60079 --- /dev/null +++ b/samples/js-menu/src/03/chats.ts @@ -0,0 +1,58 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MessageData, MessageSchema } from '@genkit-ai/ai/model'; +import * as z from 'zod'; + +// Our flow will take a sessionId along with each question to track the chat history. +// The host application should keep track of these ids somewhere. + +export const ChatSessionInputSchema = z.object({ + sessionId: z.string(), + question: z.string(), +}); + +// The flow will respond with an array of messages, +// which includes all history up until that point +// plus the last exchange with the model. + +export const ChatSessionOutputSchema = z.object({ + sessionId: z.string(), + history: z.array(MessageSchema), +}); + +export type ChatHistory = Array; + +// This is a very simple local storage for chat history. +// Each conversation is identified by a sessionId generated by the application. +// The constructor accepts a preamble of messages, which serve as a system prompt. + +export class ChatHistoryStore { + private preamble: ChatHistory; + private sessions: Map = new Map(); + + constructor(preamble: ChatHistory = []) { + this.preamble = preamble; + } + + write(sessionId: string, history: ChatHistory) { + this.sessions.set(sessionId, history); + } + + read(sessionId: string): ChatHistory { + return this.sessions.get(sessionId) || this.preamble; + } +} diff --git a/samples/js-menu/src/03/example.json b/samples/js-menu/src/03/example.json new file mode 100644 index 0000000000..19ed21af5d --- /dev/null +++ b/samples/js-menu/src/03/example.json @@ -0,0 +1,4 @@ +{ + "sessionId": "session123", + "question": "Do you have anything healthy on this menu?" +} diff --git a/samples/js-menu/src/03/flows.ts b/samples/js-menu/src/03/flows.ts new file mode 100644 index 0000000000..c9addc77d7 --- /dev/null +++ b/samples/js-menu/src/03/flows.ts @@ -0,0 +1,76 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { generate } from '@genkit-ai/ai'; +import { MessageData } from '@genkit-ai/ai/model'; +import { defineFlow, run } from '@genkit-ai/flow'; +import { geminiPro } from '@genkit-ai/vertexai'; + +import { MenuItem } from '../types'; +import { + ChatHistoryStore, + ChatSessionInputSchema, + ChatSessionOutputSchema, +} from './chats'; +import { s03_chatPreamblePrompt } from './prompts'; + +// Load the menu data from a JSON file. +const menuData = require('../../data/menu.json') as Array; + +// Render the preamble prompt that seeds our chat history. +const preamble: Array = s03_chatPreamblePrompt.renderMessages({ + menuData: menuData, + question: '', +}); + +// A simple local storage for chat session history. +// You should probably actually use Firestore for this. +const chatHistoryStore = new ChatHistoryStore(preamble); + +// Define a flow which generates a response to each question. + +export const s03_multiTurnChatFlow = defineFlow( + { + name: 's03_multiTurnChat', + inputSchema: ChatSessionInputSchema, + outputSchema: ChatSessionOutputSchema, + }, + async (input) => { + // First fetch the chat history. We'll wrap this in a run block. + // If we were going to a database for the history, + // we might want to have that db result captured in the trace. + let history = await run('fetchHistory', async () => + chatHistoryStore.read(input.sessionId) + ); + + // Generate the response + const llmResponse = await generate({ + model: geminiPro, + history: history, + prompt: { + text: input.question, + }, + }); + + // Add the exchange to the history store and return it + history = llmResponse.toHistory(); + chatHistoryStore.write(input.sessionId, history); + return { + sessionId: input.sessionId, + history: history, + }; + } +); diff --git a/samples/js-menu/src/03/prompts.ts b/samples/js-menu/src/03/prompts.ts new file mode 100644 index 0000000000..18abc13bbe --- /dev/null +++ b/samples/js-menu/src/03/prompts.ts @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineDotprompt } from '@genkit-ai/dotprompt'; +import { geminiPro } from '@genkit-ai/vertexai'; +import { DataMenuQuestionInputSchema } from '../types'; + +// This prompt will generate two messages when rendered. +// These two messages will be used to seed the exchange with the model. + +export const s03_chatPreamblePrompt = defineDotprompt( + { + name: 's03_chatPreamble', + model: geminiPro, + input: { schema: DataMenuQuestionInputSchema }, + output: { format: 'text' }, + config: { temperature: 0.3 }, + }, + ` + {{ role "user" }} + Hi. What's on the menu today? + + {{ role "model" }} + I am Walt, a helpful AI assistant here at the restaurant. + I can answer questions about the food on the menu or any other questions + you have about food in general. I probably can't help you with anything else. + Here is today's menu: + {{#each menuData~}} + - {{this.title}} \${{this.price}} + {{this.description}} + {{~/each}} + Do you have any questions about the menu? +` +); diff --git a/samples/js-menu/src/04/example.indexMenuItems.json b/samples/js-menu/src/04/example.indexMenuItems.json new file mode 100644 index 0000000000..d528d8ae9f --- /dev/null +++ b/samples/js-menu/src/04/example.indexMenuItems.json @@ -0,0 +1,57 @@ +[ + { + "title": "White Meat Crispy Chicken Wings", + "description": "All-white meat chicken wings tossed in your choice of wing sauce. Choose from classic buffalo, honey bbq, garlic parmesan, or sweet & sour", + "price": 12.0 + }, + { + "title": "Cheese Fries", + "description": "Fresh fries covered with melted cheddar cheese and bacon", + "price": 8.0 + }, + { + "title": "Reuben", + "description": "Classic Reuben sandwich with corned beef, sauerkraut, Swiss cheese, and Thousand Island dressing on grilled rye bread.", + "price": 12.0 + }, + { + "title": "Grilled Chicken Club Wrap", + "description": "Grilled chicken, bacon, lettuce, tomato, pickles, and cheddar cheese wrapped in a spinach tortilla, served with your choice of dressing", + "price": 12.0 + }, + { + "title": "Buffalo Chicken Sandwich", + "description": "Fried chicken breast coated in your choice of wing sauce, topped with lettuce, tomato, onion, and pickles on a toasted brioche roll.", + "price": 12.0 + }, + { + "title": "Half Cuban Sandwich", + "description": "Slow roasted pork butt, ham, Swiss, and yellow mustard on a toasted baguette", + "price": 12.0 + }, + { + "title": "The Albie Burger", + "description": "Classic burger topped with bacon, provolone, banana peppers, and chipotle mayo", + "price": 13.0 + }, + { + "title": "57 Chevy Burger", + "description": "Heaven burger with your choice of cheese", + "price": 14.0 + }, + { + "title": "Chicken Caesar Wrap", + "description": "Tender grilled chicken, romaine lettuce, croutons, and Parmesan cheese tossed in a creamy Caesar dressing and wrapped in a spinach tortilla", + "price": 10.0 + }, + { + "title": "Kids Hot Dog", + "description": "Kids under 12", + "price": 5.0 + }, + { + "title": "Chicken Fingers", + "description": "Tender chicken strips, grilled or fried", + "price": 8.0 + } +] diff --git a/samples/js-menu/src/04/example.menuQuestion.json b/samples/js-menu/src/04/example.menuQuestion.json new file mode 100644 index 0000000000..21c7c4647c --- /dev/null +++ b/samples/js-menu/src/04/example.menuQuestion.json @@ -0,0 +1,3 @@ +{ + "question": "I'd like something cheesy!" +} diff --git a/samples/js-menu/src/04/flows.ts b/samples/js-menu/src/04/flows.ts new file mode 100644 index 0000000000..fbef53a0c3 --- /dev/null +++ b/samples/js-menu/src/04/flows.ts @@ -0,0 +1,87 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Document, index, retrieve } from '@genkit-ai/ai/retriever'; +import { + devLocalIndexerRef, + devLocalRetrieverRef, +} from '@genkit-ai/dev-local-vectorstore'; +import { defineFlow } from '@genkit-ai/flow'; +import * as z from 'zod'; + +import { + AnswerOutputSchema, + MenuItem, + MenuItemSchema, + MenuQuestionInputSchema, +} from '../types'; +import { s04_ragDataMenuPrompt } from './prompts'; + +// Define a flow which indexes items on the menu. + +export const s04_indexMenuItemsFlow = defineFlow( + { + name: 's04_indexMenuItems', + inputSchema: z.array(MenuItemSchema), + outputSchema: z.object({ rows: z.number() }), + }, + async (menuItems) => { + // Store each document with its text indexed, + // and its original JSON data as its metadata. + const documents = menuItems.map((menuItem) => { + const text = `${menuItem.title} ${menuItem.price} \n ${menuItem.description}`; + return Document.fromText(text, menuItem); + }); + await index({ + indexer: devLocalIndexerRef('menu-items'), + documents, + }); + return { rows: menuItems.length }; + } +); + +// Define a flow which generates a response to the question, +// by retrieving relevant items from the menu. +// View this flow's trace to see the context that was retrieved, +// and how it was included in the prompt. + +export const s04_ragMenuQuestionFlow = defineFlow( + { + name: 's04_ragMenuQuestion', + inputSchema: MenuQuestionInputSchema, + outputSchema: AnswerOutputSchema, + }, + async (input) => { + // Retrieve the 3 most relevant menu items for the question + const docs = await retrieve({ + retriever: devLocalRetrieverRef('menu-items'), + query: input.question, + options: { k: 3 }, + }); + const menuData: Array = docs.map( + (doc) => (doc.metadata || {}) as MenuItem + ); + + // Generate the response + const response = await s04_ragDataMenuPrompt.generate({ + input: { + menuData: menuData, + question: input.question, + }, + }); + return { answer: response.text() }; + } +); diff --git a/samples/js-menu/src/04/prompts.ts b/samples/js-menu/src/04/prompts.ts new file mode 100644 index 0000000000..ddb5dc28a3 --- /dev/null +++ b/samples/js-menu/src/04/prompts.ts @@ -0,0 +1,44 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineDotprompt } from '@genkit-ai/dotprompt'; +import { geminiPro } from '@genkit-ai/vertexai'; +import { DataMenuQuestionInputSchema } from '../types'; + +export const s04_ragDataMenuPrompt = defineDotprompt( + { + name: 's04_ragDataMenu', + model: geminiPro, + input: { schema: DataMenuQuestionInputSchema }, + output: { format: 'text' }, + config: { temperature: 0.3 }, + }, + ` +You are acting as Walt, a helpful AI assistant here at the restaurant. +You can answer questions about the food on the menu or any other questions +customers have about food in general. + +Here are some items that are on today's menu that are relevant to +helping you answer the customer's question: +{{#each menuData~}} +- {{this.title}} \${{this.price}} + {{this.description}} +{{~/each}} + +Answer this customer's question: +{{question}}? +` +); diff --git a/samples/js-menu/src/05/example.visualMenuQuestion.json b/samples/js-menu/src/05/example.visualMenuQuestion.json new file mode 100644 index 0000000000..3c49f36d0a --- /dev/null +++ b/samples/js-menu/src/05/example.visualMenuQuestion.json @@ -0,0 +1,3 @@ +{ + "question": "What kind of burger buns do you have?" +} diff --git a/samples/js-menu/src/05/flows.ts b/samples/js-menu/src/05/flows.ts new file mode 100644 index 0000000000..4caddfa6f1 --- /dev/null +++ b/samples/js-menu/src/05/flows.ts @@ -0,0 +1,101 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineFlow, runFlow } from '@genkit-ai/flow'; +import fs from 'fs'; +import path from 'path'; + +import * as z from 'zod'; +import { + AnswerOutputSchema, + MenuQuestionInputSchema, + TextMenuQuestionInputSchema, +} from '../types'; +import { s05_readMenuPrompt, s05_textMenuPrompt } from './prompts'; + +// Define a flow that takes an image, passes it to Gemini Vision Pro, +// and extracts all of the text from the photo of the menu. +// Note that this example uses a hard-coded image file, as image input +// is not currently available in the Development UI runners. + +export const s05_readMenuFlow = defineFlow( + { + name: 's05_readMenuFlow', + inputSchema: z.void(), // input is data/menu.jpeg + outputSchema: z.object({ menuText: z.string() }), + }, + async (unused) => { + const imageDataUrl = await inlineDataUrl('menu.jpeg', 'image/jpeg'); + const response = await s05_readMenuPrompt.generate({ + input: { + imageUrl: imageDataUrl, + }, + }); + return { menuText: response.text() }; + } +); + +// Define a flow which generates a response to the question. +// Just returns the llm's text response to the question. + +export const s05_textMenuQuestionFlow = defineFlow( + { + name: 's05_textMenuQuestion', + inputSchema: TextMenuQuestionInputSchema, + outputSchema: AnswerOutputSchema, + }, + async (input) => { + const response = await s05_textMenuPrompt.generate({ + input: { + menuText: input.menuText, + question: input.question, + }, + }); + return { answer: response.text() }; + } +); + +// Define a third composite flow which chains the first two flows + +export const s05_visionMenuQuestionFlow = defineFlow( + { + name: 's05_visionMenuQuestion', + inputSchema: MenuQuestionInputSchema, + outputSchema: AnswerOutputSchema, + }, + async (input) => { + // Run the first flow to read the menu image. + const menuResult = await runFlow(s05_readMenuFlow); + + // Pass the text of the menu and the question to the second flow + // and return the answer as this output. + return runFlow(s05_textMenuQuestionFlow, { + question: input.question, + menuText: menuResult.menuText, + }); + } +); + +// Helper to read a local file and inline it as a data url + +async function inlineDataUrl( + imageFilename: string, + contentType: string +): Promise { + const filePath = path.join('./data', imageFilename); + const imageData = fs.readFileSync(filePath); + return `data:${contentType};base64,${imageData.toString('base64')}`; +} diff --git a/samples/js-menu/src/05/prompts.ts b/samples/js-menu/src/05/prompts.ts new file mode 100644 index 0000000000..cfcf10669f --- /dev/null +++ b/samples/js-menu/src/05/prompts.ts @@ -0,0 +1,61 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineDotprompt } from '@genkit-ai/dotprompt'; +import { geminiPro, geminiProVision } from '@genkit-ai/vertexai'; +import * as z from 'zod'; +import { TextMenuQuestionInputSchema } from '../types'; + +export const s05_readMenuPrompt = defineDotprompt( + { + name: 's05_readMenu', + model: geminiProVision, + input: { + schema: z.object({ + imageUrl: z.string(), + }), + }, + output: { format: 'text' }, + config: { temperature: 0.1 }, + }, + ` +Extract _all_ of the text, in order, +from the following image of a restaurant menu. + +{{media url=imageUrl}} +` +); + +export const s05_textMenuPrompt = defineDotprompt( + { + name: 's05_textMenu', + model: geminiPro, + input: { schema: TextMenuQuestionInputSchema }, + output: { format: 'text' }, + config: { temperature: 0.3 }, + }, + ` +You are acting as Walt, a helpful AI assistant here at the restaurant. +You can answer questions about the food on the menu or any other questions +customers have about food in general. + +Here is the text of today's menu to help you answer the customer's question: +{{menuText}} + +Answer this customer's question: +{{question}}? +` +); diff --git a/samples/js-menu/src/index.ts b/samples/js-menu/src/index.ts new file mode 100644 index 0000000000..364737e2e5 --- /dev/null +++ b/samples/js-menu/src/index.ts @@ -0,0 +1,60 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { configureGenkit } from '@genkit-ai/core'; +import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore'; +import { dotprompt } from '@genkit-ai/dotprompt'; +import { textEmbeddingGecko, vertexAI } from '@genkit-ai/vertexai'; + +// Initialize Genkit + +configureGenkit({ + plugins: [ + dotprompt(), + vertexAI({ location: 'us-central1' }), + devLocalVectorstore([ + { + indexName: 'menu-items', + embedder: textEmbeddingGecko, + embedderOptions: { taskType: 'RETRIEVAL_DOCUMENT' }, + }, + ]), + ], + enableTracingAndMetrics: true, + flowStateStore: 'firebase', + logLevel: 'debug', + traceStore: 'firebase', +}); + +// Export all of the example prompts and flows + +// 01 +export { s01_staticMenuDotPrompt, s01_vanillaPrompt } from './01/prompts'; +// 02 +export { s02_menuQuestionFlow } from './02/flows'; +export { s02_dataMenuPrompt } from './02/prompts'; +// 03 +export { s03_multiTurnChatFlow } from './03/flows'; +export { s03_chatPreamblePrompt } from './03/prompts'; +// 04 +export { s04_indexMenuItemsFlow, s04_ragMenuQuestionFlow } from './04/flows'; +export { s04_ragDataMenuPrompt } from './04/prompts'; +// 05 +export { + s05_readMenuFlow, + s05_textMenuQuestionFlow, + s05_visionMenuQuestionFlow, +} from './05/flows'; +export { s05_readMenuPrompt, s05_textMenuPrompt } from './05/prompts'; diff --git a/samples/js-menu/src/types.ts b/samples/js-menu/src/types.ts new file mode 100644 index 0000000000..efe7bbc99d --- /dev/null +++ b/samples/js-menu/src/types.ts @@ -0,0 +1,63 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as z from 'zod'; + +// The data model for a restaurant menu + +export const MenuItemSchema = z.object({ + title: z.string().describe('The name of the menu item'), + description: z + .string() + .describe('Details including ingredients and preparation'), + price: z.number().describe('Price in dollars'), +}); + +export type MenuItem = z.infer; + +// Input schema for a question about the menu + +export const MenuQuestionInputSchema = z.object({ + question: z.string(), +}); + +// Output schema containing an answer to a question + +export const AnswerOutputSchema = z.object({ + answer: z.string(), +}); + +// Input schema for a question about the menu +// where the menu is provided in JSON data. + +export const DataMenuQuestionInputSchema = z.object({ + menuData: z.array(MenuItemSchema), + question: z.string(), +}); + +// Input schema for a question about the menu +// where the menu is provided as unstructured text. + +export const TextMenuQuestionInputSchema = z.object({ + menuText: z.string(), + question: z.string(), +}); + +// Also export Typescript types for each of these Zod schemas +export type MenuQuestionInput = z.infer; +export type AnswerOutput = z.infer; +export type DataMenuPromptInput = z.infer; +export type TextMenuQuestionInput = z.infer; diff --git a/samples/js-menu/tsconfig.json b/samples/js-menu/tsconfig.json new file mode 100644 index 0000000000..e51f33ae38 --- /dev/null +++ b/samples/js-menu/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "noImplicitReturns": true, + "noUnusedLocals": false, + "outDir": "lib", + "sourceMap": true, + "strict": true, + "target": "es2017", + "skipLibCheck": true, + "esModuleInterop": true + }, + "compileOnSave": true, + "include": ["src"] +}