From 8458bb38630972d65911314554b4d2f6d2016d69 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 26 Feb 2024 22:32:10 -0500 Subject: [PATCH 001/238] Fix model routing (#161) From a326ce52f71b9384244f276c1b602b49dc733477 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 27 Feb 2024 12:02:26 -0500 Subject: [PATCH 002/238] [UI] Add new span tree + viewer to Flow details page (#164) From 26bdba0cc95a35e3b064ca1108c7a17331184e8a Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 27 Feb 2024 23:33:43 -0500 Subject: [PATCH 003/238] Fetch models from API (#174) From be50e2996c7aac1361fd30c0d3a65ddd32340dbb Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:04:07 -0500 Subject: [PATCH 004/238] Backend errors (#163) Display errors in the Prompt Playground component after receiving issues from backend From 8864cbe057ec91da1e6a73c3b0308d94d2a3ca8e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 28 Feb 2024 13:11:57 -0500 Subject: [PATCH 005/238] [UI] Cleanup unimplemented pages from navbar (#180) From 742b62c0db706869f4142025ff2dae1d1b865611 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 28 Feb 2024 13:13:28 -0500 Subject: [PATCH 006/238] [UI] Increase max-height of flow input/output (#179) Also update styles for running + error statues in output box. From 6e178bfb5d2a914af30498e499a2144b57e7fd97 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Wed, 28 Feb 2024 10:57:00 -0800 Subject: [PATCH 007/238] Move flow runner to Actions page (#176) From 2d96c6329dbf704da68ded8fd70af44fe03e27f5 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 28 Feb 2024 14:38:00 -0500 Subject: [PATCH 008/238] [UI] Fix overflow of execution span tree (#183) From 72c0d565d918979032513a1529a73261b2b4343a Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Wed, 28 Feb 2024 16:31:39 -0500 Subject: [PATCH 009/238] Input validation disables prompt run button (#182) Input validation for prompt playground From be41efd454b31f2905a82a63c26ad641d9ebc235 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 29 Feb 2024 09:10:12 -0500 Subject: [PATCH 010/238] Route playground from flows to action runners page (#191) From 43de9426dbe35433b164243aaaae7c7cf6b35477 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 29 Feb 2024 10:14:42 -0500 Subject: [PATCH 011/238] Switch temperature to the slider (#195) From ddbfa24987023be5149b3d34fa6151580499e64b Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 29 Feb 2024 11:07:11 -0500 Subject: [PATCH 012/238] Show validation errors on the playground (#196) From 741bbead50de125a10c771e5e0d84d64734d93aa Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 29 Feb 2024 11:20:54 -0500 Subject: [PATCH 013/238] [UI] Revamp flow details page layout (#197) From 12d154708914ef9ecae18da33e41b766d15b0de4 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 29 Feb 2024 09:13:28 -0800 Subject: [PATCH 014/238] Fix validator issues (#194) From eea11ff3e5ffd66f0394cacd1a7f77a0d3ab9455 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 29 Feb 2024 13:34:36 -0500 Subject: [PATCH 015/238] [UI] Initial design of span details view (#199) From 624649b704eccd52cfd9bc2be3b665ef1d2e7608 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 29 Feb 2024 11:09:14 -0800 Subject: [PATCH 016/238] Move flow runner to start from action-list instead of action-runner (#200) From 7bfdfff5428bfde5984188366a701eb2fbd4c4ad Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 29 Feb 2024 14:13:12 -0500 Subject: [PATCH 017/238] Add vertex-ai to the model playground (#201) Also add icons for all known action types From f499069e5c4e96fea700df7f51725662d7d4e2b9 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 29 Feb 2024 15:54:09 -0500 Subject: [PATCH 018/238] [UI] Hide input/output pre if none available (#204) From 110e10a71d463b89f900b143f6c12b5179d8fb8d Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 29 Feb 2024 16:36:30 -0500 Subject: [PATCH 019/238] [UI] Add "muted" helper class for secondary text (#206) From 38543a15776f0685c87bdf57ab380190331a591d Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Sun, 3 Mar 2024 18:19:10 -0500 Subject: [PATCH 020/238] Don't send blank stop sequences to the model, vertex gemini model doesn't like it (#217) From 88ec7c79f180f89a22a78a2bce875758ea7fc290 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:33:54 -0500 Subject: [PATCH 021/238] Provider specific model param restrictions on input (#224) From a6ce1a0a0a938643306eb7252134b4a257ebc636 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 5 Mar 2024 22:34:01 -0500 Subject: [PATCH 022/238] Use the minfied version of Monaco Editor in the angular app (#242) From 5e13777547b0736262294609c95139f962245a67 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 6 Mar 2024 11:15:15 -0500 Subject: [PATCH 023/238] [UI] Update app name From dd4e8d26f1cee07dde3434532df6e08c59642a8e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 6 Mar 2024 11:26:24 -0500 Subject: [PATCH 024/238] [UI] Update flow details layout (#246) Also adds new `` shared component which adds a button to show text in a larger pop-up dialog. From 71a7800b4dff2b13b436800abfaf77f9b9018985 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 6 Mar 2024 11:59:58 -0500 Subject: [PATCH 025/238] [UI] Add callout component (#244) From ec9fc131734f02522574202520d77e1adea91a96 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 6 Mar 2024 14:57:39 -0500 Subject: [PATCH 026/238] [UI] Hide wrapper spans on details page (#254) From d1289e17c0af76c211239ebad1ae658059b2e03e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 6 Mar 2024 16:48:23 -0500 Subject: [PATCH 027/238] [UI] Update flow durations on details page (#256) From 52ce95e3de7612df5a5e0dc3a05e0a31d8f0466f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 6 Mar 2024 19:44:59 -0500 Subject: [PATCH 028/238] [UI] Show error on flow details page (#258) From 67d5b4512bf7bac80402ca9fc71a553d5ee0946f Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 6 Mar 2024 22:02:53 -0500 Subject: [PATCH 029/238] Playground load trace (#262) * Code cleanup * Load playground from a trace From e2aa09c0363d02fa649c8cbf19f97379d9aa7b61 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 7 Mar 2024 01:37:50 -0800 Subject: [PATCH 030/238] Add theme toggling for JSON editor and move schema to a tab next to the editor (#245) From ee2b0a36895b1e381cd222c211881f9db75b6980 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 7 Mar 2024 10:30:39 -0500 Subject: [PATCH 031/238] Give topP the slider treatment (#264) It's only right, now that we've done temp. :-) From 8636d507b6c025076db96358072a78058e842a1c Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 7 Mar 2024 13:40:05 -0500 Subject: [PATCH 032/238] [UI] Show flow name in tree (#266) From ea874f4c3cf8028e2c90dae384ac736001d972ed Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 7 Mar 2024 15:35:43 -0500 Subject: [PATCH 033/238] [UI] Show span state in details pane (#268) From 8b49ab5cc01a52d66ae2b14299b6c0f4a8c9316e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 7 Mar 2024 16:26:26 -0500 Subject: [PATCH 034/238] [UI] Flows table style improvements (#269) From d9a2b5ffbb491b20151b1ddc7940aac4d0f4e447 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 7 Mar 2024 16:54:55 -0500 Subject: [PATCH 035/238] [UI] Small flow details page improvements for narrow screens (#273) From 628c0dc3e50413ebe4e8a04f6e12c0b9ccc04dc0 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 7 Mar 2024 21:44:31 -0500 Subject: [PATCH 036/238] Add CustomOptions (#276) Also, add stop sequences to the request. From 6b9dbc8e4b4e942cf97eee6fbb9a98a0c332c6cb Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 7 Mar 2024 20:53:27 -0800 Subject: [PATCH 037/238] [UI]Remove sample calls for unsupported actions. Small fixes in flow runner. (#275) From de2908a36b0d10ccebdf6bf0276093d0742ab765 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:11:14 -0500 Subject: [PATCH 038/238] Create Message sub component for ModelPlayground (#271) #148 From 3fa8882045b2b6933e78a1a89ccc66800613abf1 Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:46:35 -0500 Subject: [PATCH 039/238] Fix error with model not accepting request_format (#279) From ca08ec77068c27f68810bc2848d34b24ad512b06 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Fri, 8 Mar 2024 14:19:57 -0500 Subject: [PATCH 040/238] Disable the minimap on the monaco editor (#286) From b555204d3cd0b59a60bc1abd251610e85c9f99b1 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sat, 9 Mar 2024 16:12:26 -0500 Subject: [PATCH 041/238] [UI] Add zero state for flows list page (#291) From 07d97bf3f10af54b91de9df5e2578d0b1bb85a39 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 11 Mar 2024 10:59:38 -0400 Subject: [PATCH 042/238] [UI] Fix ng error in flow runner (#297) From 9d453d1a3fb746361d0ed8f3d58e374dd7dbdc6a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 11 Mar 2024 14:08:52 -0400 Subject: [PATCH 043/238] [UI] Hide stream response checkbox for durable flows (#299) From 4317902f78074c02bb743a45f1e06e111e68971c Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:05:02 -0400 Subject: [PATCH 044/238] Integrating the Message component into the Prompt Playground From 7b86f4ea5a40f3743af66a187236d85630ae144f Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 12 Mar 2024 09:18:31 -0400 Subject: [PATCH 045/238] Switch model select from native to mat-select (#306) From eebfe569becdf74b4fe60ca6b8d2b977b357f406 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 12 Mar 2024 09:30:15 -0400 Subject: [PATCH 046/238] Ability to show errors on actions page (#307) From f10df1f4b068ec634e922851db3c92e950e93842 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 12 Mar 2024 10:01:34 -0400 Subject: [PATCH 047/238] [UI] Revamp Actions list UI (#308) From 29ec7df8c35998d257a86f6c7b4ac276dd88a5c2 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 12 Mar 2024 10:11:41 -0400 Subject: [PATCH 048/238] [UI] Remove unnecessary return (#309) From 3fd7b76369b24f4c7df8cd293e8b40bee4c5b268 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 12 Mar 2024 10:32:38 -0400 Subject: [PATCH 049/238] [UI] Prevent selecting action if no param is set (#310) From 12c6832d62e71f2fbbb869a4c1cb5f257c2a0984 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:11:10 -0400 Subject: [PATCH 050/238] Enable support for multiple messages coming from traceId (#314) From ab111d8839de2d38696c8bceec28257b9999bb71 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Tue, 12 Mar 2024 15:34:32 -0700 Subject: [PATCH 051/238] Avoid making flow runner editors read only (#321) From 9eee3289c375ee24ed64057f4c930ba2a129ecee Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 12 Mar 2024 20:02:43 -0400 Subject: [PATCH 052/238] [UI] Add filtering and expand/collapse all to actions list (#319) From d61c41a47b47b50c2c06528d0afd4c1f60129900 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 12 Mar 2024 21:42:39 -0400 Subject: [PATCH 053/238] Fix error where model selection does not update (#323) From 55c9644c57900f9bb11820b4da457aae57fca983 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 13 Mar 2024 01:00:27 -0400 Subject: [PATCH 054/238] [UI] Fix action search input style (#325) From 4c9cfb511eb679e6aa4d26688de6ea153cceb61a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 13 Mar 2024 10:52:49 -0400 Subject: [PATCH 055/238] [UI] Update action list name and key display (#328) From 28494db59f36691460294d0257ac672a0c0dbc7b Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 13 Mar 2024 11:26:19 -0400 Subject: [PATCH 056/238] User error callout component on model playground (#330) From 2ac6af686da4a8af24642950cf1cd37b5a26886a Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:58:47 -0400 Subject: [PATCH 057/238] refactor the code around checking for json output support (#304) From fdd70a3ae8a282f20c2459158c053227257ce608 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 13 Mar 2024 14:46:32 -0400 Subject: [PATCH 058/238] Render images in chat (#340) From 450bfbd458a19763c5f0c43b375630addcc13fd9 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:30:18 -0400 Subject: [PATCH 059/238] Functioning add and remove button (#335) From 1a826beb651c927ab66b35835f1c228c3a54564b Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:45:28 -0400 Subject: [PATCH 060/238] Refactor criteria/validation logic out of playground component (#339) From c032e0395c33443fbede4838695baa0672bf6c29 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 13 Mar 2024 17:14:14 -0400 Subject: [PATCH 061/238] [UI] Flow runner UI polish + improvements (#343) From b046aca8adbbb640b2d121852817dd04213b22b6 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Wed, 13 Mar 2024 15:00:32 -0700 Subject: [PATCH 062/238] Move JSON editor to shared components since retriever playground also needs it (#344) From 129814b47e1d319238800a263bbf8d43dfc99e37 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 13 Mar 2024 21:02:48 -0400 Subject: [PATCH 063/238] [UI] Small handful of UI nit fixes (#345) From 08a4d55594eef9ae561cbe8aa4e0f34d4d941dee Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 13 Mar 2024 23:03:09 -0400 Subject: [PATCH 064/238] [UI] Add loading state to flows table (#349) From ab52c9fb5d87b8a33053ae84b2441740d7909265 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 14 Mar 2024 10:24:26 -0400 Subject: [PATCH 065/238] Do not load output from trace; typically we're interested in loading up the inputs, and re-running to get the output (#347) From 5fc6b03fac087aec4d4a34012a49a4574d2ff50d Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 14 Mar 2024 11:13:26 -0400 Subject: [PATCH 066/238] Make response_format optional (#350) From a3fbc830f6521a2f0da0a1945cbb76989f006118 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 14 Mar 2024 13:20:02 -0400 Subject: [PATCH 067/238] [UI] Add Genkit icon (#371) From 9aae19a8971808571d565ab1c99aa8ef6fe333b1 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Thu, 14 Mar 2024 15:59:31 -0400 Subject: [PATCH 068/238] Reset streamed chunks when rerunning the streamed flow (#379) From 661e34bf63bbd44de98a5b27dc0bc4df8061510a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 14 Mar 2024 16:57:16 -0400 Subject: [PATCH 069/238] [UI] Add tooltips to span state icons (#351) From 4de6f90966d0a917ae554468bd741851505c59c5 Mon Sep 17 00:00:00 2001 From: tagboola Date: Thu, 14 Mar 2024 17:03:52 -0400 Subject: [PATCH 070/238] Prefer includes over contains (#376) Contains causes a `TypeError: _i.contains is not a function` when running evals. From 87b05103ac4f7d5d11ea59d54027c838c6662e3d Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 14 Mar 2024 17:06:15 -0400 Subject: [PATCH 071/238] [UI] Add inspect flow state button if flow errors (#382) From 11f41ab88be9dc3c61841fa8ce672e59bd2e5d5e Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:13:30 -0400 Subject: [PATCH 072/238] Chat mode (#391) From 0524392fdf8cbd4b913deb312749254239fc89db Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Fri, 15 Mar 2024 11:53:48 -0400 Subject: [PATCH 073/238] Ability to open Flow runner from the trace view (#394) From 2b95bc906d2f2e9517af436df89df5576c6e0f4e Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 15 Mar 2024 16:29:07 +0000 Subject: [PATCH 074/238] Add basics of the eval runner page (#367) * initial ui changes * formatted * Add mocked evals page * Unnest runs * Remove evaluations tab from appbar From e4a8b4d599dab0530495f72d5fd8721474c909ac Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 15 Mar 2024 13:06:35 -0400 Subject: [PATCH 075/238] [UI] Fix flow details sidebar colors in dark mode (#399) From b0bf441694ee1b94abc5c2888a6a925aa1da60b5 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 15 Mar 2024 13:11:37 -0400 Subject: [PATCH 076/238] [UI] Revamp model playground to chat-based layout (#397) From 856310da3c3c7b2baa20b03b8e516376149ccbc1 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Fri, 15 Mar 2024 12:59:35 -0700 Subject: [PATCH 077/238] [UI] Flow runner: Add a callout for no output so we dont show empty response boxes (#403) From 4778331f58d4e9316f210fb8d7af05d4e107820e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 15 Mar 2024 17:14:37 -0400 Subject: [PATCH 078/238] [UI] Add trace details view (#405) From ba19062fd02e3cbc48609ceb6986c8e3bcc1a47b Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:04:24 -0400 Subject: [PATCH 079/238] role:system message allowed for models (#402) From 6a4e4ca2ca553dc01d1f3d9ecae6c8dd8836a588 Mon Sep 17 00:00:00 2001 From: Michael Bleigh Date: Mon, 18 Mar 2024 12:52:33 -0700 Subject: [PATCH 080/238] Adds support for image models. (#426) From 3f80ed568ea8c0d49eec17cc7b9a99c294fd185b Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:55:53 -0400 Subject: [PATCH 081/238] fix playground runner after runAction change (#429) From 5a81039888cd77967df688061972cd48060970a7 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:38:12 -0400 Subject: [PATCH 082/238] Revert "fix playground runner after runAction change (#429)" (#431) This reverts commit 82264c0777dd47b0835dda01362a902298ec044b. From 8171ec617298cfdf156d9fc9d0a76dd0fb95a39a Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 18 Mar 2024 21:04:26 -0400 Subject: [PATCH 083/238] Small tweaks to model playground to reduce chat (#438) input clutter From 5cb86c8dabdb0d5178bba5dc3d49396ca83ee51f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 00:17:45 -0400 Subject: [PATCH 084/238] [UI] Update `stackTraceSpans` to filter out internal spans (#439) From a8b91cce3334fed3752e192e316480f6e4f876d4 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 10:08:15 -0400 Subject: [PATCH 085/238] [UI] Add traces table to inspect index page (#448) From fb50f7cd16a12640a093db6e5d865a6923b6b86f Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:29:59 -0400 Subject: [PATCH 086/238] Adding traces to Messages (#432) From e62ebb473a9012520a3d6db2a32f3c10d54410d0 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 10:47:24 -0400 Subject: [PATCH 087/238] [UI] Update routing for inspect pages (#449) From b3a2090044fe44703a0e1da0969263989ff008f5 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 10:55:05 -0400 Subject: [PATCH 088/238] [UI] Update routing for run pages (#450) From d7bbbad170f6165f28ee77e0ea93fb9c40fd0af8 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 11:06:21 -0400 Subject: [PATCH 089/238] [UI] Fix trace display name in table (#451) From 69b09522456f5bb68d42a75f1c412fe037b30bc6 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 19 Mar 2024 11:13:39 -0400 Subject: [PATCH 090/238] Allow size to be optional (#452) Model returns error otherwise: 400 None is not of type 'string' - 'size' From 01fba6f7704147b943b3a79b52d43503c1f3c9e6 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 12:00:48 -0400 Subject: [PATCH 091/238] [UI] Fix trace deep links in model playground (#453) From 26d962251b526fe0a9299d611cdbeb8571a5943b Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:25:51 +0000 Subject: [PATCH 092/238] [UI] Add raw mat-table for evals view (#430) * initial ui changes * formatted * Add mocked evals page * Add mocked table prelim * tests * Use EvalResult for now * feedback changes From 36c20c2fe6c7f7eb140f0ce47198be011addb101 Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:27:45 -0400 Subject: [PATCH 093/238] Add embeddings models (#303) From 6cf6ed395a212ad12378748d5a90dc3853ca4585 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 12:39:22 -0400 Subject: [PATCH 094/238] [UI] Update /evaluations route to /evaluate (#454) Matches other verb-based top-level routes. From 3b2eb934fd4978e2857e65e67afd5f276926496b Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 13:25:45 -0400 Subject: [PATCH 095/238] [UI] Make all run buttons consistent in playgrounds (#455) From d2bff9a2c6d2725ae9d11367b7fddda296593e82 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 13:34:04 -0400 Subject: [PATCH 096/238] [UI] Add cmd/ctrl + enter shortcut to playground editors (#456) From 9401587c75213198ea11b8b1bc13e392b9d1ef98 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 19 Mar 2024 15:44:21 -0400 Subject: [PATCH 097/238] [UI] Add landing state for Run page (#465) From 594c177676694b6dfbe51bdb132a1c75d408413f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 08:08:43 -0400 Subject: [PATCH 098/238] [UI] Prevent mat-slider from shrinking (#473) From 1e53ef3333029228eb75c34e300a74dff255e4c3 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 08:21:42 -0400 Subject: [PATCH 099/238] [UI] Adjust element widths for narrow browsers (#474) From 09dd79699d07a5da16ae221775a74d4bf7ddf6d5 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 10:40:38 -0400 Subject: [PATCH 100/238] [UI] Prevent welcome page flicker on action refresh (#475) From c8a8684afbb47e736d5c346bf134c5a47fca6f89 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:00:42 -0400 Subject: [PATCH 101/238] Add tab for Auth input to Flow Runner action (#467) From 4c86f366979f865274882cffe171627af84ce8e3 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 14:34:43 -0400 Subject: [PATCH 102/238] [UI] Add JSON sample to flow runner (#479) From 9422f8849c97abdb0f6ec672d9bd1fec3ba4bc83 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 20 Mar 2024 15:19:10 -0400 Subject: [PATCH 103/238] Generic action runner (#484) From d4e4b43e8ec12d7aa8f006095691690b1e6344e0 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 16:04:34 -0400 Subject: [PATCH 104/238] [UI] Add support for tool primitive on dev UI run page (#488) From 04009b8aee41b1319c4eff6faebb409568cc0954 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 16:05:46 -0400 Subject: [PATCH 105/238] [UI] Tighten up spacing of actions list items (#489) From dc4b43587ea75a0126b6fb392016a855140bf9ab Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 17:12:51 -0400 Subject: [PATCH 106/238] [UI] Trigger change detection on flow runner response (#486) From 4325763669d6287e548e3da499d8ff89acfa06c2 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 17:45:31 -0400 Subject: [PATCH 107/238] [UI] Add cmd/ctrl + enter shortcut to model playground (#485) From 7f9fe6ec051917c44cb618c891108e39eebfcee0 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Wed, 20 Mar 2024 16:04:42 -0700 Subject: [PATCH 108/238] [UI] Update eval results UI to use expandable cards for results (#491) From d4860a4d3afa6d4878e31681d6f76a0f2b3cd59f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 20:15:25 -0400 Subject: [PATCH 109/238] [UI] Prevent scrolling past last line in monaco editor (#495) From eeffa6ba662b41c71da41073c6281edfd6e0690e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 20 Mar 2024 23:08:50 -0400 Subject: [PATCH 110/238] [UI] Use helper class to style pre stacktrace in callout (#502) From b7e1fb0d112ca602ddfdc73238cf95773bd29702 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Wed, 20 Mar 2024 22:37:27 -0700 Subject: [PATCH 111/238] [UI]Evals UI: Update inputs to use a table format (#496) From 67cf547bd42e66ce7bfb71cdb2cb2f60e8b2243a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 21 Mar 2024 15:11:07 -0400 Subject: [PATCH 112/238] [UI] Model playground message styling polish (#515) From f27ebf9cf46869be6d4e30a4ab78fe66391e37ee Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 21 Mar 2024 16:01:48 -0400 Subject: [PATCH 113/238] [UI] Fix json editor to ignore initial value if no schema (#517) From a01f0d35360bf900da9a46414c90b07e56b2dab1 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 21 Mar 2024 16:03:44 -0400 Subject: [PATCH 114/238] [UI] Set retriever name in playground header (#518) From 1ce61edf64d4e15ff93f22d8ff085eb536163011 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 21 Mar 2024 16:11:39 -0400 Subject: [PATCH 115/238] [UI] Prevent JSON sample pre-fill if unnecessary (#520) From f4657e0c938f9f7174b9602753f4cf822f709292 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:56:52 +0000 Subject: [PATCH 116/238] Remove fdescribe in tests (#532) From 87fd14acb38d89af6423d407cc77d742de93dc58 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:03:53 +0000 Subject: [PATCH 117/238] Fix minor UI elements in eval page (#533) * WIP Eval UI changes * Clean scss * simplify name getter * trigger checks again * undo From a2ed530d6457541ce0763d70582b88129adc1402 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:56:50 +0000 Subject: [PATCH 118/238] Add inspect trace option (#540) * WIP Eval UI changes * Clean scss * WIP add inspect button * Add inspect button * Add inspect button * remove target * Use links instead of button * remove unused dep From 895b6fed787d4c25e6259a0755a23533c9eee870 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 22 Mar 2024 20:59:36 +0000 Subject: [PATCH 119/238] Add inspect tab in the Dev UI (#546) * WIP Eval UI changes * Clean scss * WIP add inspect button * Add inspect button * Add inspect button * remove target * Use links instead of button * remove unused dep * Add evaluation tab * Update messaging From 9bec55063ab90465b096edc449fcf0f9fff859e6 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 22 Mar 2024 22:18:52 +0000 Subject: [PATCH 120/238] hide inspect button if no traces (#548) From 8cbb85273d0490fca7e5bd2f522ac6db24a59260 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sat, 23 Mar 2024 18:23:21 -0400 Subject: [PATCH 121/238] [UI] Add typewriter effect to welcoem message (#554) - Also include missing Google Sans fonts From 70ff331e7c3d15b1cd598804bc5992a52e93a08f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sat, 23 Mar 2024 23:07:00 -0400 Subject: [PATCH 122/238] [UI] Tweak logo kerning (#555) From d2e94bfaeb61d65514472aabe1c2d663f2474e31 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sun, 24 Mar 2024 13:50:40 -0400 Subject: [PATCH 123/238] [UI] UI polish for evaluate page (#553) From 52c658a2e1dbc033454b5b99a0d60421aaffacf4 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sun, 24 Mar 2024 18:39:40 -0400 Subject: [PATCH 124/238] [UI] Fix issue in action runner JSON pre-fill (#559) From 988a54bbc810f8c565f1ea29a6ee0fc51269d7c1 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 11:17:11 -0400 Subject: [PATCH 125/238] [UI] Update typewriter animation to move left-to-right (#560) From 82ebefe8cefd687f6597fc3e5be3caf755f8118d Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 12:22:38 -0400 Subject: [PATCH 126/238] [UI] Show custom metadata attributes last in span details (#563) - Also move span duration logic to shared util function and show seconds if > 1000ms. From 0a5290ad9f2dfdf56519d91b541a158e666c2b63 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 12:44:32 -0400 Subject: [PATCH 127/238] [UI] Polish for eval result details pane (#564) From a2a0a38952556139289462ee66e28d2b40072a18 Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:09:17 -0400 Subject: [PATCH 128/238] Add support for text-embeddings (#538) From f2bd0dd9a352174440a379ba6b1203c465d1fd18 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 14:30:24 -0400 Subject: [PATCH 129/238] [UI] Update default font to Google Sans (#565) From 3191ddbb09c4fcaf43b7242bd2c5f08d932084ee Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 16:21:49 -0400 Subject: [PATCH 130/238] [UI] Update span attributes styling (#568) From 2f303477cdac54b8a2baa6071d7fb4743462a902 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 17:17:30 -0400 Subject: [PATCH 131/238] [UI] Update border radius globally (#573) From 827e0abfd5b2b1df024accce4aa624019250ab5a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 17:28:07 -0400 Subject: [PATCH 132/238] [UI] Clip model playground message loading bar to card radius (#576) From 8864140df14bd8e8e3ea6dc97f5a95da05b8f896 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 25 Mar 2024 17:54:34 -0400 Subject: [PATCH 133/238] [UI] Prevent shrinkage of breadcrumb chevron (#577) From a0b276333e729ae5dccb68ba103f25e2216e3330 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 26 Mar 2024 10:13:40 -0400 Subject: [PATCH 134/238] [UI] Upgrade angular deps to ^17.3.1 (#587) From ab8bddb2e94f9f3caf4d6135f819493c2f3e6d4b Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 26 Mar 2024 10:50:38 -0400 Subject: [PATCH 135/238] [UI] Add logo lockup to app bar (#588) From 846b9dbc70d7c5a4c029f1b269dddfdf6020544a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 26 Mar 2024 16:43:20 -0400 Subject: [PATCH 136/238] [UI] Fix table not rendering for errored traces (#607) From 5d618832aa9de0ce935f453aeb8078d9fa5a22d4 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 26 Mar 2024 16:53:18 -0400 Subject: [PATCH 137/238] [UI] Render base64-encoded images in span output (#606) From 0ecb948222b2820086eb370bc69eef9855ff3b6e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 26 Mar 2024 16:53:35 -0400 Subject: [PATCH 138/238] [UI] Update label of expand text button (#608) From 4797f23bf42387ab426965e3196a70ba83e99066 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 26 Mar 2024 22:31:08 -0400 Subject: [PATCH 139/238] [UI] Update lockup with new svg asset (#623) From 11ddf577383be117ee858a14cf90e5959d662dc5 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:42:37 +0000 Subject: [PATCH 140/238] [Eval bugbash] Update tooltip to definitions, visible on entire chip (#624) * Update tooltip to definitions, visible on entire chip * typos From 7c677b3f2e1437597bd728053ef6ebb698f4e888 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:47:51 +0000 Subject: [PATCH 141/238] [Eval bugbash] Show errors as errors in eval UI (#626) * Update tooltip to definitions, visible on entire chip * typos * Mark errors as errors * use ngIf * Add TODO From e429a70204d70bc0bdc52bd2771bf3405c41fdec Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Wed, 27 Mar 2024 20:30:52 +0000 Subject: [PATCH 142/238] [Eval bugbash] Only show icon if failed evaluator (#635) * Update tooltip to definitions, visible on entire chip * typos * WIP icons * Remove unused From 3c6b6ff2c20496c5438ed2d9cf3f00b771cfb1a0 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 27 Mar 2024 16:39:32 -0400 Subject: [PATCH 143/238] [UI] Fix trace timing display now that they are millis (#638) From c7a1a6b21a67e8511b084c3aa94bb42e116b68ac Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 28 Mar 2024 10:22:02 -0700 Subject: [PATCH 144/238] [UI] Fix JSON editor to show up for optional inputs as well (#613) From 1081be96df1a0cbd0c41cecdd9649ef8325619d0 Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:46:55 -0400 Subject: [PATCH 145/238] Add trace id to model playground when error occurs (#631) From 77fb72dde1e753a1e6e603762a4f68173c586c72 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 28 Mar 2024 11:44:58 -0700 Subject: [PATCH 146/238] Display context strings separately instead of a big array (#658) From cf46a29433ddcf80735d06ee655e2aa5c51afa22 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 28 Mar 2024 12:16:41 -0700 Subject: [PATCH 147/238] [UI]: Update date format to medium (#659) From adcb33c085aa62040fef9c02a7bb90e6e992f141 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Fri, 29 Mar 2024 00:56:41 +0000 Subject: [PATCH 148/238] Update error tooltip (#665) * Update error tooltip * typos * Show error message if available From 583049bef35ec4d363e4cd638704d2f638bbce40 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 28 Mar 2024 23:19:51 -0400 Subject: [PATCH 149/238] [UI] Tighten up kerning on mat tab labels (#680) From fafefe45ada2e6698caf31d1779ad638b5983845 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 28 Mar 2024 23:55:19 -0400 Subject: [PATCH 150/238] [UI] Allow resizing of .pre-container and json editor (#682) From 1d8d3304ce3c447dd36e87d5dd5f763e593e0c2e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 07:04:10 -0400 Subject: [PATCH 151/238] [UI] Add tooltips to temperature and top_p controls (#683) From 5cad7e7f0f661a65d18ccbb485da6d58e707a45b Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 08:53:03 -0400 Subject: [PATCH 152/238] [UI] Fix JSON sample autofill in retriever playground (#684) From 0e69f3a63608a4c4b2e42cf49a3b95e342718052 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 11:07:43 -0400 Subject: [PATCH 153/238] [UI] Improve model playground param labels and add tooltips (#686) From c8951b5bec4da7c15e050d4df19d511ee99b4150 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 11:30:21 -0400 Subject: [PATCH 154/238] [UI] Fix trace status in table (#687) From 593c23ac97b6d959fa8babcfa5c003d07eb9f2dd Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 11:39:21 -0400 Subject: [PATCH 155/238] [UI] Update model icon to sparks (#688) From f2809a8b958bfc72abb4abbf6f235a50d045d592 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 12:14:19 -0400 Subject: [PATCH 156/238] [UI] Add action type to runner page title (#690) From 8202af8c45bfce299eb4daf174705ed8842ce542 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 12:37:02 -0400 Subject: [PATCH 157/238] [UI] Add title and close button to expand text dialog (#691) From 21b2276253d12dc0dedd0268fa0df8d1704e8595 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 29 Mar 2024 12:42:58 -0400 Subject: [PATCH 158/238] [UI] Remove redundant title from action runner (#692) From a71da11b878a33d0a4ce29ff8e368f6b8564f551 Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:29:34 -0400 Subject: [PATCH 159/238] Pass thru options to API (#695) From 9ad1326fd76f570c9d204366d08c01378d642cf2 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Sat, 30 Mar 2024 14:32:37 -0400 Subject: [PATCH 160/238] Bump ragas to 0.0.6 (#719) From a5177f446b26a67709bb0c031b16dbe79b84e7f2 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 1 Apr 2024 10:45:52 -0400 Subject: [PATCH 161/238] [UI] Cleanup system prompt styling in model playground (#725) From ea1a93d653291868affcfca0215bda650ed2dd3a Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 1 Apr 2024 12:54:17 -0400 Subject: [PATCH 162/238] Update system/message placeholders (#727) * Update placeholders * Update message.component.ts From 4b59d0aa84f08b7aa0dcdb55d394ba81f241916d Mon Sep 17 00:00:00 2001 From: MaesterChestnut <40321652+MaesterChestnut@users.noreply.github.com> Date: Mon, 1 Apr 2024 15:46:21 -0400 Subject: [PATCH 163/238] Update Eval Error handling (#685) From 91b266ca966af5e80ede82a8293a567e04e28ee0 Mon Sep 17 00:00:00 2001 From: Max Lord Date: Tue, 2 Apr 2024 11:15:12 -0400 Subject: [PATCH 164/238] Clarifying label on button formerly known as "Open in Playground" (#636) - Label now says 'Open in flow runner', 'Open in model runner', etc. to make it more clear which step will be run. - Changing to secondary style button to make it look less like the action will be run immediately. From b2dbf4baeb1bd46759739167381a7d1ad21f1f12 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 2 Apr 2024 12:35:49 -0400 Subject: [PATCH 165/238] [UI] Fix callout content not stretching to fit width (#757) From 9051f3cb39f1eb3cec3865942eded8cbfc7fe1fd Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Tue, 2 Apr 2024 12:13:04 -0700 Subject: [PATCH 166/238] [UI]: Add metrics table in evals results card (#747) From cb3d0f305b71413b2320f607f98c6e2f4e8e7331 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 2 Apr 2024 16:07:01 -0400 Subject: [PATCH 167/238] [UI] Add support for specifying model version in playground (#760) From e2eec3135c382743b83a48cfccd3a5ef06eef280 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 2 Apr 2024 16:56:55 -0400 Subject: [PATCH 168/238] [UI] Remove Evaluate tab in top nav bar (#765) From ffc694006d400e8abad08058c709221c44b0eae9 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 2 Apr 2024 17:33:56 -0400 Subject: [PATCH 169/238] [UI] Use flask icon for Evaluate tab (#772) From 141a756df6b7358ee41955ac79d7a53c36680d4d Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 3 Apr 2024 11:46:56 -0400 Subject: [PATCH 170/238] [UI] Style updates to eval result details (#790) From 76f981da5de51b8b551be5dc74415f7758b6f120 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 3 Apr 2024 14:05:30 -0400 Subject: [PATCH 171/238] [UI] Render eval metric name in error callout consistently (#792) From 13adb38f9b377c8a9b0f78446e034bea805085c8 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 3 Apr 2024 14:22:08 -0400 Subject: [PATCH 172/238] [UI] Fix span duration display (#797) From 29ba1719e3b9e3cbb1515d1fd5943cc016423660 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 3 Apr 2024 15:44:46 -0400 Subject: [PATCH 173/238] Show safety errors in the model runner (#800) From 9b95706a91e1a56ff169669a5cdb05b4e5b01683 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 3 Apr 2024 16:57:55 -0400 Subject: [PATCH 174/238] Rename model playground => runner (#803) From 649a2f512f2f27d270903f72a3584cccbda0e244 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 3 Apr 2024 18:22:28 -0400 Subject: [PATCH 175/238] Rename retriever playground => runner (#805) From a2c9b25631a92ff4a328664d35adede557e74cba Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 3 Apr 2024 20:53:43 -0400 Subject: [PATCH 176/238] [UI] Adjust metrics table to be full-width (#810) From 5a65e1e9217fb661fec18bc313c4f63433820a1b Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 3 Apr 2024 21:02:26 -0400 Subject: [PATCH 177/238] [UI] Only show eval zero state when loaded (#811) Prevents a quick distracting flash of the zero state when the page loads. From 9b1fbffbdb7ceedf32393aab31077048655c49a3 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 3 Apr 2024 21:16:48 -0400 Subject: [PATCH 178/238] [UI] Set All traces as default in Inspect view (#812) From debd2cedb9e7145be092cb1c3af9da77a21e8bc5 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Thu, 4 Apr 2024 08:46:31 -0700 Subject: [PATCH 179/238] [UI] ThemeToggleService unit tests (#816) From 9629202b303729305fc42653ad044a948871f54d Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 4 Apr 2024 12:29:11 -0400 Subject: [PATCH 180/238] [UI] Make spans deep-linkable in trace + flow details views (#819) From 83914bc66f6dfe2ba828fab306662b0e48f71243 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 4 Apr 2024 12:29:27 -0400 Subject: [PATCH 181/238] [UI] Update model runner title to use selected model in config (#822) From c88ea1057e638729b29bbb1771ea7f4af462470a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 4 Apr 2024 17:32:40 -0400 Subject: [PATCH 182/238] [UI] Clear out images from data-rendered upon receiving new input (#840) From 5556cdcd65953051266580a5364e4a9e8685a604 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 4 Apr 2024 23:22:35 -0400 Subject: [PATCH 183/238] [UI] Hide append mode for models that do not support multiturn (#847) From 820f77e168f242366ddf0e400268a937facebffa Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 5 Apr 2024 00:27:05 -0400 Subject: [PATCH 184/238] [UI] Show banner for unsupported models (#848) From 6efe6fd4402fe4a0a095ac89a5d3e8f54b105d22 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 5 Apr 2024 10:14:43 -0400 Subject: [PATCH 185/238] [UI] Reset scroll position of input/output when switching spans (#852) From c1e68d15c3acdc85d0931380531fa1ce05d519a5 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 5 Apr 2024 10:18:16 -0400 Subject: [PATCH 186/238] [UI] Hide "Add message" if model does not support multiturn (#853) From fe629f50df091d2cc350e26296a8f06ebd6914d8 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 5 Apr 2024 11:14:40 -0400 Subject: [PATCH 187/238] Fix missed version 0.5.0-rc.1 (#858) From 16aa95483f4570c2a390dc95d53cae2bc44dbed1 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 5 Apr 2024 12:00:40 -0400 Subject: [PATCH 188/238] [UI] Fix display of system prompt (#860) From 0104e552d910b0c3ec8c5ab870989af9e42b2146 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 5 Apr 2024 12:36:26 -0400 Subject: [PATCH 189/238] [UI] Fix tools icon (#862) From 0d5a432a519da3e104810ea11a4dfc9313d8c366 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sun, 7 Apr 2024 17:08:21 -0400 Subject: [PATCH 190/238] [UI] Prevent stuck browser back when redirecting to first evaluation run (#13) From 4ea7af7970c4635ebaa1fd3bcce6606781bef006 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 8 Apr 2024 11:54:50 -0400 Subject: [PATCH 191/238] [UI] Add missing app text color style (#16) From 1a8ea09f32b84217abf91e1e0e7aea7e524a96dc Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 8 Apr 2024 20:30:32 -0400 Subject: [PATCH 192/238] [UI] Apply theme to scrollbars (#20) From 594f959312140fe409068ae559a046f7cd1399af Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 10 Apr 2024 10:00:05 -0400 Subject: [PATCH 193/238] [UI] Clarify ID in flows/traces tables (#23) From 1a67dc0bbba1bd8de3560fdc816749614b7dae55 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 10 Apr 2024 16:01:44 -0400 Subject: [PATCH 194/238] [UI] Show flow error in trace details view, if applicable (#28) From 634969fa709d998b36ee19711c251b0cff922ea0 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 10 Apr 2024 16:40:37 -0400 Subject: [PATCH 195/238] [UI] Fix eval zero state callout spacing (#24) From 8149ebfc8d1a2daecc78b3cafec00690bdd85c08 Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:33:00 -0400 Subject: [PATCH 196/238] Export textEmbedding (#36) From bc7981395a67741c2154942c083a8c7ee5763ebe Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 11 Apr 2024 17:41:40 -0400 Subject: [PATCH 197/238] [UI] Update README doc with up-to-date instructions (#50) From 98b13c7fd945ac7244f52f6bf62fd4c4b4d1a995 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 11 Apr 2024 21:00:47 -0400 Subject: [PATCH 198/238] [UI] Create skeleton prompt runner component (#54) Will serve as a base for prompt-specific runner features that we will add. From 4aad06c49a0ae08e4f85f3079d2bb515f5eeb7b7 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 12 Apr 2024 05:01:04 -0400 Subject: [PATCH 199/238] [UI] Add icon to all view trace buttons (#57) From 1d282d134f491df1728c8dc226ab2b41a5e05c6e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 12 Apr 2024 13:10:49 -0400 Subject: [PATCH 200/238] [UI] Show template in prompt runner next to input (#58) From 979652c70767e5e72ddcde4aa7f1ff07fae00d01 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 12 Apr 2024 13:23:52 -0400 Subject: [PATCH 201/238] [UI] Use button toggle group for inspect table filter (#56) From 7a60ea12cf6467e413ebc53301bb9a08e5047d13 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 12 Apr 2024 19:13:04 -0400 Subject: [PATCH 202/238] [UI] Update play icon for run/dispatch span states (#60) From 9a4dd85c0c6be8fdfb3efb6eb75bfceb3f54b58e Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 15 Apr 2024 22:05:02 -0400 Subject: [PATCH 203/238] More sensible default model params (#65) From 0fff85eea3f665b99d55124d905dfb1ac9b1a896 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 15 Apr 2024 22:54:25 -0400 Subject: [PATCH 204/238] Always clear message when not in chat mode - otherwise if an error is shown, we'll still see the previous message. (#67) From 394fc0eff4b16650b54ab96fe62776311b620a07 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Tue, 16 Apr 2024 11:03:03 -0400 Subject: [PATCH 205/238] [UI] Show raw prompt template in modal (#70) From 5acbe659ba1e92e64f3d9be402733cc553cae587 Mon Sep 17 00:00:00 2001 From: Max Lord Date: Tue, 16 Apr 2024 12:28:19 -0400 Subject: [PATCH 206/238] Nesting user input in prompt runner (#72) From 9ff3c50f69e97444e7adb6e9a6322b99ea18f37e Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 17 Apr 2024 11:29:17 -0400 Subject: [PATCH 207/238] [UI] Add support for prompt variants (#74) From ef1c61f11f30565c2908149f49e8d6c91bd1945f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 17 Apr 2024 12:11:03 -0400 Subject: [PATCH 208/238] Allow system role for Gemini 1.5 Pro (#85) Also removes references to OpenAI from UI. From a384483a28a45eeff3199e4a84d18b5dcff8f9fb Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 17 Apr 2024 13:59:18 -0400 Subject: [PATCH 209/238] Create modular component for a multi-modal message (#83) From 7cb3f754e4663a14b1cc7b3aaf75f5af68178a86 Mon Sep 17 00:00:00 2001 From: ssbushi <66321939+ssbushi@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:58:48 -0400 Subject: [PATCH 210/238] Update faithfulness to v0.1.7 (#87) * Update faithfulness to v0.1.7 * Update METADATA From b6abf1fe5235703065e62850e0e3bba27028e4c6 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 17 Apr 2024 23:43:31 -0400 Subject: [PATCH 211/238] [UI] Add prompt variant to query params to support deep-linking (#88) From 4da8e9d550645668852a4743ac21196e1bac5ae7 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 18 Apr 2024 10:29:53 -0400 Subject: [PATCH 212/238] [UI] Fix race condition when setting content in monaco (#96) From 2435d633583a2a7b30ead87af7c977fea1268e5b Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 18 Apr 2024 10:52:08 -0400 Subject: [PATCH 213/238] [UI] Small visual fix in app nav bar (#98) From c0a1effc118f2ef1ec07850cbb5e0357bba3619a Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 18 Apr 2024 12:37:43 -0400 Subject: [PATCH 214/238] [UI] Fix incorrect height for modal runner header (#101) From c45c26eae3537a55d50ea927268bbb1e653e195f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Thu, 18 Apr 2024 12:38:08 -0400 Subject: [PATCH 215/238] [UI] Update placeholder label for model version select (#100) From 6646c4603768fbecbcf577cba394d849910d4279 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Thu, 18 Apr 2024 15:09:08 -0400 Subject: [PATCH 216/238] Message list component (#84) Co-authored-by: Chris Chestnut Co-authored-by: Michael Doyle From 9437a11088d884ebc49b4a2a1f2a0a936f20108f Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 19 Apr 2024 11:48:25 -0400 Subject: [PATCH 217/238] [UI] Fix view evaluation report button to read correct metdata (#119) From fd190097fda2f6e8e9a752d3b0e4ab5c7b0a44c3 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 19 Apr 2024 13:21:47 -0400 Subject: [PATCH 218/238] [UI] Save action sidebar expansion state to `localStorage` (#120) From c52bc156050682a7957e09d8587dfdbf76e8559b Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Fri, 19 Apr 2024 11:11:44 -0700 Subject: [PATCH 219/238] [UI]: Move model config params to a separate component (#103) From 04267f6b0a264e36e17ac8b2e6a63c427d884df5 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Fri, 19 Apr 2024 12:25:41 -0700 Subject: [PATCH 220/238] [UI] Update model runner to use the new model config component (#124) From 2e809166c47a40ff1ead3df80f6dc710e33bb940 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Fri, 19 Apr 2024 13:01:42 -0700 Subject: [PATCH 221/238] [UI] Pull the new defaults for model config into the new config component (#125) From fd8d28fadfdb00bb4bfebc2fbb57a52fe96b49c3 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Fri, 19 Apr 2024 17:06:00 -0400 Subject: [PATCH 222/238] [UI] Add ability to export prompt file from model runner (#115) From 4a32c0feebc46d6b4955c8221154ba7ff84bfb80 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sat, 20 Apr 2024 23:30:29 -0400 Subject: [PATCH 223/238] [UI] Fix model versions not being loaded on initial render (#131) Fixes https://github.com/google/genkit/issues/130. This is more of a stop-gap fix, going to explore refactoring these components to utilize Angular signals to eliminate this class of error entirely. From 3ec09083c52d33e6accdb72ea250d777b9c518f1 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Sun, 21 Apr 2024 15:15:53 -0400 Subject: [PATCH 224/238] Integrate the new MessageList component into the ModelRunner (#114) From fe36ca9810ef54731002c459f27dc6304129037c Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 22 Apr 2024 09:33:50 -0400 Subject: [PATCH 225/238] [UI] Refactor model-config to use signals (#133) From 4c8f2be81e8503dff8ab261ecca4dfa8bc9b75f4 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 22 Apr 2024 14:45:03 -0400 Subject: [PATCH 226/238] Create placeholder for system prompt and first user message (#144) From c4af59177c807274a438b25ba666fc64a9bc4dc7 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 22 Apr 2024 14:50:23 -0400 Subject: [PATCH 227/238] [UI] Remove oops from model config template (#143) From d3e1d5aaaf9d2d47b2465af62e93aab8d0472413 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 22 Apr 2024 16:04:59 -0400 Subject: [PATCH 228/238] Ensure selected model is set when using left nav (#148) From 15cd12eab8fbeaee135428db05e8ff1db0d948da Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Mon, 22 Apr 2024 16:14:11 -0400 Subject: [PATCH 229/238] [UI] Prevent button icons from flex-shrinking (#151) From 3b2b42b4cd4a8580fbcd94401846d75ae30c5e36 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Mon, 22 Apr 2024 23:29:59 -0400 Subject: [PATCH 230/238] Show large multimedia in a modal (#156) From 0dc4352f5bf30e3f94b20cb31863febb6ac1cdeb Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 23 Apr 2024 13:25:00 -0400 Subject: [PATCH 231/238] Enable all image types in model runner (#160) From 9dce79418c8647cbef1f3730cf511e7e6ef5920b Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Tue, 23 Apr 2024 15:13:42 -0400 Subject: [PATCH 232/238] Re-enable gemini vision models (#168) From efb081ba1040f31b2af199d12683866ee2bc4141 Mon Sep 17 00:00:00 2001 From: shrutip90 Date: Tue, 23 Apr 2024 13:57:46 -0700 Subject: [PATCH 233/238] [UI] Remove system prompt for single-turn models (#169) From 23b1b5593437a677f07afef6db323d991522a086 Mon Sep 17 00:00:00 2001 From: Michael Doyle Date: Wed, 24 Apr 2024 10:36:39 -0400 Subject: [PATCH 234/238] Set a reasonable (but arbitrary) number of media files per message (#172) From 0087bb0c6d21620d6883e64fa5a50a3daf8fdf57 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Wed, 24 Apr 2024 17:35:25 -0400 Subject: [PATCH 235/238] [UI] Remove obsolete MONACO_PATH provider (unused) (#182) From 57eb6042dc45916d2eb9af2b153d04415db68c13 Mon Sep 17 00:00:00 2001 From: Anthony Barone Date: Sun, 28 Apr 2024 18:44:06 -0400 Subject: [PATCH 236/238] [UI] Sort eval metrics for consistent/comparable viewing (#209) Fixes #207. From 225dad6c38998b9077d21bf8ea97a95aea4d745c Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 1 May 2024 09:24:24 -0400 Subject: [PATCH 237/238] change action latency name (#200) Change the name of the action latency histogram from "genkit.action.action_latency" to "genkit.action.latency" to avoid stutter. From 983658f789558e0792a9399f654817a175b44552 Mon Sep 17 00:00:00 2001 From: Tunde Agboola Date: Wed, 1 May 2024 17:03:31 -0400 Subject: [PATCH 238/238] Add strong response types for vertex evaluators --- js/plugins/vertexai/src/evaluation.ts | 41 +++++++++++++++++--- js/plugins/vertexai/src/evaluator_factory.ts | 31 ++++++++++++--- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/js/plugins/vertexai/src/evaluation.ts b/js/plugins/vertexai/src/evaluation.ts index c8fa1937e0..7cc94c026d 100644 --- a/js/plugins/vertexai/src/evaluation.ts +++ b/js/plugins/vertexai/src/evaluation.ts @@ -18,6 +18,7 @@ import { BaseDataPoint } from '@genkit-ai/ai/evaluator'; import { Action } from '@genkit-ai/core'; import { GoogleAuth } from 'google-auth-library'; import { JSONClient } from 'google-auth-library/build/src/auth/googleauth'; +import z from 'zod'; import { EvaluatorFactory } from './evaluator_factory'; /** @@ -57,10 +58,6 @@ export function vertexEvaluators( const metricType = isConfig(metric) ? metric.type : metric; const metricSpec = isConfig(metric) ? metric.metricSpec : {}; - console.log( - `Creating evaluator for metric ${metricType} with metricSpec ${metricSpec}` - ); - switch (metricType) { case VertexAIEvaluationMetricType.BLEU: { return createBleuEvaluator(factory, metricSpec); @@ -84,6 +81,12 @@ function isConfig( return (config as VertexAIEvaluationMetricConfig).type !== undefined; } +const BleuResponseSchema = z.object({ + bleuResults: z.object({ + bleuMetricValues: z.array(z.object({ score: z.number() })), + }), +}); + // TODO: Add support for batch inputs function createBleuEvaluator( factory: EvaluatorFactory, @@ -95,6 +98,7 @@ function createBleuEvaluator( displayName: 'BLEU', definition: 'Computes the BLEU score by comparing the output against the ground truth', + responseSchema: BleuResponseSchema, }, (datapoint) => { if (!datapoint.reference) { @@ -124,6 +128,12 @@ function createBleuEvaluator( ); } +const RougeResponseSchema = z.object({ + rougeResults: z.object({ + rougeMetricValues: z.array(z.object({ score: z.number() })), + }), +}); + // TODO: Add support for batch inputs function createRougeEvaluator( factory: EvaluatorFactory, @@ -135,6 +145,7 @@ function createRougeEvaluator( displayName: 'ROUGE', definition: 'Computes the ROUGE score by comparing the output against the ground truth', + responseSchema: RougeResponseSchema, }, (datapoint) => { if (!datapoint.reference) { @@ -162,6 +173,14 @@ function createRougeEvaluator( ); } +const SafetyResponseSchema = z.object({ + safetyResult: z.object({ + score: z.number(), + explanation: z.string(), + confidence: z.number(), + }), +}); + function createSafetyEvaluator( factory: EvaluatorFactory, metricSpec: any @@ -171,6 +190,7 @@ function createSafetyEvaluator( metric: VertexAIEvaluationMetricType.SAFETY, displayName: 'Safety', definition: 'Assesses the level of safety of an output', + responseSchema: SafetyResponseSchema, }, (datapoint) => { return { @@ -182,7 +202,7 @@ function createSafetyEvaluator( }, }; }, - (response: any, datapoint: BaseDataPoint) => { + (response, datapoint: BaseDataPoint) => { return { testCaseId: datapoint.testCaseId, evaluation: { @@ -196,6 +216,14 @@ function createSafetyEvaluator( ); } +const GroundednessResponseSchema = z.object({ + groundednessResult: z.object({ + score: z.number(), + explanation: z.string(), + confidence: z.number(), + }), +}); + function createGroundednessEvaluator( factory: EvaluatorFactory, metricSpec: any @@ -206,6 +234,7 @@ function createGroundednessEvaluator( displayName: 'Groundedness', definition: 'Assesses the ability to provide or reference information included only in the context', + responseSchema: GroundednessResponseSchema, }, (datapoint) => { return { @@ -218,7 +247,7 @@ function createGroundednessEvaluator( }, }; }, - (response: any, datapoint: BaseDataPoint) => { + (response, datapoint: BaseDataPoint) => { return { testCaseId: datapoint.testCaseId, evaluation: { diff --git a/js/plugins/vertexai/src/evaluator_factory.ts b/js/plugins/vertexai/src/evaluator_factory.ts index d2db0ff6ad..017de6a948 100644 --- a/js/plugins/vertexai/src/evaluator_factory.ts +++ b/js/plugins/vertexai/src/evaluator_factory.ts @@ -19,6 +19,7 @@ import { Action } from '@genkit-ai/core'; import { runInNewSpan } from '@genkit-ai/core/tracing'; import { GoogleAuth } from 'google-auth-library'; import { JSONClient } from 'google-auth-library/build/src/auth/googleauth'; +import z from 'zod'; import { VertexAIEvaluationMetricType } from './evaluation'; export class EvaluatorFactory { @@ -28,14 +29,18 @@ export class EvaluatorFactory { private readonly projectId: string ) {} - create( + create( config: { metric: VertexAIEvaluationMetricType; displayName: string; definition: string; + responseSchema: ResponseType; }, toRequest: (datapoint: BaseDataPoint) => any, - responseHandler: (response: any, datapoint: BaseDataPoint) => any + responseHandler: ( + response: z.infer, + datapoint: BaseDataPoint + ) => any ): Action { return defineEvaluator( { @@ -44,14 +49,21 @@ export class EvaluatorFactory { definition: config.definition, }, async (datapoint: BaseDataPoint) => { - const response = await this.evaluateInstances(toRequest(datapoint)); + const responseSchema = config.responseSchema; + const response = await this.evaluateInstances( + toRequest(datapoint), + responseSchema + ); return responseHandler(response, datapoint); } ); } - async evaluateInstances(partialRequest: any) { + async evaluateInstances( + partialRequest: any, + responseSchema: ResponseType + ): Promise> { const locationName = `projects/${this.projectId}/locations/${this.location}`; return await runInNewSpan( { @@ -64,15 +76,22 @@ export class EvaluatorFactory { location: locationName, ...partialRequest, }; + metadata.input = request; const client = await this.auth.getClient(); + const url = `https://${this.location}-aiplatform.googleapis.com/v1beta1/${locationName}:evaluateInstances`; const response = await client.request({ - url: `https://${this.location}-aiplatform.googleapis.com/v1beta1/${locationName}:evaluateInstances`, + url, method: 'POST', body: JSON.stringify(request), }); metadata.output = response.data; - return response.data as any; + + try { + return responseSchema.parse(response.data); + } catch (e) { + throw new Error(`Error parsing ${url} API response: ${e}`); + } } ); }