feat(nx-dev): store user feedback in supabase too (#18563)

nx-dev/data-access-ai/src/lib/data-access-ai.ts

@@ -1,7 +1,11 @@
 // based on:
 // https://github.com/supabase-community/nextjs-openai-doc-search/blob/main/pages/api/vector-search.ts
 
-import { createClient } from '@supabase/supabase-js';
+import {
+  PostgrestSingleResponse,
+  SupabaseClient,
+  createClient,
+} from '@supabase/supabase-js';
 import GPT3Tokenizer from 'gpt3-tokenizer';
 import {
   Configuration,
@@ -45,6 +49,8 @@ let chatFullHistory: ChatItem[] = [];
 
 let totalTokensSoFar = 0;
 
+let supabaseClient: SupabaseClient<any, 'public', any>;
+
 export async function nxDevDataAccessAi(
   query: string,
   aiResponse?: string
@@ -54,6 +60,13 @@ export async function nxDevDataAccessAi(
   sources: { heading: string; url: string }[];
   sourcesMarkdown: string;
 }> {
+  if (!supabaseClient) {
+    supabaseClient = createClient(
+      supabaseUrl as string,
+      supabaseServiceKey as string
+    );
+  }
+
   if (chatFullHistory.length > MAX_HISTORY_LENGTH) {
     chatFullHistory.slice(0, MAX_HISTORY_LENGTH - 4);
   }
@@ -65,11 +78,6 @@ export async function nxDevDataAccessAi(
       throw new UserError('Missing query in request data');
     }
 
-    const supabaseClient = createClient(
-      supabaseUrl as string,
-      supabaseServiceKey as string
-    );
-
     // Moderate the content to comply with OpenAI T&C
     const sanitizedQuery = query.trim();
     const moderationResponse: CreateModerationResponse = await openai
@@ -210,7 +218,7 @@ export async function nxDevDataAccessAi(
 
     return {
       textResponse: responseWithoutBadLinks,
-      usage: response.data.usage,
+      usage: response.data.usage as CreateCompletionResponseUsage,
       sources,
       sourcesMarkdown: toMarkdownList(sources),
     };
@@ -239,3 +247,17 @@ export function resetHistory() {
 export function getHistory(): ChatItem[] {
   return chatFullHistory;
 }
+
+export async function handleFeedback(feedback: {}): Promise<
+  PostgrestSingleResponse<null>
+> {
+  return supabaseClient.from('feedback').insert(feedback);
+}
+
+export async function handleQueryReporting(queryInfo: {}) {
+  const { error } = await supabaseClient.from('user_queries').insert(queryInfo);
+
+  if (error) {
+    console.error('Error storing the query info in Supabase: ', error);
+  }
+}

nx-dev/feature-ai/src/lib/feature-ai.tsx

@@ -1,4 +1,4 @@
-import { ReactNode, useEffect, useRef, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { Button } from '@nx/nx-dev/ui-common';
 import { sendCustomEvent } from '@nx/nx-dev/feature-analytics';
 import { renderMarkdown } from '@nx/nx-dev/ui-markdoc';
@@ -7,12 +7,13 @@ import {
   resetHistory,
   getProcessedHistory,
   ChatItem,
+  handleFeedback,
+  handleQueryReporting,
 } from '@nx/nx-dev/data-access-ai';
 import { warning, infoBox, noResults } from './utils';
 
 export function FeatureAi(): JSX.Element {
   const [chatHistory, setChatHistory] = useState<ChatItem[] | null>([]);
-  const [finalResult, setFinalResult] = useState<null | ReactNode>(null);
   const [textResponse, setTextResponse] = useState<undefined | string>('');
   const [error, setError] = useState(null);
   const [query, setSearchTerm] = useState('');
@@ -60,6 +61,10 @@ export function FeatureAi(): JSX.Element {
     }
     sendCustomEvent('ai_query', 'ai', 'query', undefined, {
       query,
+    });
+    handleQueryReporting({
+      action: 'ai_query',
+      query,
       ...usage,
     });
     const sourcesMd =
@@ -73,9 +78,6 @@ ${sourcesMarkdown}
 \n
         `;
 
-    setFinalResult(
-      renderMarkdown(completeText + sourcesMd, { filePath: '' }).node
-    );
     if (completeText) {
       setChatHistory([
         ...getProcessedHistory(),
@@ -84,11 +86,14 @@ ${sourcesMarkdown}
     }
   };
 
-  const handleFeedback = (type: 'good' | 'bad', index: number) => {
+  const handleUserFeedback = (result: 'good' | 'bad', index: number) => {
     try {
-      sendCustomEvent('ai_feedback', 'ai', type, undefined, {
+      sendCustomEvent('ai_feedback', 'ai', result);
+      handleFeedback({
+        action: 'evaluation',
+        result,
         query,
-        result: finalResult,
+        response: textResponse,
         sources,
       });
       setFeedbackSent((prev) => ({ ...prev, [index]: true }));
@@ -99,7 +104,6 @@ ${sourcesMarkdown}
 
   const handleReset = () => {
     resetHistory();
-    setFinalResult(null);
     setSearchTerm('');
     setTextResponse('');
     setSources('');
@@ -180,7 +184,7 @@ ${sourcesMarkdown}
               <Button
                 variant="primary"
                 size="small"
-                onClick={() => handleFeedback('good', index)}
+                onClick={() => handleUserFeedback('good', index)}
               >
                 Answer was helpful{' '}
                 <span role="img" aria-label="thumbs-up">
@@ -190,7 +194,7 @@ ${sourcesMarkdown}
               <Button
                 variant="primary"
                 size="small"
-                onClick={() => handleFeedback('bad', index)}
+                onClick={() => handleUserFeedback('bad', index)}
               >
                 Answer looks wrong{' '}
                 <span role="img" aria-label="thumbs-down">