From ce28096835460cf957d06ca1dd566bc74a045624 Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 09:18:51 -0500 Subject: [PATCH 1/8] OpenAI Organization added as request param + env var, and Pinecone fix, set index and namespace as env vars --- apps/AppStandalone/.env.template | 7 ++++- .../lib/backend/api_requests/api_calls.dart | 28 +++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/apps/AppStandalone/.env.template b/apps/AppStandalone/.env.template index c2d89b3a..2da4127f 100644 --- a/apps/AppStandalone/.env.template +++ b/apps/AppStandalone/.env.template @@ -1,5 +1,8 @@ OPENAI_API_KEY= +OPENAI_ORGANIZATION= PINECONE_API_KEY= +PINECONE_INDEX_URL= +PINECONE_INDEX_NAMESPACE= FIREBASE_API_KEY= FIREBASE_AUTH_DOMAIN= @@ -7,4 +10,6 @@ FIREBASE_PROJECT_ID= FIREBASE_STORAGE_BUCKET= FIREBASE_MESSAGE_SENDER_ID= FIREBASE_APP_ID= -FIREBASE_MEASUREMENT_ID= \ No newline at end of file +FIREBASE_MEASUREMENT_ID= +SUPABASE_URL= +SUPABASE_ANON_KEY= \ No newline at end of file diff --git a/apps/AppStandalone/lib/backend/api_requests/api_calls.dart b/apps/AppStandalone/lib/backend/api_requests/api_calls.dart index 6a7038cc..4b786ab2 100644 --- a/apps/AppStandalone/lib/backend/api_requests/api_calls.dart +++ b/apps/AppStandalone/lib/backend/api_requests/api_calls.dart @@ -1,5 +1,7 @@ import 'dart:convert'; +import 'package:flutter/material.dart'; + import '/flutter_flow/flutter_flow_util.dart'; import 'api_manager.dart'; import '../../env/env.dart'; @@ -29,6 +31,7 @@ class StructuredMemoryCall { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -70,6 +73,7 @@ class ChatGPTFeedbackCall { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -116,6 +120,7 @@ class TestCall { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -150,6 +155,7 @@ class ChatGPTWhisperCall { headers: { 'Content-Type': 'multipart/form-data', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: { 'file': file, @@ -193,6 +199,7 @@ class VoiceCommandRespondCall { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -233,6 +240,7 @@ class DailyMemoriesCall { headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -273,6 +281,7 @@ class SummariesCall { headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -314,6 +323,7 @@ class IsFeeedbackUsefulCall { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -355,6 +365,7 @@ class BoolTestCall { headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -390,6 +401,7 @@ class SendFullPromptCall { callType: ApiCallType.POST, headers: { 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -478,11 +490,11 @@ class VectorizeCall { static Future call({ String? input = '', }) async { - final ffApiRequestBody = ''' -{ - "input": "$input", - "model": "text-embedding-3-small" -}'''; + final ffApiRequestBody = jsonEncode({ + 'input': input, + 'model': 'text-embedding-3-small', + }); + debugPrint('ffApiRequestBody: $ffApiRequestBody'); return ApiManager.instance.makeApiCall( callName: 'Vectorize', apiUrl: 'https://api.openai.com/v1/embeddings', @@ -490,6 +502,7 @@ class VectorizeCall { headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }, params: {}, body: ffApiRequestBody, @@ -532,12 +545,11 @@ class CreateVectorPineconeCall { } } ], - "namespace": "ns1" + "namespace": "${Env.pineconeIndexNamespace}" }'''; return ApiManager.instance.makeApiCall( callName: 'createVectorPinecone', - apiUrl: - 'https://index-i7j24t4.svc.gcp-starter.pinecone.io/vectors/upsert', + apiUrl: '${Env.pineconeIndexUrl}/vectors/upsert', callType: ApiCallType.POST, headers: { 'Api-Key': Env.pineconeApiKey, From e704f28910fc04efffb621be5430dac0e6fc22c5 Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 09:19:15 -0500 Subject: [PATCH 2/8] env.dart updated file to handle openai + pinecone env vars --- apps/AppStandalone/lib/env/env.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/AppStandalone/lib/env/env.dart b/apps/AppStandalone/lib/env/env.dart index 506e02ef..86fc8713 100644 --- a/apps/AppStandalone/lib/env/env.dart +++ b/apps/AppStandalone/lib/env/env.dart @@ -6,10 +6,16 @@ abstract class Env { // OpenAI @EnviedField(varName: 'OPENAI_API_KEY') static const String openAIApiKey = _Env.openAIApiKey; + @EnviedField(varName: 'OPENAI_ORGANIZATION') + static const String openAIOrganization = _Env.openAIOrganization; // Pinecone @EnviedField(varName: 'PINECONE_API_KEY') static const String pineconeApiKey = _Env.pineconeApiKey; + @EnviedField(varName: 'PINECONE_INDEX_URL') + static const String pineconeIndexUrl = _Env.pineconeIndexUrl; + @EnviedField(varName: 'PINECONE_INDEX_NAMESPACE') + static const String pineconeIndexNamespace = _Env.pineconeIndexNamespace; // Firebase @EnviedField(varName: 'FIREBASE_API_KEY') From fdad59b8de2c3cc2bad54e9bd40fc151ba45a6ec Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 09:21:47 -0500 Subject: [PATCH 3/8] Refactor actions.dart, split in multiple functions, just made it understandable and modular --- apps/AppStandalone/lib/actions/actions.dart | 591 ++++++++++---------- 1 file changed, 289 insertions(+), 302 deletions(-) diff --git a/apps/AppStandalone/lib/actions/actions.dart b/apps/AppStandalone/lib/actions/actions.dart index 7c4eece3..9358781f 100644 --- a/apps/AppStandalone/lib/actions/actions.dart +++ b/apps/AppStandalone/lib/actions/actions.dart @@ -1,3 +1,4 @@ +import 'package:flutter/material.dart'; import '/auth/firebase_auth/auth_util.dart'; import '/backend/api_requests/api_calls.dart'; import '/backend/backend.dart'; @@ -6,340 +7,323 @@ import '/flutter_flow/flutter_flow_util.dart'; import '/actions/actions.dart' as action_blocks; import '/custom_code/actions/index.dart' as actions; import '/flutter_flow/custom_functions.dart' as functions; -import 'package:flutter/material.dart'; -Future memoryCreationBlock(BuildContext context) async { - ApiCallResponse? sMemory; - ApiCallResponse? fMemory; - ApiCallResponse? isFeedbackUsefulResult; - ApiCallResponse? vectorFromOpenai; - MemoriesRecord? memoryDone; - ApiCallResponse? vectorAdded; +// Process the creation of memory records +Future memoryCreationBlock(BuildContext context) async { + var structuredMemoryResponse = await structureMemory(); + if (functions.memoryContainsNA(extractContent(structuredMemoryResponse))) { + await saveFailureMemory(structuredMemoryResponse); + } else { + await processStructuredMemory(structuredMemoryResponse); + } +} - // StructureMemoryAPI +// Call to the API to get structured memory +Future structureMemory() async { logFirebaseEvent('memoryCreationBlock_StructureMemoryAPI'); - sMemory = await StructuredMemoryCall.call( + return await StructuredMemoryCall.call( memory: functions.jsonEncodeString(FFAppState().lastMemory), ); - if (functions.memoryContainsNA(getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString()) == - true) { - logFirebaseEvent('memoryCreationBlock_backend_call'); - - await MemoriesRecord.collection.doc().set({ - ...createMemoriesRecordData( - memory: FFAppState().lastMemory, - user: currentUserReference, - structuredMemory: getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - feedback: '', - toShowToUserShowHide: 'Hide', - emptyMemory: - FFAppState().lastMemory == '', - isUselessMemory: true, - ), - ...mapToFirestore( - { - 'date': FieldValue.serverTimestamp(), - }, - ), - }); +} + +// Extract content from API response +String extractContent(ApiCallResponse response) { + return getJsonField( + (response.jsonBody ?? ''), + r'''$.choices[0].message.content''', + ).toString(); +} + +// Save failure memory when structured memory contains NA +Future saveFailureMemory(ApiCallResponse response) async { + logFirebaseEvent('memoryCreationBlock_backend_call'); + await MemoriesRecord.collection.doc().set({ + ...createMemoriesRecordData( + memory: FFAppState().lastMemory, + user: currentUserReference, + structuredMemory: extractContent(response), + feedback: '', + toShowToUserShowHide: 'Hide', + emptyMemory: FFAppState().lastMemory == '', + isUselessMemory: true, + ), + ...mapToFirestore({ + 'date': FieldValue.serverTimestamp(), + }), + }); +} + +// Process structured memory when it's valid +Future processStructuredMemory(ApiCallResponse response) async { + if (functions.xGreaterThany(functions.wordCount(FFAppState().lastMemory), 1)!) { + updateAppStateForProcessing(); + var feedbackResponse = await requestFeedback(response); + await evaluateFeedback(feedbackResponse); + updateFinalAppState(feedbackResponse); } else { - if (functions.xGreaterThany( - functions.wordCount(FFAppState().lastMemory), 1)!) { - logFirebaseEvent('memoryCreationBlock_update_app_state'); - FFAppState().update(() { - FFAppState().memoryCreationProcessing = true; - }); - // FEEDBACKapi - logFirebaseEvent('memoryCreationBlock_FEEDBACKapi'); - fMemory = await ChatGPTFeedbackCall.call( - memory: functions.jsonEncodeString(FFAppState().lastMemory), - structuredMemory: functions.jsonEncodeString(getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString()), - ); - logFirebaseEvent('memoryCreationBlock_custom_action'); - await actions.debugLog( - getJsonField( - (fMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - ); - logFirebaseEvent('memoryCreationBlock_backend_call'); - isFeedbackUsefulResult = await IsFeeedbackUsefulCall.call( - memory: FFAppState().lastMemory, - feedback: getJsonField( - (fMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - ); - logFirebaseEvent('memoryCreationBlock_update_app_state'); - FFAppState().update(() { - FFAppState().feedback = functions.jsonEncodeString(getJsonField( - (fMemory?.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString())!; - FFAppState().isFeedbackUseful = functions.jsonEncodeString(getJsonField( - (isFeedbackUsefulResult?.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString())!; - FFAppState().chatHistory = functions.saveChatHistory( - FFAppState().chatHistory, - functions.convertToJSONRole( - getJsonField( - (fMemory?.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - 'assistant')!)!; - FFAppState().memoryCreationProcessing = false; - }); - } else { - logFirebaseEvent('memoryCreationBlock_update_app_state'); - FFAppState().update(() { - FFAppState().feedback = ''; - FFAppState().isFeedbackUseful = 'Hide'; - }); - } - - logFirebaseEvent('memoryCreationBlock_backend_call'); - vectorFromOpenai = await VectorizeCall.call( - input: StructuredMemoryCall.responsegpt( - (sMemory.jsonBody ?? ''), - ), - ); - logFirebaseEvent('memoryCreationBlock_backend_call'); - - var memoriesRecordReference2 = MemoriesRecord.collection.doc(); - await memoriesRecordReference2.set({ - ...createMemoriesRecordData( - memory: FFAppState().lastMemory, - user: currentUserReference, - structuredMemory: getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - feedback: FFAppState().feedback, - toShowToUserShowHide: FFAppState().isFeedbackUseful, - emptyMemory: - FFAppState().lastMemory == '', - isUselessMemory: functions.memoryContainsNA(getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString()), - ), - ...mapToFirestore( - { - 'date': FieldValue.serverTimestamp(), - 'vector': VectorizeCall.embedding( - (vectorFromOpenai.jsonBody ?? ''), - ), - }, - ), - }); - memoryDone = MemoriesRecord.getDocumentFromData({ - ...createMemoriesRecordData( - memory: FFAppState().lastMemory, - user: currentUserReference, - structuredMemory: getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - feedback: FFAppState().feedback, - toShowToUserShowHide: FFAppState().isFeedbackUseful, - emptyMemory: - FFAppState().lastMemory == '', - isUselessMemory: functions.memoryContainsNA(getJsonField( - (sMemory.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString()), - ), - ...mapToFirestore( - { - 'date': DateTime.now(), - 'vector': VectorizeCall.embedding( - (vectorFromOpenai.jsonBody ?? ''), - ), - }, - ), - }, memoriesRecordReference2); - logFirebaseEvent('memoryCreationBlock_backend_call'); - vectorAdded = await CreateVectorPineconeCall.call( - vectorList: VectorizeCall.embedding( - (vectorFromOpenai.jsonBody ?? ''), - ), - id: memoryDone.reference.id, - structuredMemory: memoryDone.structuredMemory, - ); - if ((memoryDone.toShowToUserShowHide == 'Show') && - !memoryDone.emptyMemory && - !memoryDone.isUselessMemory) { - logFirebaseEvent('memoryCreationBlock_trigger_push_notific'); - triggerPushNotification( - notificationTitle: 'Sama', - notificationText: FFAppState().feedback, - userRefs: [currentUserReference!], - initialPageName: 'chat', - parameterData: {}, - ); - } + updateAppStateForEmptyFeedback(); } + logFirebaseEvent('memoryCreationBlock_backend_call', parameters: buildLogParameters(response)); + await finalizeMemoryRecord(response); +} - // clear unprocessed memories - logFirebaseEvent('memoryCreationBlock_clearunprocessedmemo'); +// Update app state when starting memory processing +void updateAppStateForProcessing() { + logFirebaseEvent('memoryCreationBlock_update_app_state'); FFAppState().update(() { - FFAppState().lastMemory = ''; + FFAppState().memoryCreationProcessing = true; }); } -Future voiceCommandBlock(BuildContext context) async { - List? latestMemories; - ApiCallResponse? voiceCommandResult; +// Request feedback for the given memory +Future requestFeedback(ApiCallResponse structuredMemoryResponse) async { + logFirebaseEvent('memoryCreationBlock_FEEDBACKapi'); + return await ChatGPTFeedbackCall.call( + memory: functions.jsonEncodeString(FFAppState().lastMemory), + structuredMemory: functions.jsonEncodeString(extractContent(structuredMemoryResponse)), + ); +} - if (!FFAppState().commandIsProcessing) { - logFirebaseEvent('voiceCommandBlock_update_app_state'); - FFAppState().commandIsProcessing = true; - FFAppState().commandState = 'Listening...'; - logFirebaseEvent('voiceCommandBlock_firestore_query'); - latestMemories = await queryMemoriesRecordOnce( - queryBuilder: (memoriesRecord) => memoriesRecord - .where( - 'user', - isEqualTo: currentUserReference, - ) - .where( - 'isUselessMemory', - isEqualTo: false, - ) - .where( - 'emptyMemory', - isEqualTo: false, - ) - .orderBy('date', descending: true), - limit: 30, - ); - logFirebaseEvent('voiceCommandBlock_wait__delay'); - await Future.delayed(const Duration(milliseconds: 6000)); - logFirebaseEvent('voiceCommandBlock_update_app_state'); - FFAppState().update(() { - FFAppState().commandState = 'Thinking...'; - }); - logFirebaseEvent('voiceCommandBlock_backend_call'); - voiceCommandResult = await VoiceCommandRespondCall.call( - memory: functions.limitTranscript(FFAppState().stt, 12000), - longTermMemory: functions.jsonEncodeString( - functions.documentsToText(latestMemories.toList())), +// Evaluate feedback usefulness +Future evaluateFeedback(ApiCallResponse feedbackResponse) async { + logFirebaseEvent('memoryCreationBlock_custom_action'); + await actions.debugLog(extractContent(feedbackResponse)); + logFirebaseEvent('memoryCreationBlock_backend_call'); + await IsFeeedbackUsefulCall.call( + memory: FFAppState().lastMemory, + feedback: extractContent(feedbackResponse), + ); +} + +// Update app state after processing feedback +void updateFinalAppState(ApiCallResponse feedbackResponse) { + logFirebaseEvent('memoryCreationBlock_update_app_state'); + FFAppState().update(() { + FFAppState().feedback = functions.jsonEncodeString(extractContent(feedbackResponse))!; + FFAppState().isFeedbackUseful = functions.jsonEncodeString(extractContent(feedbackResponse))!; + FFAppState().chatHistory = functions.saveChatHistory( + FFAppState().chatHistory, functions.convertToJSONRole(extractContent(feedbackResponse), 'assistant')); + FFAppState().memoryCreationProcessing = false; + }); +} + +// Update app state when feedback is empty +void updateAppStateForEmptyFeedback() { + logFirebaseEvent('memoryCreationBlock_update_app_state'); + FFAppState().update(() { + FFAppState().feedback = ''; + FFAppState().isFeedbackUseful = 'Hide'; + }); +} + +// Build log parameters for structured memory processing +Map buildLogParameters(ApiCallResponse response) { + return { + 'jsonBody': (response.jsonBody ?? ''), + 'memory': FFAppState().lastMemory, + 'user': currentUserReference, + }; +} + +// Finalize memory record after processing feedback +Future finalizeMemoryRecord(ApiCallResponse structuredMemoryResponse) async { + var vectorResponse = await vectorizeMemory(structuredMemoryResponse); + logFirebaseEvent('memoryCreationBlock_backend_call'); + var memoryRecord = await createMemoryRecord(structuredMemoryResponse, vectorResponse); + logFirebaseEvent('memoryCreationBlock_backend_call'); + await storeVectorData(memoryRecord, vectorResponse); +} + +// Call vectorization API for structured memory +Future vectorizeMemory(ApiCallResponse structuredMemoryResponse) async { + // debugPrint('vectorizeMemory ${structuredMemoryResponse.jsonBody}'); + var input = StructuredMemoryCall.responsegpt( + (structuredMemoryResponse.jsonBody ?? ''), + ); + return await VectorizeCall.call(input: input?.toString()); +} + +// Create memory record +Future createMemoryRecord( + ApiCallResponse structuredMemoryResponse, ApiCallResponse vectorResponse) async { + var recordRef = MemoriesRecord.collection.doc(); + await recordRef.set({ + ...createMemoriesRecordData( + memory: FFAppState().lastMemory, + user: currentUserReference, + structuredMemory: extractContent(structuredMemoryResponse), + feedback: FFAppState().feedback, + toShowToUserShowHide: FFAppState().isFeedbackUseful, + emptyMemory: FFAppState().lastMemory == '', + isUselessMemory: functions.memoryContainsNA(extractContent(structuredMemoryResponse)), + ), + ...mapToFirestore({ + 'date': FieldValue.serverTimestamp(), + 'vector': VectorizeCall.embedding((vectorResponse.jsonBody ?? '')), + }), + }); + return MemoriesRecord.getDocumentFromData({ + ...createMemoriesRecordData( + memory: FFAppState().lastMemory, + user: currentUserReference, + structuredMemory: extractContent(structuredMemoryResponse), + feedback: FFAppState().feedback, + toShowToUserShowHide: FFAppState().isFeedbackUseful, + emptyMemory: FFAppState().lastMemory == '', + isUselessMemory: functions.memoryContainsNA(extractContent(structuredMemoryResponse)), + ), + ...mapToFirestore({ + 'date': DateTime.now(), + 'vector': VectorizeCall.embedding((vectorResponse.jsonBody ?? '')), + }), + }, recordRef); +} + +// Store vector data after memory record creation +Future storeVectorData(MemoriesRecord memoryRecord, ApiCallResponse vectorResponse) async { + // debugPrint('storeVectorData: memoryRecord -> $memoryRecord'); + // debugPrint('storeVectorData: vectorResponse -> ${vectorResponse.jsonBody}'); + + var vectorAdded = await CreateVectorPineconeCall.call( + vectorList: VectorizeCall.embedding((vectorResponse.jsonBody ?? '')), + id: memoryRecord.reference.id, + structuredMemory: memoryRecord.structuredMemory, + ); + // debugPrint('storeVectorData VectorAdded: ${vectorAdded.statusCode} ${vectorAdded.jsonBody}'); + if (memoryRecord.toShowToUserShowHide == 'Show' && !memoryRecord.emptyMemory && !memoryRecord.isUselessMemory) { + logFirebaseEvent('memoryCreationBlock_trigger_push_notific'); + triggerPushNotification( + notificationTitle: 'Sama', + notificationText: FFAppState().feedback, + userRefs: [currentUserReference!], + initialPageName: 'chat', + parameterData: {}, ); - logFirebaseEvent('voiceCommandBlock_update_app_state'); - FFAppState().update(() { - FFAppState().commandState = ' Query'; - }); - if ((voiceCommandResult.succeeded ?? true)) { - logFirebaseEvent('voiceCommandBlock_trigger_push_notificat'); - triggerPushNotification( - notificationTitle: 'Sama', - notificationText: getJsonField( - (voiceCommandResult.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - notificationSound: 'default', - userRefs: [currentUserReference!], - initialPageName: 'chat', - parameterData: {}, - ); - logFirebaseEvent('voiceCommandBlock_backend_call'); - - await MemoriesRecord.collection.doc().set({ - ...createMemoriesRecordData( - user: currentUserReference, - feedback: getJsonField( - (voiceCommandResult.jsonBody ?? ''), - r'''$.choices[0].message.content''', - ).toString().toString(), - toShowToUserShowHide: 'Show', - emptyMemory: true, - isUselessMemory: false, - ), - ...mapToFirestore( - { - 'date': FieldValue.serverTimestamp(), - }, - ), - }); - } - logFirebaseEvent('voiceCommandBlock_update_app_state'); + } +} + +// Process voice commands +Future voiceCommandBlock(BuildContext context) async { + if (!FFAppState().commandIsProcessing) { + startProcessingCommand(); + var latestMemories = await fetchLatestMemories(); + await processVoiceCommand(latestMemories); FFAppState().commandIsProcessing = false; } } -Future periodicAction(BuildContext context) async { - String? lastWords; - - logFirebaseEvent('periodicAction_custom_action'); - lastWords = await actions.getLastWords(); - if (lastWords == '') { - if (FFAppState().lastMemory == '') { - // convo just started - logFirebaseEvent('periodicAction_convojuststarted'); - await actions.debugLog( - 'convo just started.', - ); - } else { - // add lastwords to memory - logFirebaseEvent('periodicAction_addlastwordstomemory'); - FFAppState().lastMemory = '${FFAppState().lastMemory} $lastWords'; - logFirebaseEvent('periodicAction_action_block'); - await action_blocks.memoryCreationBlock(context); - return; - } - } else { - logFirebaseEvent('periodicAction_update_app_state'); - FFAppState().lastMemory = '${FFAppState().lastMemory} $lastWords'; +// Start processing a voice command +void startProcessingCommand() { + logFirebaseEvent('voiceCommandBlock_update_app_state'); + FFAppState().update(() { + FFAppState().commandIsProcessing = true; + FFAppState().commandState = 'Listening...'; + }); +} + +// Fetch latest valid memories +Future> fetchLatestMemories() async { + logFirebaseEvent('voiceCommandBlock_firestore_query'); + return await queryMemoriesRecordOnce( + queryBuilder: (memoriesRecord) => memoriesRecord + .where('user', isEqualTo: currentUserReference) + .where('isUselessMemory', isEqualTo: false) + .where('emptyMemory', isEqualTo: false) + .orderBy('date', descending: true), + limit: 30, + ); +} + +// Process the voice command with potential delays and updates +Future processVoiceCommand(List memories) async { + await Future.delayed(const Duration(milliseconds: 6000)); + logFirebaseEvent('voiceCommandBlock_update_app_state'); + FFAppState().update(() { + FFAppState().commandState = 'Thinking...'; + }); + + var result = await VoiceCommandRespondCall.call( + memory: functions.limitTranscript(FFAppState().stt, 12000), + longTermMemory: functions.jsonEncodeString(functions.documentsToText(memories.toList())), + ); + + updateAppStateForVoiceResult(result); + if (result.succeeded ?? true) { + triggerVoiceCommandNotification(result); + await saveVoiceCommandMemory(result); } +} - logFirebaseEvent('periodicAction_backend_call'); +// Update app state after voice command results are processed +void updateAppStateForVoiceResult(ApiCallResponse result) { + FFAppState().commandState = 'Query'; +} +// Trigger notifications based on voice command results +void triggerVoiceCommandNotification(ApiCallResponse result) { + logFirebaseEvent('voiceCommandBlock_trigger_push_notification'); + triggerPushNotification( + notificationTitle: 'Sama', + notificationText: extractContent(result), + notificationSound: 'default', + userRefs: [currentUserReference!], + initialPageName: 'chat', + parameterData: {}, + ); +} + +// Save the memory from a voice command +Future saveVoiceCommandMemory(ApiCallResponse result) async { await MemoriesRecord.collection.doc().set({ ...createMemoriesRecordData( - memory: '', user: currentUserReference, - structuredMemory: 'N/A!', - feedback: '', - toShowToUserShowHide: 'Hide', + feedback: extractContent(result), + toShowToUserShowHide: 'Show', emptyMemory: true, - isUselessMemory: true, - ), - ...mapToFirestore( - { - 'date': FieldValue.serverTimestamp(), - }, + isUselessMemory: false, ), + ...mapToFirestore({ + 'date': FieldValue.serverTimestamp(), + }), }); } -Future onFinishAction(BuildContext context) async { - String? lastWordsOnFinishAction; +// Perform actions periodically +Future periodicAction(BuildContext context) async { + String lastWords = await actions.getLastWords(); + updateLastMemory(lastWords); + if (lastWords.isNotEmpty) { + logFirebaseEvent('periodicAction_addlastwordstomemory'); + await action_blocks.memoryCreationBlock(context); + } +} + +// Update last memory based on the last words +void updateLastMemory(String lastWords) { + logFirebaseEvent('periodicAction_update_app_state'); + FFAppState().lastMemory = '${FFAppState().lastMemory} $lastWords'; +} +// Perform final actions on finish +Future onFinishAction(BuildContext context) async { + String lastWordsOnFinishAction = await actions.getLastWords(); logFirebaseEvent('OnFinishAction_custom_action'); - lastWordsOnFinishAction = await actions.getLastWords(); - logFirebaseEvent('OnFinishAction_custom_action'); - await actions.debugLog( - 'LAST WORDS FINISH: $lastWordsOnFinishAction', - ); - logFirebaseEvent('OnFinishAction_update_app_state'); - FFAppState().lastMemory = - '${FFAppState().lastMemory} $lastWordsOnFinishAction'; - logFirebaseEvent('OnFinishAction_custom_action'); - await actions.debugLog( - 'LAST MEM FINISH: ${FFAppState().lastMemory}__LAST TRNASCRIPT: ${FFAppState().lastTranscript}', - ); + await actions.debugLog('LAST WORDS FINISH: $lastWordsOnFinishAction'); + + updateMemoryOnFinish(lastWordsOnFinishAction); logFirebaseEvent('OnFinishAction_action_block'); await action_blocks.memoryCreationBlock(context); + + triggerFinishNotification(); +} + +// Update memory when finishing an action +void updateMemoryOnFinish(String lastWords) { + FFAppState().lastMemory = '${FFAppState().lastMemory} $lastWords'; +} + +// Trigger notification to indicate recording is disabled +void triggerFinishNotification() { logFirebaseEvent('OnFinishAction_trigger_push_notification'); triggerPushNotification( notificationTitle: 'Sama', @@ -350,4 +334,7 @@ Future onFinishAction(BuildContext context) async { ); } -Future startRecording(BuildContext context) async {} +// Start recording function placeholder +Future startRecording(BuildContext context) async { + // Implement recording start logic here +} From ae8111f4db890358acf74c7745980c001ed171bc Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 09:24:29 -0500 Subject: [PATCH 4/8] triggerPushNotification conditional if currentUserReference != null --- apps/AppStandalone/lib/actions/actions.dart | 50 ++++++++++++--------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/apps/AppStandalone/lib/actions/actions.dart b/apps/AppStandalone/lib/actions/actions.dart index 9358781f..e2f92573 100644 --- a/apps/AppStandalone/lib/actions/actions.dart +++ b/apps/AppStandalone/lib/actions/actions.dart @@ -192,13 +192,15 @@ Future storeVectorData(MemoriesRecord memoryRecord, ApiCallResponse vector // debugPrint('storeVectorData VectorAdded: ${vectorAdded.statusCode} ${vectorAdded.jsonBody}'); if (memoryRecord.toShowToUserShowHide == 'Show' && !memoryRecord.emptyMemory && !memoryRecord.isUselessMemory) { logFirebaseEvent('memoryCreationBlock_trigger_push_notific'); - triggerPushNotification( - notificationTitle: 'Sama', - notificationText: FFAppState().feedback, - userRefs: [currentUserReference!], - initialPageName: 'chat', - parameterData: {}, - ); + if (currentUserReference != null) { + triggerPushNotification( + notificationTitle: 'Sama', + notificationText: FFAppState().feedback, + userRefs: [currentUserReference!], + initialPageName: 'chat', + parameterData: {}, + ); + } } } @@ -262,14 +264,16 @@ void updateAppStateForVoiceResult(ApiCallResponse result) { // Trigger notifications based on voice command results void triggerVoiceCommandNotification(ApiCallResponse result) { logFirebaseEvent('voiceCommandBlock_trigger_push_notification'); - triggerPushNotification( - notificationTitle: 'Sama', - notificationText: extractContent(result), - notificationSound: 'default', - userRefs: [currentUserReference!], - initialPageName: 'chat', - parameterData: {}, - ); + if (currentUserReference != null) { + triggerPushNotification( + notificationTitle: 'Sama', + notificationText: extractContent(result), + notificationSound: 'default', + userRefs: [currentUserReference!], + initialPageName: 'chat', + parameterData: {}, + ); + } } // Save the memory from a voice command @@ -325,13 +329,15 @@ void updateMemoryOnFinish(String lastWords) { // Trigger notification to indicate recording is disabled void triggerFinishNotification() { logFirebaseEvent('OnFinishAction_trigger_push_notification'); - triggerPushNotification( - notificationTitle: 'Sama', - notificationText: 'Recording is disabled! Please restart audio recording', - userRefs: [currentUserReference!], - initialPageName: 'chat', - parameterData: {}, - ); + if (currentUserReference != null) { + triggerPushNotification( + notificationTitle: 'Sama', + notificationText: 'Recording is disabled! Please restart audio recording', + userRefs: [currentUserReference!], + initialPageName: 'chat', + parameterData: {}, + ); + } } // Start recording function placeholder From d284f91821ef9acc4dee1edf765afe17e7237b43 Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 09:29:54 -0500 Subject: [PATCH 5/8] pinecone QueryVectors request fix using env vars --- apps/AppStandalone/lib/backend/api_requests/api_calls.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/AppStandalone/lib/backend/api_requests/api_calls.dart b/apps/AppStandalone/lib/backend/api_requests/api_calls.dart index 4b786ab2..621dcf1b 100644 --- a/apps/AppStandalone/lib/backend/api_requests/api_calls.dart +++ b/apps/AppStandalone/lib/backend/api_requests/api_calls.dart @@ -575,7 +575,7 @@ class QueryVectorsCall { final ffApiRequestBody = ''' { - "namespace": "ns1", + "namespace": "${Env.pineconeIndexNamespace}", "vector": $vector, "topK": 10, "includeValues": true, @@ -587,7 +587,7 @@ class QueryVectorsCall { }'''; return ApiManager.instance.makeApiCall( callName: 'QueryVectors', - apiUrl: 'https://index-i7j24t4.svc.gcp-starter.pinecone.io/query', + apiUrl: '${Env.pineconeIndexUrl}/query', callType: ApiCallType.POST, headers: { 'Api-Key': Env.pineconeApiKey, From 2a21dfe9032518fa99372a260ae347657aafa408 Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 09:44:47 -0500 Subject: [PATCH 6/8] stream_api_response using Open AI Organization --- .../actions/monitor_listenner.dart | 21 +++++++++++-------- .../actions/stream_api_response.dart | 1 + 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/AppStandalone/lib/custom_code/actions/monitor_listenner.dart b/apps/AppStandalone/lib/custom_code/actions/monitor_listenner.dart index 62cd482d..602e8d88 100644 --- a/apps/AppStandalone/lib/custom_code/actions/monitor_listenner.dart +++ b/apps/AppStandalone/lib/custom_code/actions/monitor_listenner.dart @@ -24,15 +24,18 @@ Future monitorListenner() async { FFAppState().latestUse = now; } if (now.difference(FFAppState().latestUse!).inMinutes > 5) { - triggerPushNotification( - notificationTitle: 'Sama', - notificationText: - 'Recording is disabled. Please restart audio recording', - notificationSound: 'default', - userRefs: [currentUserReference!], - initialPageName: 'homePage', - parameterData: {}, - ); + if (currentUserReference != null) { + triggerPushNotification( + notificationTitle: 'Sama', + notificationText: + 'Recording is disabled. Please restart audio recording', + notificationSound: 'default', + userRefs: [currentUserReference!], + initialPageName: 'homePage', + parameterData: {}, + ); + } + } } }); diff --git a/apps/AppStandalone/lib/custom_code/actions/stream_api_response.dart b/apps/AppStandalone/lib/custom_code/actions/stream_api_response.dart index cebf5a34..41175776 100644 --- a/apps/AppStandalone/lib/custom_code/actions/stream_api_response.dart +++ b/apps/AppStandalone/lib/custom_code/actions/stream_api_response.dart @@ -40,6 +40,7 @@ Future streamApiResponse( final headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer ${Env.openAIApiKey}', + 'OpenAI-Organization': Env.openAIOrganization, }; // Create Request From c461eda7b1758a700d2a2b9666510c41767f11a0 Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 10:04:22 -0500 Subject: [PATCH 7/8] RunnerDebug.entitlements added to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1ba03fd2..2e3b4c65 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ apps/AppWithWearable/ios/Runner/GeneratedPluginRegistrant.h apps/AppWithWearable/ios/Runner/GeneratedPluginRegistrant.m apps/AppWithWearable/macos/Flutter/ephemeral/flutter_export_environment.sh apps/AppWithWearable/macos/Flutter/ephemeral/Flutter-Generated.xcconfig +apps/AppStandalone/ios/Runner/RunnerDebug.entitlements From 2265449e46786795b410475f0005bdd4a848a675 Mon Sep 17 00:00:00 2001 From: Joan Cabezas Date: Fri, 26 Apr 2024 10:14:58 -0500 Subject: [PATCH 8/8] Updated README with pinecone index creation steps --- apps/AppStandalone/README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/AppStandalone/README.md b/apps/AppStandalone/README.md index da0dd7b2..b31fef2a 100644 --- a/apps/AppStandalone/README.md +++ b/apps/AppStandalone/README.md @@ -57,16 +57,25 @@ Before starting, make sure you have the following installed: ``` Follow the prompts to select your project and target platforms (Android, iOS, Web). -5. **API Keys**: + +5. **Pinecone Index**: + - If you don't have one yet, create a new [Pinecone](https://www.pinecone.io/) account. + - Navigate to the Pinecone Console and initiate a new index. + - Define the index name. + - Set the number of dimensions to `1,536`, as required for the `text-embedding-3-small` embeddings model. + - Keep the remaining settings as default and proceed with creation. + - Retrieve your index URL and namespace values, and update them in the environment variables file. + +6. **API Keys**: Add your Firebase and other necessary API keys to the `.env` file. -6. **Run Build Runner**: +7. **Run Build Runner**: Generate necessary files with Build Runner: ``` dart run build_runner build ``` -7. **Configure Xcode**: +8. **Configure Xcode**: - Open your iOS project in Xcode: ``` open ios/Runner.xcworkspace @@ -106,7 +115,7 @@ Before starting, make sure you have the following installed: ), ``` -8. **Run the App**: +9. **Run the App**: - Select your target device in Xcode or Android Studio. - Run the app.