diff --git a/apps/AppWithWearable/Tasks.md b/apps/AppWithWearable/Tasks.md index 69682d6d..24b31544 100644 --- a/apps/AppWithWearable/Tasks.md +++ b/apps/AppWithWearable/Tasks.md @@ -12,28 +12,25 @@ structuring but as context, not as part of the structure, so that if there's some reference to a person, and then you use a pronoun, the LLM understands what you are referring to. - [ ] Migrate MemoryRecord from SharedPreferences to sqlite -- [ ] Implement [similarity search](https://www.pinecone.io/learn/vector-similarity/) locally - - [ ] Use from the AppStandalone `_ragContext` function as a baseline for creating the query +- [X] Implement [similarity search](https://www.pinecone.io/learn/vector-similarity/) locally + - [X] Use from the AppStandalone `_ragContext` function as a baseline for creating the query embedding. - - [ ] When a memory is created, compute the vector embedding and store it locally. - - [ ] When the user sends a question in the chat, extract from the AppStandalone + - [X] When a memory is created, compute the vector embedding and store it locally. + - [X] When the user sends a question in the chat, extract from the AppStandalone the `function_calling` that determines if the message requires context, if that's the case, retrieve the top 10 most similar vectors ~~ For an initial version we can read all memories from sqlite or SharedPreferences, and compute the formula between the query and each vector. - - [ ] Use that as context, and ask to the LLM. Retrieve the prompt from the AppStandalone. - - [X] ----- -- [ ] Another option is to use one of the vector db libraries available for - dart https://github.com/FastCodeAI/DVDB or https://pub.dev/packages/chromadb + - [X] Use that as context, and ask to the LLM. Retrieve the prompt from the AppStandalone. + - [ ] Improve function call way of parsing the text sent to the RAG, GPT should format the input + better for RAG to retrieve better context. - [ ] Settings Deepgram + openAI key are forced to be set - [ ] In case an API key fails, either Deepgram WebSocket connection fails, or GPT requests, let - the - user know the error message, either has no more credits, api key is invalid, etc. + the user know the error message, either has no more credits, api key is invalid, etc. - [ ] Improve connected device page UI, including transcription text, and when memory creates after 30 seconds, let the user know - [ ] Structure the memory asking JSON output `{"title", "summary"}`, in that way we can have - better - parsed data. + better parsed data. - [x] Test/Implement [speaker diarization](https://developers.deepgram.com/docs/diarization) to recognize multiple speakers in transcription, use that for better context when creating the structured memory. @@ -44,6 +41,7 @@ conversation, also, remove Speaker $i in transcript. - [ ] Allow users who don't have a GCP bucket to store their recordings locally. - [ ] Improve recordings player. + --- - [x] Multilanguage option, implement settings selector, and use that for the deepgram websocket diff --git a/apps/AppWithWearable/lib/actions/actions.dart b/apps/AppWithWearable/lib/actions/actions.dart index 6a655d84..934b2a8b 100644 --- a/apps/AppWithWearable/lib/actions/actions.dart +++ b/apps/AppWithWearable/lib/actions/actions.dart @@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:friend_private/backend/storage/vector_db.dart'; import 'package:friend_private/backend/storage/memories.dart'; import 'package:uuid/uuid.dart'; import '/backend/api_requests/api_calls.dart'; @@ -43,8 +44,11 @@ void changeAppStateMemoryCreating() { // Finalize memory record after processing feedback Future finalizeMemoryRecord(String rawMemory, String structuredMemory, String? audioFilePath) async { - await createMemoryRecord(rawMemory, structuredMemory, audioFilePath); + MemoryRecord createdMemory = await createMemoryRecord(rawMemory, structuredMemory, audioFilePath); changeAppStateMemoryCreating(); + List vector = await getEmbeddingsFromInput(structuredMemory); + storeMemoryVector(createdMemory, vector); + // storeMemoryVector } // Create memory record diff --git a/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart b/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart index 6bf6cfe0..7c9b0841 100644 --- a/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart +++ b/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart @@ -164,3 +164,75 @@ String qaStreamedFullMemories(List memories, List chatHis }); return body; } + +// ------ + +Future determineRequiresContext(String lastMessage, List chatHistory) async { + var tools = [ + { + "type": "function", + "function": { + "name": "retrieve_rag_context", + "description": "Retrieve pieces of user memories as context.", + "parameters": { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": ''' + Based on the current conversation, determine if the message is a question and if there's + context that needs to be retrieved from the user recorded audio memories in order to answer that question. + If that's the case, return the question better parsed so that retrieved pieces of context are better. + ''', + }, + }, + }, + }, + } + ]; + String message = ''' + Conversation: + ${chatHistory.map((e) => '${e['role'].toString().toUpperCase()}: ${e['content']}').join('\n')}\n + USER:$lastMessage + ''' + .replaceAll(' ', ''); + debugPrint('determineRequiresContext message: $message'); + var response = await gptApiCall( + model: 'gpt-4-turbo', + messages: [ + {"role": "user", "content": message} + ], + tools: tools); + if (response.toString().contains('retrieve_rag_context')) { + var args = jsonDecode(response[0]['function']['arguments']); + return args['question']; + } + return null; +} + +String qaStreamedBody(String context, List chatHistory) { + var prompt = ''' + You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. + If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise. + If the message doesn't require context, it will be empty, so answer the question casually. + + Conversation History: + ${chatHistory.map((e) => '${e['role'].toString().toUpperCase()}: ${e['content']}').join('\n')} + + Context: + ``` + $context + ``` + Answer: + ''' + .replaceAll(' ', ''); + debugPrint(prompt); + var body = jsonEncode({ + "model": "gpt-4-turbo", + "messages": [ + {"role": "system", "content": prompt} + ], + "stream": true, + }); + return body; +} diff --git a/apps/AppWithWearable/lib/backend/api_requests/stream_api_response.dart b/apps/AppWithWearable/lib/backend/api_requests/stream_api_response.dart index fe749de0..24906b7b 100644 --- a/apps/AppWithWearable/lib/backend/api_requests/stream_api_response.dart +++ b/apps/AppWithWearable/lib/backend/api_requests/stream_api_response.dart @@ -27,8 +27,8 @@ Future streamApiResponse( 'Authorization': 'Bearer $apiKey', }; - // String body = qaStreamedBody(context, retrieveMostRecentMessages(FFAppState().chatHistory)); - String body = qaStreamedFullMemories(FFAppState().memories, retrieveMostRecentMessages(FFAppState().chatHistory)); + String body = qaStreamedBody(context, retrieveMostRecentMessages(FFAppState().chatHistory)); + // String body = qaStreamedFullMemories(FFAppState().memories, retrieveMostRecentMessages(FFAppState().chatHistory)); var request = http.Request("POST", Uri.parse(url)) ..headers.addAll(headers) ..body = body; diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/README.md b/apps/AppWithWearable/lib/backend/storage/dvdb/README.md new file mode 100644 index 00000000..da37ccb4 --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/README.md @@ -0,0 +1 @@ +https://github.com/FastCodeAI/DVDB \ No newline at end of file diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/collection.dart b/apps/AppWithWearable/lib/backend/storage/dvdb/collection.dart new file mode 100644 index 00000000..ee3ecadb --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/collection.dart @@ -0,0 +1,107 @@ +import 'dart:convert'; +import 'dart:core'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:friend_private/backend/storage/dvdb/document.dart'; +import 'package:friend_private/backend/storage/dvdb/math.dart'; +import 'package:friend_private/backend/storage/dvdb/search_result.dart'; +import 'package:path/path.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:uuid/uuid.dart'; + +class Collection { + Collection(this.name); + + final String name; + final Map documents = {}; + + void addDocument(String? id, String text, Float64List embedding, {Map? metadata}) { + var uuid = Uuid(); + final Document document = Document( + id: id ?? uuid.v1(), + text: text, + embedding: embedding, + metadata: metadata, + ); + + documents[document.id] = document; + _writeDocument(document); + } + + void addDocuments(List docs) { + for (final Document doc in docs) { + documents[doc.id] = doc; + _writeDocument(doc); + } + } + + void removeDocument(String id) { + if (documents.containsKey(id)) { + documents.remove(id); + _saveAllDocuments(); // Re-saving all documents after removal + } + } + + List search(Float64List query, {int numResults = 10, double? threshold}) { + final List similarities = []; + for (var document in documents.values) { + final double similarity = MathFunctions().cosineSimilarity(query, document.embedding); + + if (threshold != null && similarity < threshold) { + continue; + } + + similarities.add(SearchResult(id: document.id, text: document.text, score: similarity)); + } + + similarities.sort((SearchResult a, SearchResult b) => b.score.compareTo(a.score)); + return similarities.take(numResults).toList(); + } + + Future _writeDocument(Document document) async { + Directory documentsDirectory = await getApplicationDocumentsDirectory(); + String path = join(documentsDirectory.path, '$name.json'); + final File file = File(path); + + var encodedDocument = json.encode(document.toJson()); + List bytes = utf8.encode('$encodedDocument\n'); + + file.writeAsBytesSync(bytes, mode: FileMode.append); + } + + Future _saveAllDocuments() async { + Directory documentsDirectory = await getApplicationDocumentsDirectory(); + String path = join(documentsDirectory.path, '$name.json'); + final File file = File(path); + + file.writeAsStringSync(''); // Clearing the file + for (var document in documents.values) { + _writeDocument(document); + } + } + + Future load() async { + Directory documentsDirectory = await getApplicationDocumentsDirectory(); + String path = join(documentsDirectory.path, '$name.json'); + final File file = File(path); + + if (!file.existsSync()) { + documents.clear(); + return; + } + + final lines = file.readAsLinesSync(); + + for (var line in lines) { + var decodedDocument = json.decode(line) as Map; + var document = Document.fromJson(decodedDocument); + documents[document.id] = document; + } + } + + void clear() { + documents.clear(); + _saveAllDocuments(); + } +} diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/document.dart b/apps/AppWithWearable/lib/backend/storage/dvdb/document.dart new file mode 100644 index 00000000..4fefa801 --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/document.dart @@ -0,0 +1,43 @@ +import 'dart:math'; +import 'dart:typed_data'; +import 'package:uuid/uuid.dart'; + +class Document { + Document({String? id, required this.text, required this.embedding, Map? metadata}) + : id = id ?? _generateUuid(), + magnitude = _calculateMagnitude(embedding), + metadata = metadata ?? Map(); + + final String id; + final String text; + final Float64List embedding; + final double magnitude; + final Map metadata; + + static String _generateUuid() { + return Uuid().v1(); + } + + static double _calculateMagnitude(Float64List embedding) { + return sqrt(embedding.fold(0, (num sum, double element) => sum + element * element)); + } + + Map toJson() { + return { + 'id': id, + 'text': text, + 'embedding': embedding, + 'magnitude': magnitude, + 'metadata': metadata, + }; + } + + factory Document.fromJson(Map json) { + return Document( + id: json['id'], + text: json['text'], + embedding: Float64List.fromList(json['embedding'].cast()), + metadata: Map.from(json['metadata']) + ); + } +} \ No newline at end of file diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/dvdb_helper.dart b/apps/AppWithWearable/lib/backend/storage/dvdb/dvdb_helper.dart new file mode 100644 index 00000000..90039ac1 --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/dvdb_helper.dart @@ -0,0 +1,39 @@ +import 'package:friend_private/backend/storage/dvdb/collection.dart'; + +class DVDB { + DVDB._internal(); + + static final DVDB _shared = DVDB._internal(); + + factory DVDB() { + return _shared; + } + + final Map _collections = {}; + + Collection collection(String name) { + if (_collections.containsKey(name)) { + return _collections[name]!; + } + + final Collection collection = Collection(name); + _collections[name] = collection; + collection.load(); + return collection; + } + + Collection? getCollection(String name) { + return _collections[name]; + } + + void releaseCollection(String name) { + _collections.remove(name); + } + + void reset() { + for (final Collection collection in _collections.values) { + collection.clear(); + } + _collections.clear(); + } +} diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/errors.dart b/apps/AppWithWearable/lib/backend/storage/dvdb/errors.dart new file mode 100644 index 00000000..da2c4ca2 --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/errors.dart @@ -0,0 +1,22 @@ +enum VectorDBError { + collectionAlreadyExists, +} + +class CollectionError implements Exception { + CollectionError._(this.message); + + final String message; + + factory CollectionError.fileNotFound() { + return CollectionError._("File not found."); + } + + factory CollectionError.loadFailed(String errorMessage) { + return CollectionError._("Load failed: $errorMessage"); + } + + @override + String toString() { + return message; + } +} \ No newline at end of file diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/math.dart b/apps/AppWithWearable/lib/backend/storage/dvdb/math.dart new file mode 100644 index 00000000..163876f5 --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/math.dart @@ -0,0 +1,29 @@ +import 'package:ml_linalg/linalg.dart'; + +class MathFunctions { + MathFunctions._internal(); + + static final MathFunctions _shared = MathFunctions._internal(); + + factory MathFunctions() { + return _shared; + } + + double cosineSimilarity(List a, List b) { + assert(a.length == b.length); + + Vector aVector = Vector.fromList(a); + Vector bVector = Vector.fromList(b); + + double dotProduct = aVector.dot(bVector); + + double aNorm = aVector.norm(); + double bNorm = bVector.norm(); + + if (aNorm == 0 || bNorm == 0) { + return 0.0; + } else { + return dotProduct / (aNorm * bNorm); + } + } +} \ No newline at end of file diff --git a/apps/AppWithWearable/lib/backend/storage/dvdb/search_result.dart b/apps/AppWithWearable/lib/backend/storage/dvdb/search_result.dart new file mode 100644 index 00000000..09e50cab --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/dvdb/search_result.dart @@ -0,0 +1,26 @@ +import 'dart:core'; + +class SearchResult { + SearchResult({required this.id, required this.text, required this.score}); + + final String id; + final String text; + final double score; + + // Optional: If you need to serialize/deserialize to/from JSON + factory SearchResult.fromJson(Map json) { + return SearchResult( + id: json['id'], + text: json['text'], + score: json['score'].toDouble(), + ); + } + + Map toJson() { + return { + 'id': id, + 'text': text, + 'score': score, + }; + } +} diff --git a/apps/AppWithWearable/lib/backend/storage/memories.dart b/apps/AppWithWearable/lib/backend/storage/memories.dart index 1487b102..7f7be92a 100644 --- a/apps/AppWithWearable/lib/backend/storage/memories.dart +++ b/apps/AppWithWearable/lib/backend/storage/memories.dart @@ -76,6 +76,17 @@ class MemoryStorage { return memories.where((memory) => !memory.isUseless).toList(); } + static Future> getAllMemoriesByIds(List memoriesId) async { + List memories = await getAllMemories(); + List filtered = []; + for (MemoryRecord memory in memories) { + if (memoriesId.contains(memory.id)) { + filtered.add(memory); + } + } + return filtered; + } + static Future updateMemory(String memoryId, String updatedMemory) async { final SharedPreferences prefs = await SharedPreferences.getInstance(); List allMemories = prefs.getStringList(_storageKey) ?? []; diff --git a/apps/AppWithWearable/lib/backend/storage/vector_db.dart b/apps/AppWithWearable/lib/backend/storage/vector_db.dart new file mode 100644 index 00000000..2778be66 --- /dev/null +++ b/apps/AppWithWearable/lib/backend/storage/vector_db.dart @@ -0,0 +1,15 @@ +import 'dart:typed_data'; + +import 'package:friend_private/backend/storage/dvdb/dvdb_helper.dart'; +import 'package:friend_private/backend/storage/memories.dart'; + +var collection = DVDB().collection("memories"); + +Future storeMemoryVector(MemoryRecord memory, List embedding) async { + collection.addDocument(memory.id, memory.structuredMemory, Float64List.fromList(embedding)); +} + +List querySimilarVectors(List queryEmbedding) { + final query = collection.search(Float64List.fromList(queryEmbedding), numResults: 10); + return query.map((e) => e.id).toList(); +} diff --git a/apps/AppWithWearable/lib/pages/ble/device_data/widget.dart b/apps/AppWithWearable/lib/pages/ble/device_data/widget.dart index 63db64c5..aeebda13 100644 --- a/apps/AppWithWearable/lib/pages/ble/device_data/widget.dart +++ b/apps/AppWithWearable/lib/pages/ble/device_data/widget.dart @@ -70,7 +70,7 @@ class DeviceDataWidgetState extends State { } _initiateTimer() { - _timer = Timer(const Duration(seconds: 30), () async { + _timer = Timer(const Duration(seconds: 5), () async { debugPrint('Creating memory from whispers'); String transcript = _buildDiarizedTranscriptMessage(); debugPrint('Transcript: \n${transcript.trim()}'); diff --git a/apps/AppWithWearable/lib/pages/chat/page.dart b/apps/AppWithWearable/lib/pages/chat/page.dart index da7d0864..85077aaf 100644 --- a/apps/AppWithWearable/lib/pages/chat/page.dart +++ b/apps/AppWithWearable/lib/pages/chat/page.dart @@ -1,4 +1,7 @@ +import 'package:friend_private/backend/api_requests/api_calls.dart'; import 'package:friend_private/backend/api_requests/stream_api_response.dart'; +import 'package:friend_private/backend/storage/memories.dart'; +import 'package:friend_private/backend/storage/vector_db.dart'; import 'package:friend_private/flutter_flow/custom_functions.dart'; import 'package:friend_private/pages/ble/blur_bot/blur_bot_widget.dart'; @@ -585,11 +588,11 @@ class _ChatPageWidgetState extends State { ), showLoadingIndicator: true, onPressed: () async { - // String ragContext = - // await _retrieveRAGContext(_model.textController.text); + String ragContext = + await _retrieveRAGContext(_model.textController.text); await uiUpdatesChatQA(); await streamApiResponse( - '', + ragContext, _callbackFunctionChatStreaming(), ); @@ -630,23 +633,25 @@ class _ChatPageWidgetState extends State { ); } - // Future _retrieveRAGContext(String message) async { - // String? betterContextQuestion = - // await determineRequiresContext(message, retrieveMostRecentMessages(FFAppState().chatHistory)); - // debugPrint('_retrieveRAGContext: $betterContextQuestion'); - // if (betterContextQuestion == null) { - // return ''; - // } - // List vectorizedMessage = await getEmbeddingsFromInput( - // message, - // ); - // List foundVectors = await queryPineconeVectors(vectorizedMessage); - // String context = ''; - // for (var element in foundVectors) { - // context += element['structuredMemory'] + '\n'; - // } - // return context; - // } + Future _retrieveRAGContext(String message) async { + String? betterContextQuestion = + await determineRequiresContext(message, retrieveMostRecentMessages(FFAppState().chatHistory)); + debugPrint('_retrieveRAGContext betterContextQuestion: $betterContextQuestion'); + if (betterContextQuestion == null) { + return ''; + } + List vectorizedMessage = await getEmbeddingsFromInput( + message, + ); + List memoriesId = querySimilarVectors(vectorizedMessage); + debugPrint('querySimilarVectors memories retrieved: $memoriesId'); + if (memoriesId.isEmpty) { + return ''; + } + List memories = await MemoryStorage.getAllMemoriesByIds(memoriesId); + return MemoryRecord.memoriesToString(memories); + + } uiUpdatesChatQA() async { setState(() { diff --git a/apps/AppWithWearable/pubspec.yaml b/apps/AppWithWearable/pubspec.yaml index 0b961f37..b3909ea6 100644 --- a/apps/AppWithWearable/pubspec.yaml +++ b/apps/AppWithWearable/pubspec.yaml @@ -56,6 +56,7 @@ dependencies: tuple: ^2.0.2 googleapis_auth: ^1.6.0 audioplayers: ^5.2.1 + ml_linalg: ^13.12.2 dependency_overrides: http: 1.2.0