-
Notifications
You must be signed in to change notification settings - Fork 220
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
vector database rag context logic + new prompts (#134)
- Loading branch information
Showing
16 changed files
with
409 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
https://github.com/FastCodeAI/DVDB |
107 changes: 107 additions & 0 deletions
107
apps/AppWithWearable/lib/backend/storage/dvdb/collection.dart
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
import 'dart:convert'; | ||
import 'dart:core'; | ||
import 'dart:io'; | ||
import 'dart:typed_data'; | ||
|
||
import 'package:friend_private/backend/storage/dvdb/document.dart'; | ||
import 'package:friend_private/backend/storage/dvdb/math.dart'; | ||
import 'package:friend_private/backend/storage/dvdb/search_result.dart'; | ||
import 'package:path/path.dart'; | ||
import 'package:path_provider/path_provider.dart'; | ||
import 'package:uuid/uuid.dart'; | ||
|
||
class Collection { | ||
Collection(this.name); | ||
|
||
final String name; | ||
final Map<String, Document> documents = {}; | ||
|
||
void addDocument(String? id, String text, Float64List embedding, {Map<String, String>? metadata}) { | ||
var uuid = Uuid(); | ||
final Document document = Document( | ||
id: id ?? uuid.v1(), | ||
text: text, | ||
embedding: embedding, | ||
metadata: metadata, | ||
); | ||
|
||
documents[document.id] = document; | ||
_writeDocument(document); | ||
} | ||
|
||
void addDocuments(List<Document> docs) { | ||
for (final Document doc in docs) { | ||
documents[doc.id] = doc; | ||
_writeDocument(doc); | ||
} | ||
} | ||
|
||
void removeDocument(String id) { | ||
if (documents.containsKey(id)) { | ||
documents.remove(id); | ||
_saveAllDocuments(); // Re-saving all documents after removal | ||
} | ||
} | ||
|
||
List<SearchResult> search(Float64List query, {int numResults = 10, double? threshold}) { | ||
final List<SearchResult> similarities = <SearchResult>[]; | ||
for (var document in documents.values) { | ||
final double similarity = MathFunctions().cosineSimilarity(query, document.embedding); | ||
|
||
if (threshold != null && similarity < threshold) { | ||
continue; | ||
} | ||
|
||
similarities.add(SearchResult(id: document.id, text: document.text, score: similarity)); | ||
} | ||
|
||
similarities.sort((SearchResult a, SearchResult b) => b.score.compareTo(a.score)); | ||
return similarities.take(numResults).toList(); | ||
} | ||
|
||
Future<void> _writeDocument(Document document) async { | ||
Directory documentsDirectory = await getApplicationDocumentsDirectory(); | ||
String path = join(documentsDirectory.path, '$name.json'); | ||
final File file = File(path); | ||
|
||
var encodedDocument = json.encode(document.toJson()); | ||
List<int> bytes = utf8.encode('$encodedDocument\n'); | ||
|
||
file.writeAsBytesSync(bytes, mode: FileMode.append); | ||
} | ||
|
||
Future<void> _saveAllDocuments() async { | ||
Directory documentsDirectory = await getApplicationDocumentsDirectory(); | ||
String path = join(documentsDirectory.path, '$name.json'); | ||
final File file = File(path); | ||
|
||
file.writeAsStringSync(''); // Clearing the file | ||
for (var document in documents.values) { | ||
_writeDocument(document); | ||
} | ||
} | ||
|
||
Future<void> load() async { | ||
Directory documentsDirectory = await getApplicationDocumentsDirectory(); | ||
String path = join(documentsDirectory.path, '$name.json'); | ||
final File file = File(path); | ||
|
||
if (!file.existsSync()) { | ||
documents.clear(); | ||
return; | ||
} | ||
|
||
final lines = file.readAsLinesSync(); | ||
|
||
for (var line in lines) { | ||
var decodedDocument = json.decode(line) as Map<String, dynamic>; | ||
var document = Document.fromJson(decodedDocument); | ||
documents[document.id] = document; | ||
} | ||
} | ||
|
||
void clear() { | ||
documents.clear(); | ||
_saveAllDocuments(); | ||
} | ||
} |
43 changes: 43 additions & 0 deletions
43
apps/AppWithWearable/lib/backend/storage/dvdb/document.dart
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
import 'dart:math'; | ||
import 'dart:typed_data'; | ||
import 'package:uuid/uuid.dart'; | ||
|
||
class Document { | ||
Document({String? id, required this.text, required this.embedding, Map<String, String>? metadata}) | ||
: id = id ?? _generateUuid(), | ||
magnitude = _calculateMagnitude(embedding), | ||
metadata = metadata ?? Map<String, String>(); | ||
|
||
final String id; | ||
final String text; | ||
final Float64List embedding; | ||
final double magnitude; | ||
final Map<String, String> metadata; | ||
|
||
static String _generateUuid() { | ||
return Uuid().v1(); | ||
} | ||
|
||
static double _calculateMagnitude(Float64List embedding) { | ||
return sqrt(embedding.fold(0, (num sum, double element) => sum + element * element)); | ||
} | ||
|
||
Map<String, dynamic> toJson() { | ||
return { | ||
'id': id, | ||
'text': text, | ||
'embedding': embedding, | ||
'magnitude': magnitude, | ||
'metadata': metadata, | ||
}; | ||
} | ||
|
||
factory Document.fromJson(Map<String, dynamic> json) { | ||
return Document( | ||
id: json['id'], | ||
text: json['text'], | ||
embedding: Float64List.fromList(json['embedding'].cast<double>()), | ||
metadata: Map<String, String>.from(json['metadata']) | ||
); | ||
} | ||
} |
39 changes: 39 additions & 0 deletions
39
apps/AppWithWearable/lib/backend/storage/dvdb/dvdb_helper.dart
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
import 'package:friend_private/backend/storage/dvdb/collection.dart'; | ||
|
||
class DVDB { | ||
DVDB._internal(); | ||
|
||
static final DVDB _shared = DVDB._internal(); | ||
|
||
factory DVDB() { | ||
return _shared; | ||
} | ||
|
||
final Map<String, Collection> _collections = {}; | ||
|
||
Collection collection(String name) { | ||
if (_collections.containsKey(name)) { | ||
return _collections[name]!; | ||
} | ||
|
||
final Collection collection = Collection(name); | ||
_collections[name] = collection; | ||
collection.load(); | ||
return collection; | ||
} | ||
|
||
Collection? getCollection(String name) { | ||
return _collections[name]; | ||
} | ||
|
||
void releaseCollection(String name) { | ||
_collections.remove(name); | ||
} | ||
|
||
void reset() { | ||
for (final Collection collection in _collections.values) { | ||
collection.clear(); | ||
} | ||
_collections.clear(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
enum VectorDBError { | ||
collectionAlreadyExists, | ||
} | ||
|
||
class CollectionError implements Exception { | ||
CollectionError._(this.message); | ||
|
||
final String message; | ||
|
||
factory CollectionError.fileNotFound() { | ||
return CollectionError._("File not found."); | ||
} | ||
|
||
factory CollectionError.loadFailed(String errorMessage) { | ||
return CollectionError._("Load failed: $errorMessage"); | ||
} | ||
|
||
@override | ||
String toString() { | ||
return message; | ||
} | ||
} |
Oops, something went wrong.