feat: support for migration between milli versions (#79)

Fixes #33, fixes #9

.github/workflows/build.yml

@@ -25,21 +25,22 @@ jobs:
           components: rustfmt, clippy
 
       # Rust
-      - name: Check Rust code format with rustfmt
-        working-directory: ./packages/mimir/native
-        run: rustfmt --check src/*
-      - name: Check Rust code with clippy
-        working-directory: ./packages/mimir/native
+      - name: Check Rust format
+        working-directory: ./packages/mimir/native/src
+        run: rustfmt --check lib.rs
+      - name: Rust code analysis
         run: cargo clippy -- -D warnings
-      - name: Build Rust code (embedded_milli)
+      - name: Run Rust tests
+        run: cargo test
+      - name: Build Rust code (embedded_milli) for Dart tests
         run: cargo build -r
 
       # Dart/Flutter
-      - name: Check format
+      - name: Check Dart format
         run: melos run check-format --no-select
-      - name: Analyze
+      - name: Dart code analysis
         run: melos run analyze --no-select
-      - name: Run tests
+      - name: Run Dart tests
         run: melos run test
 
   macos_integration_test:

README.md

@@ -215,24 +215,20 @@ as it almost reads as pure English, even for complex conditions.
 ## Important Caveats
 Please read these caveats _before_ adding mimir to your project.
 
-- Right now, mimir is in alpha status _and should not be used in production_!
-  - Major APIs are somewhat solidified; however, internal workings are _incomplete_
-  - [The biggest blocker from being production ready right now](https://github.com/GregoryConrad/mimir/issues/9)
-  - After I take care of this issue, and a few others, mimir _should_ be production ready
 - Document `id`s
-  - Documents without an `id` field will have one autogenerated (out of necessity)
-    - I would recommend adding an `id` field to your documents when possible though
+  - Documents in mimir need to have an `id` field!
+    - Can be a number or a string matching this regex: `^[a-zA-Z0-9-_]*$`
   - Current bug in milli: you need your `id` field at _the top_ of your documents
     - See https://github.com/meilisearch/product/discussions/206 for more
 - Resource usage
   - Note: most modern devices run mimir just fine; however:
   - Milli, a core component of Meilisearch, is what gives mimir a lot of its power
   - Unfortunately, Milli is somewhat resource intensive, making mimir itself somewhat heavy-weight
   - Several thousand detailed documents can easily consume several MB of disk space & RAM
-  - If you do not need all the features provided by mimir, also consider an alternative!
+  - If you do not need all the features provided by mimir, also consider a more lightweight alternative!
     - [Hive](https://pub.dev/packages/hive) for simple key-value storage
     - [Isar](https://pub.dev/packages/isar) for more sophisticated use-cases
-    - If you need easy, typo-tolerant full-text search, you will want mimir
+      - Note: while Isar does have full-text search, it is *not* typo-tolerant!
+    - If you need easy, typo-tolerant full-text search, you will want mimir!
       - I am unaware of any other databases that currently provide typo-tolerant full-text search in Flutter
-        - Why I made mimir in the first place!
-        - Note: Isar does have full-text search, but it is _not_ typo tolerant
+      - Why I made mimir in the first place!

packages/mimir/lib/src/bridge_generated.dart

@@ -19,13 +19,25 @@ abstract class EmbeddedMilli {
 
   FlutterRustBridgeTaskConstMeta get kEnforceBindingConstMeta;
 
+  /// Ensures an instance of milli (represented by just a directory) is initialized
+  Future<void> ensureInstanceInitialized(
+      {required String instaceDir, dynamic hint});
+
+  FlutterRustBridgeTaskConstMeta get kEnsureInstanceInitializedConstMeta;
+
+  /// Ensures a milli index is initialized
+  Future<void> ensureIndexInitialized(
+      {required String instanceDir, required String indexName, dynamic hint});
+
+  FlutterRustBridgeTaskConstMeta get kEnsureIndexInitializedConstMeta;
+
   /// Adds the given list of documents to the specified milli index
   ///
   /// Replaces documents that already exist in the index based on document ids.
   Future<void> addDocuments(
       {required String instanceDir,
       required String indexName,
-      required List<String> jsonDocuments,
+      required List<String> documents,
       dynamic hint});
 
   FlutterRustBridgeTaskConstMeta get kAddDocumentsConstMeta;
@@ -49,7 +61,7 @@ abstract class EmbeddedMilli {
   Future<void> setDocuments(
       {required String instanceDir,
       required String indexName,
-      required List<String> jsonDocuments,
+      required List<String> documents,
       dynamic hint});
 
   FlutterRustBridgeTaskConstMeta get kSetDocumentsConstMeta;
@@ -256,28 +268,67 @@ class EmbeddedMilliImpl implements EmbeddedMilli {
         argNames: [],
       );
 
+  Future<void> ensureInstanceInitialized(
+      {required String instaceDir, dynamic hint}) {
+    var arg0 = _platform.api2wire_String(instaceDir);
+    return _platform.executeNormal(FlutterRustBridgeTask(
+      callFfi: (port_) =>
+          _platform.inner.wire_ensure_instance_initialized(port_, arg0),
+      parseSuccessData: _wire2api_unit,
+      constMeta: kEnsureInstanceInitializedConstMeta,
+      argValues: [instaceDir],
+      hint: hint,
+    ));
+  }
+
+  FlutterRustBridgeTaskConstMeta get kEnsureInstanceInitializedConstMeta =>
+      const FlutterRustBridgeTaskConstMeta(
+        debugName: "ensure_instance_initialized",
+        argNames: ["instaceDir"],
+      );
+
+  Future<void> ensureIndexInitialized(
+      {required String instanceDir, required String indexName, dynamic hint}) {
+    var arg0 = _platform.api2wire_String(instanceDir);
+    var arg1 = _platform.api2wire_String(indexName);
+    return _platform.executeNormal(FlutterRustBridgeTask(
+      callFfi: (port_) =>
+          _platform.inner.wire_ensure_index_initialized(port_, arg0, arg1),
+      parseSuccessData: _wire2api_unit,
+      constMeta: kEnsureIndexInitializedConstMeta,
+      argValues: [instanceDir, indexName],
+      hint: hint,
+    ));
+  }
+
+  FlutterRustBridgeTaskConstMeta get kEnsureIndexInitializedConstMeta =>
+      const FlutterRustBridgeTaskConstMeta(
+        debugName: "ensure_index_initialized",
+        argNames: ["instanceDir", "indexName"],
+      );
+
   Future<void> addDocuments(
       {required String instanceDir,
       required String indexName,
-      required List<String> jsonDocuments,
+      required List<String> documents,
       dynamic hint}) {
     var arg0 = _platform.api2wire_String(instanceDir);
     var arg1 = _platform.api2wire_String(indexName);
-    var arg2 = _platform.api2wire_StringList(jsonDocuments);
+    var arg2 = _platform.api2wire_StringList(documents);
     return _platform.executeNormal(FlutterRustBridgeTask(
       callFfi: (port_) =>
           _platform.inner.wire_add_documents(port_, arg0, arg1, arg2),
       parseSuccessData: _wire2api_unit,
       constMeta: kAddDocumentsConstMeta,
-      argValues: [instanceDir, indexName, jsonDocuments],
+      argValues: [instanceDir, indexName, documents],
       hint: hint,
     ));
   }
 
   FlutterRustBridgeTaskConstMeta get kAddDocumentsConstMeta =>
       const FlutterRustBridgeTaskConstMeta(
         debugName: "add_documents",
-        argNames: ["instanceDir", "indexName", "jsonDocuments"],
+        argNames: ["instanceDir", "indexName", "documents"],
       );
 
   Future<void> deleteDocuments(
@@ -327,25 +378,25 @@ class EmbeddedMilliImpl implements EmbeddedMilli {
   Future<void> setDocuments(
       {required String instanceDir,
       required String indexName,
-      required List<String> jsonDocuments,
+      required List<String> documents,
       dynamic hint}) {
     var arg0 = _platform.api2wire_String(instanceDir);
     var arg1 = _platform.api2wire_String(indexName);
-    var arg2 = _platform.api2wire_StringList(jsonDocuments);
+    var arg2 = _platform.api2wire_StringList(documents);
     return _platform.executeNormal(FlutterRustBridgeTask(
       callFfi: (port_) =>
           _platform.inner.wire_set_documents(port_, arg0, arg1, arg2),
       parseSuccessData: _wire2api_unit,
       constMeta: kSetDocumentsConstMeta,
-      argValues: [instanceDir, indexName, jsonDocuments],
+      argValues: [instanceDir, indexName, documents],
       hint: hint,
     ));
   }
 
   FlutterRustBridgeTaskConstMeta get kSetDocumentsConstMeta =>
       const FlutterRustBridgeTaskConstMeta(
         debugName: "set_documents",
-        argNames: ["instanceDir", "indexName", "jsonDocuments"],
+        argNames: ["instanceDir", "indexName", "documents"],
       );
 
   Future<String?> getDocument(
@@ -905,17 +956,56 @@ class EmbeddedMilliWire implements FlutterRustBridgeWireBase {
   late final _wire_enforce_binding =
       _wire_enforce_bindingPtr.asFunction<void Function(int)>();
 
+  void wire_ensure_instance_initialized(
+    int port_,
+    ffi.Pointer<wire_uint_8_list> instace_dir,
+  ) {
+    return _wire_ensure_instance_initialized(
+      port_,
+      instace_dir,
+    );
+  }
+
+  late final _wire_ensure_instance_initializedPtr = _lookup<
+          ffi.NativeFunction<
+              ffi.Void Function(ffi.Int64, ffi.Pointer<wire_uint_8_list>)>>(
+      'wire_ensure_instance_initialized');
+  late final _wire_ensure_instance_initialized =
+      _wire_ensure_instance_initializedPtr
+          .asFunction<void Function(int, ffi.Pointer<wire_uint_8_list>)>();
+
+  void wire_ensure_index_initialized(
+    int port_,
+    ffi.Pointer<wire_uint_8_list> instance_dir,
+    ffi.Pointer<wire_uint_8_list> index_name,
+  ) {
+    return _wire_ensure_index_initialized(
+      port_,
+      instance_dir,
+      index_name,
+    );
+  }
+
+  late final _wire_ensure_index_initializedPtr = _lookup<
+      ffi.NativeFunction<
+          ffi.Void Function(ffi.Int64, ffi.Pointer<wire_uint_8_list>,
+              ffi.Pointer<wire_uint_8_list>)>>('wire_ensure_index_initialized');
+  late final _wire_ensure_index_initialized =
+      _wire_ensure_index_initializedPtr.asFunction<
+          void Function(int, ffi.Pointer<wire_uint_8_list>,
+              ffi.Pointer<wire_uint_8_list>)>();
+
   void wire_add_documents(
     int port_,
     ffi.Pointer<wire_uint_8_list> instance_dir,
     ffi.Pointer<wire_uint_8_list> index_name,
-    ffi.Pointer<wire_StringList> json_documents,
+    ffi.Pointer<wire_StringList> documents,
   ) {
     return _wire_add_documents(
       port_,
       instance_dir,
       index_name,
-      json_documents,
+      documents,
     );
   }
 
@@ -980,13 +1070,13 @@ class EmbeddedMilliWire implements FlutterRustBridgeWireBase {
     int port_,
     ffi.Pointer<wire_uint_8_list> instance_dir,
     ffi.Pointer<wire_uint_8_list> index_name,
-    ffi.Pointer<wire_StringList> json_documents,
+    ffi.Pointer<wire_StringList> documents,
   ) {
     return _wire_set_documents(
       port_,
       instance_dir,
       index_name,
-      json_documents,
+      documents,
     );
   }
 

packages/mimir/lib/src/impl/index_impl.dart

@@ -24,7 +24,7 @@ class MimirIndexImpl with MimirIndex {
     await milli.addDocuments(
       instanceDir: instanceDir,
       indexName: name,
-      jsonDocuments: documents.map((d) => json.encode(d)).toList(),
+      documents: documents.map((d) => json.encode(d)).toList(),
     );
     _changes.add(null);
   }
@@ -53,7 +53,7 @@ class MimirIndexImpl with MimirIndex {
     await milli.setDocuments(
       instanceDir: instanceDir,
       indexName: name,
-      jsonDocuments: documents.map((d) => json.encode(d)).toList(),
+      documents: documents.map((d) => json.encode(d)).toList(),
     );
     _changes.add(null);
   }

packages/mimir/native/Cargo.toml

@@ -16,7 +16,5 @@ serde = "1.0.*"
 serde_json = "1.0.*"
 parking_lot = "0.12.*"
 lazy_static = "1.*"
-# milli = { git = "https://github.com/meilisearch/milli", tag = "v0.38.0", package = "milli" }
-# filter-parser = { git = "https://github.com/meilisearch/milli", tag = "v0.38.0", package = "filter-parser" }
-milli = { git = "https://github.com/meilisearch/milli", package = "milli" }
-filter-parser = { git = "https://github.com/meilisearch/milli", package = "filter-parser" }
+# milli_v1 = { git = "https://github.com/meilisearch/milli", tag = "v0.38.0", package = "milli" }
+milli_v1 = { git = "https://github.com/GregoryConrad/milli", branch="filter-parser-convenience", package = "milli" }