Disorganized but working c

pgml-sdks/pgml/.gitignore

@@ -167,3 +167,8 @@ cython_debug/
 # local scratch pad
 scratch.sql
 scratch.py
+
+# Some SDK specific things
+expanded.rs
+test
+pgml.h

pgml-sdks/pgml/Cargo.toml

@@ -13,6 +13,9 @@ keywords = ["postgres", "machine learning", "vector databases", "embeddings"]
 name = "pgml"
 crate-type = ["lib", "cdylib"]
 
+[rust-analyzer.checkOnSave]
+extraArgs = ["--target-dir", "/path/to/proect/target/check"]
+
 [dependencies]
 rust_bridge = {path = "../rust-bridge/rust-bridge", version = "0.1.0"}
 sqlx = { version = "0.7.3", features = [ "runtime-tokio-rustls", "postgres", "json", "time", "uuid"] }
@@ -50,3 +53,4 @@ once_cell = "1.19.0"
 default = []
 python = ["dep:pyo3", "dep:pyo3-asyncio"]
 javascript = ["dep:neon"]
+c = []

pgml-sdks/pgml/go/Makefile

@@ -0,0 +1,31 @@
+BINARY_NAME=pgml
+
+build:
+	cargo build --features c
+	cargo expand --features c > expanded.rs
+	cbindgen --lang C -o pgml.h expanded.rs
+	# GOARCH=amd64 GOOS=darwin go build -o ${BINARY_NAME}-darwin main.go
+	GOARCH=amd64 GOOS=linux go build -o ${BINARY_NAME}-linux pgml.go
+	# GOARCH=amd64 GOOS=windows go build -o ${BINARY_NAME}-windows main.go
+
+build_test:
+	cargo build --features c
+	cargo expand --features c > expanded.rs
+	cbindgen --lang C -o pgml.h expanded.rs
+	gcc test.c -o test -l pgml -L ./../target/debug
+
+test: build_test
+	LD_LIBRARY_PATH=./../target/debug ./test
+
+test_c:
+	gcc test.c -o test -l pgml -L ./../target/debug
+	LD_LIBRARY_PATH=./../target/debug ./test
+
+run: build
+	LD_LIBRARY_PATH=./../target/debug ./${BINARY_NAME}-linux
+
+clean:
+	go clean
+	# rm ${BINARY_NAME}-darwin
+	rm ${BINARY_NAME}-linux
+	# rm ${BINARY_NAME}-windows

pgml-sdks/pgml/go/go.mod

@@ -0,0 +1,3 @@
+module pgml
+
+go 1.22.2

pgml-sdks/pgml/go/pgml.go

@@ -0,0 +1,23 @@
+package main
+
+/*
+#cgo LDFLAGS: -l pgml -L ./../target/debug
+#include "pgml.h"
+*/
+import "C"
+
+import (
+	"unsafe"
+)
+
+type Collection struct {
+	collection *C.CollectionC
+}
+
+func main() {
+	c_string_p := C.CString("Test CString")
+	defer C.free(unsafe.Pointer(c_string_p))
+	collection := C.new_collection(c_string_p)
+	C.test_collection(collection)
+	defer C.free_collection(collection)
+}

pgml-sdks/pgml/go/test.c

@@ -0,0 +1,37 @@
+#include <stdio.h>
+
+#include "pgml.h"
+
+int main() {
+  // Create the Collection and Pipeline
+  CollectionC * collection = CollectionC_new("test_c", NULL);
+  PipelineC * pipeline = PipelineC_new("test_c", "{\"text\": {\"splitter\": {\"model\": \"recursive_character\"},\"semantic_search\": {\"model\": \"intfloat/e5-small\"}}}");
+
+  // Add the Pipeline to the Collection
+  CollectionC_add_pipeline(collection, pipeline);
+
+  // Upsert the documents
+  char * documents_to_upsert[2] = {"{\"id\": \"doc1\", \"text\": \"test1\"}", "{\"id\": \"doc2\", \"text\": \"test2\"}"};
+  CollectionC_upsert_documents(collection, documents_to_upsert, 2, NULL);
+
+  // Retrieve the documents
+  unsigned long r_size = 0;
+  char** documents = CollectionC_get_documents(collection, NULL, &r_size);
+
+  // Print the documents
+  printf("\n\nPrinting documents:\n");
+  int i;
+  for (i = 0; i < r_size; i++) {
+    printf("Document %u -> %s\n", i, documents[i]);
+  }
+
+  // Search over the documents
+  r_size = 0;
+  char** results = CollectionC_vector_search(collection, "{\"query\": {\"fields\": {\"text\": {\"query\": \"Test query!\"}}}, \"limit\": 5}", pipeline, &r_size);
+  printf("\n\nPrinting results:\n");
+  for (i = 0; i < r_size; i++) {
+    printf("Result %u -> %s\n", i, results[i]);
+  }
+
+  return 0;
+}

pgml-sdks/pgml/src/builtins.rs

@@ -3,7 +3,8 @@ use sqlx::Row;
 use tracing::instrument;
 
 /// Provides access to builtin database methods
-#[derive(alias, Debug, Clone)]
+// #[derive(alias, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct Builtins {
     database_url: Option<String>,
 }
@@ -13,7 +14,7 @@ use crate::{get_or_initialize_pool, query_runner::QueryRunner, types::Json};
 #[cfg(feature = "python")]
 use crate::{query_runner::QueryRunnerPython, types::JsonPython};
 
-#[alias_methods(new, query, transform)]
+// #[alias_methods(new, query, transform)]
 impl Builtins {
     pub fn new(database_url: Option<String>) -> Self {
         Self { database_url }

pgml-sdks/pgml/src/collection.rs

@@ -33,6 +33,9 @@ use crate::{
 #[cfg(feature = "python")]
 use crate::{pipeline::PipelinePython, query_builder::QueryBuilderPython, types::JsonPython};
 
+#[cfg(feature = "c")]
+use crate::{languages::c::JsonC, pipeline::PipelineC, query_builder::QueryBuilderC};
+
 /// Our project tasks
 #[derive(Debug, Clone)]
 pub enum ProjectTask {
@@ -99,6 +102,35 @@ pub(crate) struct CollectionDatabaseData {
     pub project_info: ProjectInfo,
 }
 
+// #[repr(C)]
+// pub struct CollectionC {
+//     pub collection: *mut Collection,
+// }
+
+// #[no_mangle]
+// pub unsafe extern "C" fn new_collection(name: *const std::ffi::c_char) -> *mut CollectionC {
+//     let name = std::ffi::CStr::from_ptr(name).to_str().unwrap();
+//     println!("Nice one Silas: {}", name);
+//     let collection = Box::into_raw(Box::new(Collection::new(name, None).unwrap()));
+//     Box::into_raw(Box::new(CollectionC { collection }))
+// }
+
+// #[no_mangle]
+// pub unsafe extern "C" fn free_collection(collection: *mut CollectionC) {
+//     if collection.is_null() {
+//         return;
+//     }
+//     drop(Box::from_raw(collection));
+// }
+
+// #[no_mangle]
+// pub unsafe extern "C" fn test_collection(collection: *mut CollectionC) {
+//     let collection: *mut Collection = (*collection).collection;
+//     let collection: Collection = (*collection).clone();
+//     println!("Nice one Silas x two: {}", collection.name);
+//     println!("test");
+// }
+
 /// A collection of documents
 #[derive(alias, Debug, Clone)]
 pub struct Collection {

pgml-sdks/pgml/src/languages/c.rs

@@ -0,0 +1,18 @@
+use crate::types::{DateTime, GeneralJsonAsyncIterator, GeneralJsonIterator, Json};
+use rust_bridge::c::CustomInto;
+
+pub type JsonC = std::ffi::c_char;
+
+unsafe impl CustomInto<Json> for *mut JsonC {
+    unsafe fn custom_into(self) -> Json {
+        let s = std::ffi::CStr::from_ptr(self).to_str().unwrap();
+        serde_json::from_str::<serde_json::Value>(s).unwrap().into()
+    }
+}
+
+unsafe impl CustomInto<*mut JsonC> for Json {
+    unsafe fn custom_into(self) -> *mut JsonC {
+        let s = serde_json::to_string(&self).unwrap();
+        std::ffi::CString::new(s).unwrap().into_raw()
+    }
+}

pgml-sdks/pgml/src/languages/mod.rs

@@ -3,3 +3,6 @@ pub mod javascript;
 
 #[cfg(feature = "python")]
 pub mod python;
+
+#[cfg(feature = "c")]
+pub mod c;

pgml-sdks/pgml/src/model.rs

@@ -52,7 +52,8 @@ pub(crate) struct ModelDatabaseData {
 }
 
 /// A model used for embedding, inference, etc...
-#[derive(alias, Debug, Clone)]
+// #[derive(alias, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct Model {
     pub(crate) name: String,
     pub(crate) runtime: ModelRuntime,
@@ -66,7 +67,7 @@ impl Default for Model {
     }
 }
 
-#[alias_methods(new, transform)]
+// #[alias_methods(new, transform)]
 impl Model {
     /// Creates a new [Model]
     pub fn new(name: Option<String>, source: Option<String>, parameters: Option<Json>) -> Self {

pgml-sdks/pgml/src/open_source_ai.rs

@@ -14,7 +14,8 @@ use crate::{
 use crate::types::{GeneralJsonAsyncIteratorPython, GeneralJsonIteratorPython, JsonPython};
 
 /// A drop in replacement for OpenAI
-#[derive(alias, Debug, Clone)]
+// #[derive(alias, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct OpenSourceAI {
     database_url: Option<String>,
 }
@@ -162,13 +163,13 @@ impl Iterator for AsyncToSyncJsonIterator {
     }
 }
 
-#[alias_methods(
-    new,
-    chat_completions_create,
-    chat_completions_create_async,
-    chat_completions_create_stream,
-    chat_completions_create_stream_async
-)]
+// #[alias_methods(
+//     new,
+//     chat_completions_create,
+//     chat_completions_create_async,
+//     chat_completions_create_stream,
+//     chat_completions_create_stream_async
+// )]
 impl OpenSourceAI {
     /// Creates a new [OpenSourceAI]
     ///

pgml-sdks/pgml/src/pipeline.rs

@@ -19,6 +19,9 @@ use crate::{
 #[cfg(feature = "python")]
 use crate::types::JsonPython;
 
+#[cfg(feature = "c")]
+use crate::languages::c::JsonC;
+
 type ParsedSchema = HashMap<String, FieldAction>;
 
 #[derive(Deserialize)]

pgml-sdks/pgml/src/query_runner.rs

@@ -17,22 +17,23 @@ enum BindValue {
     Json(Json),
 }
 
-#[derive(alias, Clone, Debug)]
+// #[derive(alias, Clone, Debug)]
+#[derive(Clone, Debug)]
 pub struct QueryRunner {
     query: String,
     bind_values: Vec<BindValue>,
     database_url: Option<String>,
 }
 
-#[alias_methods(
-    fetch_all,
-    execute,
-    bind_string,
-    bind_int,
-    bind_float,
-    bind_bool,
-    bind_json
-)]
+// #[alias_methods(
+//     fetch_all,
+//     execute,
+//     bind_string,
+//     bind_int,
+//     bind_float,
+//     bind_bool,
+//     bind_json
+// )]
 impl QueryRunner {
     pub fn new(query: &str, database_url: Option<String>) -> Self {
         Self {

pgml-sdks/pgml/src/splitter.rs

@@ -19,7 +19,8 @@ pub(crate) struct SplitterDatabaseData {
 }
 
 /// A text splitter
-#[derive(alias, Debug, Clone)]
+// #[derive(alias, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct Splitter {
     pub(crate) name: String,
     pub(crate) parameters: Json,
@@ -32,7 +33,7 @@ impl Default for Splitter {
     }
 }
 
-#[alias_methods(new)]
+// #[alias_methods(new)]
 impl Splitter {
     /// Creates a new [Splitter]
     ///

pgml-sdks/pgml/src/transformer_pipeline.rs

@@ -4,7 +4,8 @@ use sqlx::Row;
 use tracing::instrument;
 
 /// Provides access to builtin database methods
-#[derive(alias, Debug, Clone)]
+// #[derive(alias, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct TransformerPipeline {
     task: Json,
     database_url: Option<String>,
@@ -16,7 +17,7 @@ use crate::{get_or_initialize_pool, types::Json};
 #[cfg(feature = "python")]
 use crate::types::{GeneralJsonAsyncIteratorPython, JsonPython};
 
-#[alias_methods(new, transform, transform_stream)]
+// #[alias_methods(new, transform, transform_stream)]
 impl TransformerPipeline {
     /// Creates a new [TransformerPipeline]
     ///