integrate falkordbrs (#544)

* integrate falkordbrs

* fix submodule

* fix

* fix

* separate build for debug

* build rust with sanitizer flags when needed

* add workspace and update

* fix leak

* update

* address review

* use alpine image in sanitizer

* move to ubuntu

* enable cargo

* fix

* fix

* fix

* fix

* update

* use target dir

* addres review

* use nightly rust in sanitizer

* address review

* fix

* fix

* update build

* fix

* address review

* build

* update

* update

* fix build

* update

* fix codeql and address review

* address review

* add alpine

* update for alpine

* update

* fix build

* remove debian

* update

* update

* use current headers instead of generated one

* clean

* fix for mac

* document alloc funtion

* move to ubuntu image

* change docker tag

* update to latest

* update format

* revert

* fix leak

* always compile rust

* fix makefile

* review

* address review

* address review

.github/workflows/codeql-analysis.yml

@@ -28,7 +28,7 @@ jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
-    container: falkordb/falkordb-build:latest
+    container: falkordb/falkordb-build:ubuntu
     permissions:
       actions: read
       contents: read
@@ -88,6 +88,7 @@ jobs:
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Build
       run: |
+        rustup default stable
         make build
 
     # ℹ️ Command-line programs to run using the OS shell.

.github/workflows/sanitize.yml

@@ -41,6 +41,8 @@ jobs:
 
     - name: Build
       run: |
+        rustup toolchain list
+        rustup default nightly
         apt-get update
         apt-get install -y clang libomp-dev libc6-dbg python3-venv
         python3 -m venv venv

.gitignore

@@ -67,3 +67,5 @@ deps/GraphBLAS/Config/GraphBLAS.h.tmp
 /venv/
 /.vscode/
 /1/
+
+target

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "deps/oniguruma"]
 	path = deps/oniguruma
 	url = https://github.com/kkos/oniguruma.git
+[submodule "deps/FalkorDB-rs"]
+	path = deps/FalkorDB-rs
+	url = https://github.com/FalkorDB/FalkorDB-rs.git

CMakeLists.txt

@@ -52,8 +52,8 @@ set(FALKORDB_OBJECTS $<TARGET_OBJECTS:falkordb>)
 
 find_package(OpenSSL)
 
-lists_from_env(GRAPHBLAS LIBXXHASH RAX LIBCYPHER_PARSER REDISEARCH_LIBS UTF8PROC ONIGURUMA)
-set(FALKORDB_LIBS ${GRAPHBLAS} ${LIBXXHASH} ${RAX} ${LIBCYPHER_PARSER} ${REDISEARCH_LIBS} ${UTF8PROC} ${ONIGURUMA} OpenSSL::SSL)
+lists_from_env(GRAPHBLAS LIBXXHASH RAX LIBCYPHER_PARSER REDISEARCH_LIBS UTF8PROC ONIGURUMA FalkorDBRS)
+set(FALKORDB_LIBS ${GRAPHBLAS} ${LIBXXHASH} ${RAX} ${LIBCYPHER_PARSER} ${REDISEARCH_LIBS} ${UTF8PROC} ${ONIGURUMA} ${FalkorDBRS} OpenSSL::SSL)
 
 target_link_options(falkordb PRIVATE ${CMAKE_LD_FLAGS_LIST} ${CMAKE_SO_LD_FLAGS_LIST})
 target_link_libraries(falkordb PRIVATE ${FALKORDB_LIBS} ${CMAKE_LD_LIBS})

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+resolver = "2"
+members = [
+    "deps/FalkorDB-rs",
+]

Makefile

@@ -137,9 +137,13 @@ REDISEARCH_DIR = $(ROOT)/deps/RediSearch
 export REDISEARCH_BINROOT=$(BINROOT)
 include $(ROOT)/build/RediSearch/Makefile.defs
 
+FalkorDBRS_DIR = $(ROOT)/deps/FalkorDB-rs
+export FalkorDBRS_BINDIR=$(BINROOT)/FalkorDB-rs
+include $(ROOT)/build/FalkorDB-rs/Makefile.defs
+
 BIN_DIRS += $(REDISEARCH_BINROOT)/search-static
 
-LIBS=$(RAX) $(LIBXXHASH) $(GRAPHBLAS) $(REDISEARCH_LIBS) $(LIBCYPHER_PARSER) $(UTF8PROC) $(ONIGURUMA)
+LIBS=$(RAX) $(LIBXXHASH) $(GRAPHBLAS) $(REDISEARCH_LIBS) $(LIBCYPHER_PARSER) $(UTF8PROC) $(ONIGURUMA) $(FalkorDBRS)
 
 #----------------------------------------------------------------------------------------------
 
@@ -189,11 +193,13 @@ ifneq ($(call files_missing,$(REDISEARCH_LIBS)),)
 MISSING_DEPS += $(REDISEARCH_LIBS)
 endif
 
+MISSING_DEPS += falkordbrs
+
 ifneq ($(MISSING_DEPS),)
 DEPS=1
 endif
 
-DEPENDENCIES=libcypher-parser graphblas redisearch rax libxxhash utf8proc oniguruma
+DEPENDENCIES=libcypher-parser graphblas redisearch rax libxxhash utf8proc oniguruma falkordbrs
 
 ifneq ($(filter all deps $(DEPENDENCIES) pack,$(MAKECMDGOALS)),)
 DEPS=1
@@ -219,7 +225,7 @@ include $(MK)/rules
 
 ifeq ($(DEPS),1)
 
-deps: $(LIBCYPHER_PARSER) $(GRAPHBLAS) $(LIBXXHASH) $(RAX) $(REDISEARCH_LIBS) $(UTF8PROC) $(ONIGURUMA)
+deps: $(LIBCYPHER_PARSER) $(GRAPHBLAS) $(LIBXXHASH) $(RAX) $(REDISEARCH_LIBS) $(UTF8PROC) $(ONIGURUMA) falkordbrs
 
 libxxhash: $(LIBXXHASH)
 
@@ -265,7 +271,21 @@ $(REDISEARCH_LIBS):
 	@echo Building $@ ...
 	$(SHOW)$(MAKE) -C $(REDISEARCH_DIR) STATIC=1 BINROOT=$(REDISEARCH_BINROOT) CC=$(CC) CXX=$(CXX)
 
-.PHONY: libcypher-parser graphblas redisearch libxxhash rax utf8proc oniguruma
+
+ifneq ($(DEBUG),1)
+CARGO_FLAGS=--release
+endif
+
+ifneq ($(SAN),)
+export RUSTFLAGS=-Zsanitizer=$(SAN)
+CARGO_FLAGS=--target x86_64-unknown-linux-gnu
+endif
+
+falkordbrs:
+	@echo Building $@ ...
+	cd deps/FalkorDB-rs && cargo build $(CARGO_FLAGS) --features falkordb_allocator --target-dir $(FalkorDBRS_BINDIR)
+
+.PHONY: libcypher-parser graphblas redisearch libxxhash rax utf8proc oniguruma falkordbrs
 
 #----------------------------------------------------------------------------------------------
 

build/FalkorDB-rs/Makefile.defs

@@ -0,0 +1,10 @@
+
+ifneq ($(SAN),)
+export FalkorDBRS=$(FalkorDBRS_BINDIR)/x86_64-unknown-linux-gnu/debug/libFalkorDB_rs.a
+else
+ifeq ($(DEBUG),1)
+export FalkorDBRS=$(FalkorDBRS_BINDIR)/debug/libFalkorDB_rs.a
+else
+export FalkorDBRS=$(FalkorDBRS_BINDIR)/release/libFalkorDB_rs.a
+endif
+endif

src/arithmetic/arithmetic_expression.c

@@ -235,7 +235,7 @@ AR_ExpNode *AR_EXP_NewAttributeAccessNode(AR_ExpNode *entity,
 	GraphContext *gc = QueryCtx_GetGraphCtx();
 	SIValue prop_idx = SI_LongVal(ATTRIBUTE_ID_NONE);
 	SIValue prop_name = SI_ConstStringVal((char *)attr);
-	Attribute_ID idx = GraphContext_GetAttributeID(gc, attr);
+	AttributeID idx = GraphContext_GetAttributeID(gc, attr);
 
 	if(idx != ATTRIBUTE_ID_NONE) prop_idx = SI_LongVal(idx);
 

src/arithmetic/entity_funcs/entity_funcs.c

@@ -265,7 +265,7 @@ SIValue AR_PROPERTY(SIValue *argv, int argc, void *private_data) {
 		// retrieve entity property
 		GraphEntity *graph_entity = (GraphEntity *)obj.ptrval;
 		const char *prop_name     = argv[1].stringval;
-		Attribute_ID prop_idx     = argv[2].longval;
+		AttributeID prop_idx     = argv[2].longval;
 
 		// We have the property string, attempt to look up the index now.
 		if(prop_idx == ATTRIBUTE_ID_NONE) {

src/bulk_insert/bulk_insert.c

@@ -86,7 +86,7 @@ static int* _BulkInsert_ReadHeaderLabels
 }
 
 // read the property keys from a header
-static Attribute_ID* _BulkInsert_ReadHeaderProperties
+static AttributeID* _BulkInsert_ReadHeaderProperties
 (
 	GraphContext* gc,
 	SchemaType t,
@@ -105,7 +105,7 @@ static Attribute_ID* _BulkInsert_ReadHeaderProperties
 
     if (*prop_count == 0) return NULL;
 
-    Attribute_ID* prop_indices = rm_malloc(*prop_count * sizeof(Attribute_ID));
+    AttributeID* prop_indices = rm_malloc(*prop_count * sizeof(AttributeID));
 
     // the rest of the line is [char *prop_key] * prop_count
 	for (uint j = 0; j < *prop_count; j++) {
@@ -207,7 +207,7 @@ static int _BulkInsert_ProcessNodeFile
     int* label_ids = _BulkInsert_ReadHeaderLabels(gc, SCHEMA_NODE, data, &data_idx);
     uint label_count = array_len(label_ids);
     // read the CSV header properties and collect their indices
-    Attribute_ID* prop_indices = _BulkInsert_ReadHeaderProperties(gc, SCHEMA_NODE, data,
+    AttributeID* prop_indices = _BulkInsert_ReadHeaderProperties(gc, SCHEMA_NODE, data,
 	&data_idx, &prop_count);
 
     // sync each matrix once
@@ -266,7 +266,7 @@ static int _BulkInsert_ProcessEdgeFile
     ASSERT(type_count == 1);
 
     int type_id = type_ids[0];
-    Attribute_ID* prop_indices = _BulkInsert_ReadHeaderProperties(gc, SCHEMA_EDGE,
+    AttributeID* prop_indices = _BulkInsert_ReadHeaderProperties(gc, SCHEMA_EDGE,
 	data, &data_idx, &prop_count);
 
     // sync matrix once

src/commands/cmd_constraint.c

@@ -9,7 +9,6 @@
 #include "../query_ctx.h"
 #include "../index/indexer.h"
 #include "../graph/graph_hub.h"
-#include "../undo_log/undo_log.h"
 #include "../graph/graphcontext.h"
 #include "constraint/constraint.h"
 
@@ -21,13 +20,13 @@ typedef enum {
 	CT_DROP     // drop constraint
 } ConstraintOp;
 
-static inline int _cmp_Attribute_ID
+static inline int _cmp_AttributeID
 (
 	const void *a,
 	const void *b
 ) {
-	const Attribute_ID *_a = a;
-	const Attribute_ID *_b = b;
+	const AttributeID *_a = a;
+	const AttributeID *_b = b;
 	return *_a - *_b;
 }
 
@@ -146,7 +145,7 @@ static bool _Constraint_Drop
 	const char **props      // properties
 ) {
 	bool res = true;  // optimistic
-	Attribute_ID attrs[n];
+	AttributeID attrs[n];
 
 	//--------------------------------------------------------------------------
 	// try to get graph
@@ -179,7 +178,7 @@ static bool _Constraint_Drop
 		const char *prop = props[i];
 
 		// try to get property ID
-		Attribute_ID id = GraphContext_GetAttributeID(gc, prop);
+		AttributeID id = GraphContext_GetAttributeID(gc, prop);
 
 		if(id == ATTRIBUTE_ID_NONE) {
 			// attribute missing
@@ -261,7 +260,7 @@ static bool _Constraint_Create
 	// convert attribute name to attribute ID
 	//--------------------------------------------------------------------------
 
-	Attribute_ID attr_ids[n];
+	AttributeID attr_ids[n];
 	for(uint i = 0; i < n; i++) {
 		attr_ids[i] = FindOrAddAttribute(gc, props[i], true);
 	}
@@ -272,7 +271,7 @@ static bool _Constraint_Create
 
 	// sort the properties for an easy comparison later
 	bool dups = false;
-	qsort(attr_ids, n, sizeof(Attribute_ID), _cmp_Attribute_ID);
+	qsort(attr_ids, n, sizeof(AttributeID), _cmp_AttributeID);
 	for(uint i = 0; i < n - 1; i++) {
 		if(attr_ids[i] == attr_ids[i+1]) {
 			dups = true;
@@ -291,7 +290,7 @@ static bool _Constraint_Create
 	// must be aligned with attribute names array
 	for(uint i = 0; i < n; i++) {
 		// get attribute id for attribute name
-		Attribute_ID attr_id = attr_ids[i];
+		AttributeID attr_id = attr_ids[i];
 
 		// update props to hold graph context's attribute name
 		props[i] = GraphContext_GetAttributeString(gc, attr_id);

src/commands/index_operations.c

@@ -126,7 +126,7 @@ static bool index_delete
 	//--------------------------------------------------------------------------
 
 	// quickly return if attribute doesn't exist
-	Attribute_ID attr_id = GraphContext_GetAttributeID(gc, attr);
+	AttributeID attr_id = GraphContext_GetAttributeID(gc, attr);
 	if(attr_id == ATTRIBUTE_ID_NONE) {
 		ErrorCtx_SetError(EMSG_UNABLE_TO_DROP_INDEX, lbl, attr);
 		return false;

src/constraint/constraint.c

@@ -24,7 +24,7 @@ typedef struct _Constraint {
 	Constraint_SetPrivateDataCB set_pdata;  // set private data
 	Constraint_GetPrivateDataCB get_pdata;  // get private data
 	int schema_id;                          // enforced label/relationship-type
-	Attribute_ID *attrs;                    // enforced attributes
+	AttributeID *attrs;                     // enforced attributes
 	const char **attr_names;                // enforced attribute names
 	ConstraintStatus status;                // constraint status
 	uint _Atomic pending_changes;           // number of pending changes
@@ -37,7 +37,7 @@ typedef struct _Constraint {
 extern Constraint Constraint_UniqueNew
 (
 	LabelID l,                // label/relation ID
-	Attribute_ID *fields,     // enforced fields
+	AttributeID *fields,      // enforced fields
 	const char **attr_names,  // enforced attribute names
 	uint8_t n_fields,         // number of fields
 	GraphEntityType et,       // entity type
@@ -48,7 +48,7 @@ extern Constraint Constraint_UniqueNew
 extern Constraint Constraint_MandatoryNew
 (
 	LabelID l,                // label/relation ID
-	Attribute_ID *fields,     // enforced fields
+	AttributeID *fields,      // enforced fields
 	const char **attr_names,  // enforced attribute names
 	uint8_t n_fields,         // number of fields
 	GraphEntityType et        // entity type
@@ -88,7 +88,7 @@ Constraint Constraint_New
 	struct GraphContext *gc,
 	ConstraintType t,         // type of constraint
 	int schema_id,            // schema ID
-	Attribute_ID *fields,     // enforced fields
+	AttributeID *fields,      // enforced fields
 	const char **attr_names,  // enforced attribute names
 	uint8_t n_fields,         // number of fields
 	GraphEntityType et,       // entity type
@@ -237,9 +237,9 @@ void *Constraint_GetPrivateData
 // returns a shallow copy of constraint attributes
 uint8_t Constraint_GetAttributes
 (
-	const Constraint c,             // constraint from which to get attributes
-	const Attribute_ID **attr_ids,  // array of constraint attribute IDs
-	const char ***attr_names        // array of constraint attribute names
+	const Constraint c,            // constraint from which to get attributes
+	const AttributeID **attr_ids,  // array of constraint attribute IDs
+	const char ***attr_names       // array of constraint attribute names
 ) {
 	ASSERT(c != NULL);
 
@@ -257,8 +257,8 @@ uint8_t Constraint_GetAttributes
 // checks if constraint enforces attribute
 bool Constraint_ContainsAttribute
 (
-	Constraint c,         // constraint to query
-	Attribute_ID attr_id  // enforced attribute
+	Constraint c,        // constraint to query
+	AttributeID attr_id  // enforced attribute
 ) {
 	for(uint8_t i = 0; i < c->n_attr; i++) {
 		if(c->attrs[i] == attr_id) {

src/constraint/constraint.h

@@ -59,7 +59,7 @@ Constraint Constraint_New
 	struct GraphContext *gc,
 	ConstraintType t,         // type of constraint
 	LabelID l,                // label/relation ID
-	Attribute_ID *fields,     // enforced fields
+	AttributeID *fields,      // enforced fields
 	const char **attr_names,  // enforced attribute names
 	uint8_t n_fields,         // number of fields
 	GraphEntityType et,       // entity type
@@ -128,16 +128,16 @@ void *Constraint_GetPrivateData
 // returns a shallow copy of constraint attributes
 uint8_t Constraint_GetAttributes
 (
-	const Constraint c,             // constraint from which to get attributes
-	const Attribute_ID **attr_ids,  // array of constraint attribute IDs
-	const char ***attr_names        // array of constraint attribute names
+	const Constraint c,            // constraint from which to get attributes
+	const AttributeID **attr_ids,  // array of constraint attribute IDs
+	const char ***attr_names       // array of constraint attribute names
 );
 
 // checks if constraint enforces attribute
 bool Constraint_ContainsAttribute
 (
-	Constraint c,         // constraint to query
-	Attribute_ID attr_id  // enforced attribute
+	Constraint c,        // constraint to query
+	AttributeID attr_id  // enforced attribute
 );
 
 // returns number of pending changes