4.0.10 (#679)

* 600 add browser usage for the docker image (#601)

* add instructions to run browser UI

* set hostname to 0.0.0.0

* fix crash in rewrite call subquery (#604)

* fix crash in rewrite call subquery

* simplified test query

---------

Co-authored-by: Roi Lipman <swilly22@users.noreply.github.com>

* use debug assert in rust (#606)

* fix merge init (#609)

* fix merge init

* remove unused function, simplified branch assignment logic

* clean

---------

Co-authored-by: Roi Lipman <roilipman@gmail.com>

* Unwind persist (#613)

* persist unwind record

* deep clone base record

* Disable jit (#612)

* persist unwind record

* disable GraphBLAS JIT

* Update module api (#617)

* update RediSearch submodule

* bump RediSearch version

* updated redis module api header file

* make sure there's a record to emit (#623)

* update RediSearch (#640)

* Fix #634 Add Cloud link to README.md (#635)

* fix #645 Add objective to README (#646)

* Add objective to README

* Update wordlist.txt

---------

Co-authored-by: Roi Lipman <swilly22@users.noreply.github.com>

* fix union validation with call subquery (#648)

* Seek by id runtime optimization (#643)

* wip runtime optimization

* fix leak

* fix node by label scan

* fix leak

* fix leak

* address review

* add tests

* address review

* review

* review

* review

* add memory hook to roaring bitmap

* introduce bitmap range

---------

Co-authored-by: Roi Lipman <roilipman@gmail.com>
Co-authored-by: Roi Lipman <swilly22@users.noreply.github.com>

* zero dim vector save/load (#654)

* update redisearch, do not escape TAG field (#667)

* Code coverage (#674)

* add coverage build

* rename

* fix

* fix cache

* install redis in coverage

* ignore some path in code coverage (#678)

* Ref count record (#663)

* ref count record

* nullify record on deletion

* handle record ref count within execution plan return record

* address PR comments

* remove head & tail calls to persist

* trying to simplify and make value passing safer and simpler

* free aggregated group once yield

* removed Record_PersistScalars

* removed override flag

* remove extra persists

* group no longer holds a keys array

* only persists keys which are not graph entities

* allow for record cloning from different execution plans

---------

Co-authored-by: Dudi <16744955+dudizimber@users.noreply.github.com>
Co-authored-by: Avi Avni <avi.avni@gmail.com>
Co-authored-by: Guy Korland <gkorland@gmail.com>

.github/workflows/build.yml

@@ -91,7 +91,7 @@ jobs:
         if: steps.cache_graphblas.outputs.cache-hit == false || steps.cache_parser.outputs.cache-hit == false || steps.cache_search.outputs.cache-hit == false
         run: |
           id=$(docker create --platform=${{ matrix.platform }} localhost:5000/falkordb/falkordb-compiler)
-          docker cp $id:/FalkorDB/bin /FalkorDB/bin
+          docker cp $id:/FalkorDB/bin .
           docker rm -v $id
           ls -l -R /FalkorDB/bin
 

.github/workflows/coverage.yml

@@ -0,0 +1,66 @@
+name: Code Coverage
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [master, "[0-9]+.[0-9]+"]
+  push:
+    branches:
+      - master
+      - "[0-9]+.[0-9]+"
+
+jobs:
+  code-coverage:
+    runs-on: ubuntu-latest
+    container: falkordb/falkordb-build:ubuntu
+    steps:
+
+    - name: Safe dir
+      run: git config --global --add safe.directory '*'
+
+    - uses: actions/checkout@v4
+      with:
+        set-safe-directory: '*'
+        submodules: recursive
+
+    - name: Cache GraphBLAS
+      id: cache_graphblas
+      uses: actions/cache@v4
+      with:
+        path: ./bin/linux-x64-release-cov/GraphBLAS
+        key: graphblas-coverage-${{ hashFiles('./deps/GraphBLAS/Include/GraphBLAS.h') }}
+
+    - name: Cache parser
+      id: cache_parser
+      uses: actions/cache@v4
+      with:
+        path: ./bin/linux-x64-release-cov/libcypher-parser
+        key: parser-coverage-${{ hashFiles('./deps/libcypher-parser/lib/src/parser.c') }}
+
+    - name: Cache search
+      id: cache_search
+      uses: actions/cache@v4
+      with:
+        path: ./bin/linux-x64-debug-cov/search-static
+        key: search-coverage-${{ hashFiles('./deps/RediSearch/src/version.h') }}
+
+    - name: Build
+      run: |
+        rustup toolchain list
+        rustup default nightly
+        apt-get update
+        apt-get install -y clang libomp-dev libc6-dbg python3-venv
+        python3 -m venv venv
+        . venv/bin/activate
+        pip install -r tests/requirements.txt
+        apt update && apt install lsb-release -y
+        curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg && \
+        echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/redis.list
+        apt update && apt install redis -y
+        make coverage NPROC=16
+      continue-on-error: true
+
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4.0.1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/sanitize.yml

@@ -1,8 +1,13 @@
 name: Sanitizer
 
 on:
-  push:
   workflow_dispatch:
+  pull_request:
+    branches: [master, "[0-9]+.[0-9]+"]
+  push:
+    branches:
+      - master
+      - "[0-9]+.[0-9]+"
 
 jobs:
   sanitize-test:

codecov.yml

@@ -0,0 +1,5 @@
+ignore:
+  - "deps"
+  - "src/util/sds"
+  - "src/util/roaring.c"
+  - "tests"

src/arithmetic/comprehension_funcs/comprehension_funcs.c

@@ -19,9 +19,10 @@
 void ListComprehension_Free(void *ctx_ptr) {
 	ListComprehensionCtx *ctx = ctx_ptr;
 
-	// If this comprehension has a filter tree, free it.
+	// if this comprehension has a filter tree, free it
 	if(ctx->ft) FilterTree_Free(ctx->ft);
-	// If this comprehension has an eval routine, free it.
+
+	// if this comprehension has an eval routine, free it
 	if(ctx->eval_exp) AR_EXP_Free(ctx->eval_exp);
 
 	if(ctx->local_record) {
@@ -54,19 +55,23 @@ void *ListComprehension_Clone(void *orig) {
 	return ctx_clone;
 }
 
-static void _PopulateComprehensionCtx(ListComprehensionCtx *ctx, Record outer_record) {
+static void _PopulateComprehensionCtx
+(
+	ListComprehensionCtx *ctx,
+	Record outer_record
+) {
 	rax *local_record_map = raxClone(outer_record->mapping);
 	intptr_t id = raxSize(local_record_map);
 	raxTryInsert(local_record_map, (unsigned char *)ctx->variable_str,
 						  strlen(ctx->variable_str), (void *)id, NULL);
 	ctx->local_record = Record_New(local_record_map);
 
-	// This could just be assigned to 'id', but for safety we'll use a Record lookup.
+	// this could just be assigned to 'id'
+	// but for safety we'll use a Record lookup
 	ctx->variable_idx = Record_GetEntryIdx(ctx->local_record, ctx->variable_str);
 	ASSERT(ctx->variable_idx != INVALID_INDEX);
 }
 
-
 SIValue AR_ANY(SIValue *argv, int argc, void *private_data) {
 	if(SI_TYPE(argv[0]) == T_NULL) return SI_NullVal();
 	// ANY comprehensions are invoked with three children:
@@ -92,7 +97,7 @@ SIValue AR_ANY(SIValue *argv, int argc, void *private_data) {
 		// Add the current element to the record at its allocated position.
 		Record_AddScalar(r, ctx->variable_idx, current_elem);
 
-		// If any element in an ANY function passes the predicate, return true.
+		// if any element in an ANY function passes the predicate, return true.
 		FT_Result res = FilterTree_applyFilters(ctx->ft, r);
 		if(res == FILTER_PASS) return SI_BoolVal(true);
 		is_null |= (res == FILTER_NULL);
@@ -224,30 +229,36 @@ SIValue AR_NONE(SIValue *argv, int argc, void *private_data) {
 }
 
 
-SIValue AR_LIST_COMPREHENSION(SIValue *argv, int argc, void *private_data) {
+SIValue AR_LIST_COMPREHENSION
+(
+	SIValue *argv,
+	int argc,
+	void *private_data
+) {
 	if(SI_TYPE(argv[0]) == T_NULL) return SI_NullVal();
-	/* List comprehensions are invoked with three children:
-	 * The list to iterate over.
-	 * The current Record.
-	 * The function context. */
+
+	// list comprehensions are invoked with three children:
+	// the list to iterate over
+	// the current Record
+	// the function context
 	SIValue list = argv[0];
 	Record outer_record = argv[1].ptrval;
 	ListComprehensionCtx *ctx = private_data;
 
-	// On the first invocation, build the local Record.
+	// on the first invocation, build the local Record
 	if(ctx->local_record == NULL) _PopulateComprehensionCtx(ctx, outer_record);
 	Record r = ctx->local_record;
 
-	// Populate the local Record with the contents of the outer Record.
+	// populate the local Record with the contents of the outer Record
 	Record_Clone(outer_record, r);
-	// Instantiate the array to be returned.
+	// instantiate the array to be returned
 	SIValue retval = SI_Array(0);
 
 	uint len = SIArray_Length(list);
 	for(uint i = 0; i < len; i++) {
-		// Retrieve the current element.
+		// retrieve the current element
 		SIValue current_elem = SIArray_Get(list, i);
-		// Add the current element to the record at its allocated position.
+		// add the current element to the record at its allocated position
 		Record_AddScalar(r, ctx->variable_idx, current_elem);
 
 		// if the comprehension has a filter tree

src/arithmetic/list_funcs/list_funcs.c

@@ -399,7 +399,6 @@ SIValue AR_HEAD(SIValue *argv, int argc, void *private_data) {
 	uint arrayLen = SIArray_Length(value);
 	if(arrayLen == 0) return SI_NullVal();
 	SIValue retval = SIArray_Get(value, 0);
-	SIValue_Persist(&retval);
 	return retval;
 }
 
@@ -413,7 +412,6 @@ SIValue AR_LAST(SIValue *argv, int argc, void *private_data) {
 	uint arrayLen = SIArray_Length(value);
 	if(arrayLen == 0) return SI_NullVal();
 	SIValue retval = SIArray_Get(value, arrayLen-1);
-	SIValue_Persist(&retval);
 	return retval;
 }
 

src/datatypes/vector.c

@@ -82,7 +82,9 @@ SIValue SIVector_FromBinary
 	size_t elem_size = sizeof(float);
 
 	// set vector's elements
-	fread_assert(SIVector_Elements(v), dim * elem_size, stream);
+	if(dim > 0) {
+		fread_assert(SIVector_Elements(v), dim * elem_size, stream);
+	}
 
 	return v;
 }

src/effects/effects.c

@@ -238,7 +238,9 @@ static void EffectsBuffer_WriteSIVector
 	size_t elem_size = sizeof(float);
 	size_t n = dim * elem_size;
 
-	EffectsBuffer_WriteBytes(elements, n, buff);
+	if(n > 0) {
+		EffectsBuffer_WriteBytes(elements, n, buff);
+	}
 }
 
 // dump attributes to stream

src/execution_plan/execution_plan.c

@@ -385,20 +385,42 @@ inline rax *ExecutionPlan_GetMappings(const ExecutionPlan *plan) {
 	return plan->record_map;
 }
 
-Record ExecutionPlan_BorrowRecord(ExecutionPlan *plan) {
+Record ExecutionPlan_BorrowRecord
+(
+	ExecutionPlan *plan
+) {
 	rax *mapping = ExecutionPlan_GetMappings(plan);
 	ASSERT(plan->record_pool);
 
-	// Get a Record from the pool and set its owner and mapping.
+	// get a Record from the pool and set its owner and mapping
 	Record r = ObjectPool_NewItem(plan->record_pool);
-	r->owner = plan;
-	r->mapping = mapping;
+
+	r->owner     = plan;
+	r->mapping   = mapping;
+	r->ref_count = 1;
+
 	return r;
 }
 
-void ExecutionPlan_ReturnRecord(const ExecutionPlan *plan, Record r) {
+void ExecutionPlan_ReturnRecord
+(
+	const ExecutionPlan *plan,
+	Record r
+) {
 	ASSERT(plan && r);
-	ObjectPool_DeleteItem(plan->record_pool, r);
+	ASSERT(r->ref_count > 0);
+
+	// decrease record ref count
+	r->ref_count--;
+
+	// free record when ref count reached 0
+	if(r->ref_count == 0) {
+		// call recursively for parent
+		if(r->parent != NULL) {
+			ExecutionPlan_ReturnRecord(r->parent->owner, r->parent);
+		}
+		ObjectPool_DeleteItem(plan->record_pool, r);
+	}
 }
 
 //------------------------------------------------------------------------------

src/execution_plan/ops/op.c

@@ -295,15 +295,11 @@ Record OpBase_CloneRecord
 ) {
 	Record clone = ExecutionPlan_BorrowRecord((struct ExecutionPlan *)r->owner);
 	Record_Clone(r, clone);
-	return clone;
-}
 
-Record OpBase_DeepCloneRecord
-(
-	Record r
-) {
-	Record clone = ExecutionPlan_BorrowRecord((struct ExecutionPlan *)r->owner);
-	Record_DeepClone(r, clone);
+	// increase r's ref count and set r as clone's parent
+	r->ref_count++;
+	clone->parent = r;
+
 	return clone;
 }
 
@@ -337,9 +333,25 @@ OpBase *OpBase_GetChild
 
 inline void OpBase_DeleteRecord
 (
-	Record r
+	Record *r
+) {
+	ExecutionPlan_ReturnRecord((*r)->owner, *r);
+	// nullify record
+	*r = NULL;
+}
+
+// merge src into dest and deletes src
+void OpBase_MergeRecords
+(
+	Record dest,  // entries are merged into this record
+	Record *src   // entries are merged from this record
 ) {
-	ExecutionPlan_ReturnRecord(r->owner, r);
+	ASSERT(dest != NULL);
+	ASSERT(src  != NULL && *src != NULL);
+	ASSERT(dest != *src);
+
+	Record_Merge(dest, *src);
+	OpBase_DeleteRecord(src);
 }
 
 OpBase *OpBase_Clone

src/execution_plan/ops/op.h

@@ -287,14 +287,16 @@ Record OpBase_CloneRecord
 	Record r
 );
 
-// deep clones given record
-Record OpBase_DeepCloneRecord
+// release record
+void OpBase_DeleteRecord
 (
-	Record r
+	Record *r
 );
 
-// release record
-void OpBase_DeleteRecord
+// merge src into dest and deletes src
+void OpBase_MergeRecords
 (
-	Record r
+	Record dest,  // entries are merged into this record
+	Record *src   // entries are merged from this record
 );
+