4.0.5 (#588)

* Add self hosted (#584)

* add self hosted runner

* change self hosted label

* Revert "change self hosted label"

This reverts commit 9c13255.

* re-add arm64 for flow tests

* remove runtime tests, remove extra prints (#575)

* remove runtime tests, remove extra prints

* update tests requirements

* remove pathos requierment, switch to python asyncio

* switch to asyncio.run

* install RLTest directly from github repo

* update dockerfile compiler

* install redis

* remove sudu

* remove sudo

* fix setup redis

* fix install redis

* use venv

* set venv in path

* fix sanitizer

* fix sanitizer build

* fix sanitier build

* fix sanitizer build

* fix sanitizer build

* add clang to sanitizer

* add libc6-dbg

* .

* add libomp

* fix sanitizer

* fix redis install

* fix redis

* reduce flow test parallelism

* create connection pool inside async func and make sure to close it

* close async connection

* remove flushall

* specify port for connection pool

* address PR comments

* Update build.yml

* Update test_cache.py

* Update test_concurrent_query.py

* Update test_concurrent_query.py

* Update test_edge_index_scans.py

* Update test_encode_decode.py

* Update test_graph_deletion.py

* Update test_graph_deletion.py

* Update test_path_filter.py

* address PR comments

---------

Co-authored-by: Avi Avni <avi.avni@gmail.com>

* update deps (#587)

* integrate falkordbrs (#544)

* integrate falkordbrs

* fix submodule

* fix

* fix

* separate build for debug

* build rust with sanitizer flags when needed

* add workspace and update

* fix leak

* update

* address review

* use alpine image in sanitizer

* move to ubuntu

* enable cargo

* fix

* fix

* fix

* fix

* update

* use target dir

* addres review

* use nightly rust in sanitizer

* address review

* fix

* fix

* update build

* fix

* address review

* build

* update

* update

* fix build

* update

* fix codeql and address review

* address review

* add alpine

* update for alpine

* update

* fix build

* remove debian

* update

* update

* use current headers instead of generated one

* clean

* fix for mac

* document alloc funtion

* move to ubuntu image

* change docker tag

* update to latest

* update format

* revert

* fix leak

* always compile rust

* fix makefile

* review

* address review

* address review

* GRAPH.COPY (#585)

* initial work graph.copy

* pipe serializer

* abstract serializer

* graph encoder switch to abstract serializer

* switch decoder to abstract serializer

* copy command wip

* first copy

* early flow test

* additional tests

* transition to worker thread execution

* switch to worker thread

* retry when fork failed

* skip graph copy replication test when running under sanitizer

* import SANITIZER

* plant cloned key only after decoding is done

* switch to CRON job

* fork writes to file

* add logs

* fix leak

* replicate via RESTORE, switch to FILE*

* add graph.restore

* fix unit-test wrong stream acquisition

* fork requires gil

* rename encoder/decoder v14 to latest

* version bump

---------

Co-authored-by: Dudi <16744955+dudizimber@users.noreply.github.com>
Co-authored-by: Avi Avni <avi.avni@gmail.com>

.github/workflows/build.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ARM64' }}
     container: falkordb/falkordb-build:latest
     services:
       registry:
@@ -67,9 +67,6 @@ jobs:
           mkdir -p /FalkorDB
           mv $GITHUB_WORKSPACE/* /FalkorDB
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
@@ -139,7 +136,7 @@ jobs:
 
   unit-tests:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ARM64' }}
     strategy:
       fail-fast: false
       matrix:
@@ -154,11 +151,7 @@ jobs:
         with:
           name: falkordb-tests-${{ env.ARCH_SUFFIX }}
           path: /tmp
-
-      - name: Set up QEMU
-        if: ${{ matrix.platform == 'linux/arm64' }}
-        uses: docker/setup-qemu-action@v3
-
+
       - name: Load image
         id: load_image
         run: |
@@ -170,13 +163,13 @@ jobs:
 
   flow-tests:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ARM64' }}
     strategy:
       fail-fast: false
       matrix:
         platform:
           - linux/amd64
-          # - linux/arm64
+          - linux/arm64
     env:
       ARCH_SUFFIX: ${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}
     steps:
@@ -186,10 +179,6 @@ jobs:
           name: falkordb-tests-${{ env.ARCH_SUFFIX }}
           path: /tmp
 
-      - name: Set up QEMU
-        if: ${{ matrix.platform == 'linux/arm64' }}
-        uses: docker/setup-qemu-action@v3
-
       - name: Load image
         id: load_image
         run: |
@@ -201,7 +190,7 @@ jobs:
 
   tck-tests:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ARM64' }}
     strategy:
       fail-fast: false
       matrix:
@@ -217,10 +206,6 @@ jobs:
           name: falkordb-tests-${{ env.ARCH_SUFFIX }}
           path: /tmp
 
-      - name: Set up QEMU
-        if: ${{ matrix.platform == 'linux/arm64' }}
-        uses: docker/setup-qemu-action@v3
-
       - name: Load image
         id: load_image
         run: |
@@ -232,7 +217,7 @@ jobs:
 
   fuzz-tests:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ARM64' }}
     strategy:
       fail-fast: false
       matrix:
@@ -248,15 +233,11 @@ jobs:
           name: falkordb-tests-${{ env.ARCH_SUFFIX }}
           path: /tmp
 
-      - name: Set up QEMU
-        if: ${{ matrix.platform == 'linux/arm64' }}
-        uses: docker/setup-qemu-action@v3
-
       - name: Load image
         id: load_image
         run: |
           docker load --input /tmp/falkordb-tests-${{ env.ARCH_SUFFIX }}.tar
 
       - name: Fuzz tests
         run: |
-          docker run --platform ${{ matrix.platform }} --rm falkordb/falkordb-tests make fuzz TIMEOUT=180
+          docker run --platform ${{ matrix.platform }} --rm falkordb/falkordb-tests make fuzz TIMEOUT=180

.github/workflows/codeql-analysis.yml

@@ -28,7 +28,7 @@ jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
-    container: falkordb/falkordb-build:latest
+    container: falkordb/falkordb-build:ubuntu
     permissions:
       actions: read
       contents: read
@@ -88,6 +88,7 @@ jobs:
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Build
       run: |
+        rustup default stable
         make build
 
     # ℹ️ Command-line programs to run using the OS shell.

.github/workflows/sanitize.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   sanitize-test:
     runs-on: ubuntu-latest
-    container: falkordb/falkordb-build:latest
+    container: falkordb/falkordb-build:ubuntu
     steps:
 
     - name: Safe dir
@@ -40,7 +40,15 @@ jobs:
         key: search-sanitizer-${{ hashFiles('./deps/RediSearch/src/version.h') }}
 
     - name: Build
-      run: make CLANG=1 SAN=address NPROC=16
+      run: |
+        rustup toolchain list
+        rustup default nightly
+        apt-get update
+        apt-get install -y clang libomp-dev libc6-dbg python3-venv
+        python3 -m venv venv
+        . venv/bin/activate
+        pip install -r tests/requirements.txt
+        make CLANG=1 SAN=address NPROC=16
 
     - name: Unit tests
       id: unit_tests
@@ -55,7 +63,9 @@ jobs:
 
     - name: Flow tests
       id: flow_tests
-      run: make flow-tests CLEAR_LOGS=0 SAN=address
+      run: |
+        . venv/bin/activate
+        make flow-tests CLEAR_LOGS=0 SAN=address
       continue-on-error: true
 
     - uses: actions/upload-artifact@v4
@@ -66,7 +76,9 @@ jobs:
 
     - name: TCK tests
       id: tck_tests
-      run: make tck-tests CLEAR_LOGS=0 SAN=address
+      run: |
+        . venv/bin/activate
+        make tck-tests CLEAR_LOGS=0 SAN=address
       continue-on-error: true
 
     - uses: actions/upload-artifact@v4

.gitignore

@@ -67,3 +67,5 @@ deps/GraphBLAS/Config/GraphBLAS.h.tmp
 /venv/
 /.vscode/
 /1/
+
+target

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "deps/oniguruma"]
 	path = deps/oniguruma
 	url = https://github.com/kkos/oniguruma.git
+[submodule "deps/FalkorDB-rs"]
+	path = deps/FalkorDB-rs
+	url = https://github.com/FalkorDB/FalkorDB-rs.git

CMakeLists.txt

@@ -52,8 +52,8 @@ set(FALKORDB_OBJECTS $<TARGET_OBJECTS:falkordb>)
 
 find_package(OpenSSL)
 
-lists_from_env(GRAPHBLAS LIBXXHASH RAX LIBCYPHER_PARSER REDISEARCH_LIBS UTF8PROC ONIGURUMA)
-set(FALKORDB_LIBS ${GRAPHBLAS} ${LIBXXHASH} ${RAX} ${LIBCYPHER_PARSER} ${REDISEARCH_LIBS} ${UTF8PROC} ${ONIGURUMA} OpenSSL::SSL)
+lists_from_env(GRAPHBLAS LIBXXHASH RAX LIBCYPHER_PARSER REDISEARCH_LIBS UTF8PROC ONIGURUMA FalkorDBRS)
+set(FALKORDB_LIBS ${GRAPHBLAS} ${LIBXXHASH} ${RAX} ${LIBCYPHER_PARSER} ${REDISEARCH_LIBS} ${UTF8PROC} ${ONIGURUMA} ${FalkorDBRS} OpenSSL::SSL)
 
 target_link_options(falkordb PRIVATE ${CMAKE_LD_FLAGS_LIST} ${CMAKE_SO_LD_FLAGS_LIST})
 target_link_libraries(falkordb PRIVATE ${FALKORDB_LIBS} ${CMAKE_LD_LIBS})

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+resolver = "2"
+members = [
+    "deps/FalkorDB-rs",
+]

Makefile

@@ -137,9 +137,13 @@ REDISEARCH_DIR = $(ROOT)/deps/RediSearch
 export REDISEARCH_BINROOT=$(BINROOT)
 include $(ROOT)/build/RediSearch/Makefile.defs
 
+FalkorDBRS_DIR = $(ROOT)/deps/FalkorDB-rs
+export FalkorDBRS_BINDIR=$(BINROOT)/FalkorDB-rs
+include $(ROOT)/build/FalkorDB-rs/Makefile.defs
+
 BIN_DIRS += $(REDISEARCH_BINROOT)/search-static
 
-LIBS=$(RAX) $(LIBXXHASH) $(GRAPHBLAS) $(REDISEARCH_LIBS) $(LIBCYPHER_PARSER) $(UTF8PROC) $(ONIGURUMA)
+LIBS=$(RAX) $(LIBXXHASH) $(GRAPHBLAS) $(REDISEARCH_LIBS) $(LIBCYPHER_PARSER) $(UTF8PROC) $(ONIGURUMA) $(FalkorDBRS)
 
 #----------------------------------------------------------------------------------------------
 
@@ -189,11 +193,13 @@ ifneq ($(call files_missing,$(REDISEARCH_LIBS)),)
 MISSING_DEPS += $(REDISEARCH_LIBS)
 endif
 
+MISSING_DEPS += falkordbrs
+
 ifneq ($(MISSING_DEPS),)
 DEPS=1
 endif
 
-DEPENDENCIES=libcypher-parser graphblas redisearch rax libxxhash utf8proc oniguruma
+DEPENDENCIES=libcypher-parser graphblas redisearch rax libxxhash utf8proc oniguruma falkordbrs
 
 ifneq ($(filter all deps $(DEPENDENCIES) pack,$(MAKECMDGOALS)),)
 DEPS=1
@@ -219,7 +225,7 @@ include $(MK)/rules
 
 ifeq ($(DEPS),1)
 
-deps: $(LIBCYPHER_PARSER) $(GRAPHBLAS) $(LIBXXHASH) $(RAX) $(REDISEARCH_LIBS) $(UTF8PROC) $(ONIGURUMA)
+deps: $(LIBCYPHER_PARSER) $(GRAPHBLAS) $(LIBXXHASH) $(RAX) $(REDISEARCH_LIBS) $(UTF8PROC) $(ONIGURUMA) falkordbrs
 
 libxxhash: $(LIBXXHASH)
 
@@ -265,7 +271,21 @@ $(REDISEARCH_LIBS):
 	@echo Building $@ ...
 	$(SHOW)$(MAKE) -C $(REDISEARCH_DIR) STATIC=1 BINROOT=$(REDISEARCH_BINROOT) CC=$(CC) CXX=$(CXX)
 
-.PHONY: libcypher-parser graphblas redisearch libxxhash rax utf8proc oniguruma
+
+ifneq ($(DEBUG),1)
+CARGO_FLAGS=--release
+endif
+
+ifneq ($(SAN),)
+export RUSTFLAGS=-Zsanitizer=$(SAN)
+CARGO_FLAGS=--target x86_64-unknown-linux-gnu
+endif
+
+falkordbrs:
+	@echo Building $@ ...
+	cd deps/FalkorDB-rs && cargo build $(CARGO_FLAGS) --features falkordb_allocator --target-dir $(FalkorDBRS_BINDIR)
+
+.PHONY: libcypher-parser graphblas redisearch libxxhash rax utf8proc oniguruma falkordbrs
 
 #----------------------------------------------------------------------------------------------
 

build/FalkorDB-rs/Makefile.defs

@@ -0,0 +1,10 @@
+
+ifneq ($(SAN),)
+export FalkorDBRS=$(FalkorDBRS_BINDIR)/x86_64-unknown-linux-gnu/debug/libFalkorDB_rs.a
+else
+ifeq ($(DEBUG),1)
+export FalkorDBRS=$(FalkorDBRS_BINDIR)/debug/libFalkorDB_rs.a
+else
+export FalkorDBRS=$(FalkorDBRS_BINDIR)/release/libFalkorDB_rs.a
+endif
+endif

build/docker/Dockerfile.compiler

@@ -1,9 +1,9 @@
 ARG TARGETPLATFORM=linux/amd64
 
-FROM --platform=$TARGETPLATFORM falkordb/falkordb-build:latest as builder
+FROM --platform=$TARGETPLATFORM falkordb/falkordb-build:ubuntu as builder
 
 WORKDIR /FalkorDB
 
 COPY . /FalkorDB
 
-RUN make
+RUN make

demo/__init__.py

@@ -3,20 +3,18 @@ class QueryInfo(object):
     This class contains the needed data about a query
     """
 
-    def __init__(self, query=None, description=None, max_run_time_ms=None, expected_result=None, reversible=True):
+    def __init__(self, query=None, description=None, expected_result=None, reversible=True):
         """
         QueryInfo contructor
 
         :param query: The query itself (string)
         :param description: The information about what the query does (string)
-        :param max_run_time_ms: The max run time of the query in milliseconds (float)
         :param expected_result: The expected result of the query (list of lists, where the first list
                                 is the columns names, and the rest is the result)
         """
 
         self.query = query
         self.description = description
         self.expected_result = expected_result
-        self.max_run_time_ms = max_run_time_ms
         self.reversible = reversible