diff --git a/.github/wordlist.txt b/.github/wordlist.txt
index f5df9ca1..652a293a 100644
--- a/.github/wordlist.txt
+++ b/.github/wordlist.txt
@@ -64,3 +64,5 @@ vertx
wjso
wq
yadazula
+backport
+mitigations
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000..6ada7478
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,426 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+ test-recent-ubuntu:
+ runs-on: ${{ matrix.builder-os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ # jammy and bionic
+ builder-os: ['ubuntu-22.04', 'ubuntu-20.04']
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install build dependencies
+ run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool cmake lcov valgrind
+ - name: Setup Python for testing
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.9'
+ architecture: 'x64'
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run:
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: cd redis && make -j 4
+ - name: Build
+ run: make -j 4
+ - name: Run tests
+ run: make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ test-old-ubuntu:
+ runs-on: ubuntu-latest
+ container: ${{ matrix.builder-container }}
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ builder-container: ['ubuntu:xenial', 'ubuntu:bionic']
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install build dependencies
+ run: |
+ apt-get update && apt-get install -y software-properties-common
+ add-apt-repository ppa:git-core/ppa -y
+ apt-get update
+ apt-get install -y build-essential make autoconf automake libtool lcov git wget zlib1g-dev lsb-release libssl-dev openssl ca-certificates
+ wget https://cmake.org/files/v3.28/cmake-3.28.0.tar.gz
+ tar -xzvf cmake-3.28.0.tar.gz
+ cd cmake-3.28.0
+ ./configure
+ make -j `nproc`
+ make install
+ cd ..
+ ln -s /usr/local/bin/cmake /usr/bin/cmake
+ wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz
+ tar -xvf Python-3.9.6.tgz
+ cd Python-3.9.6
+ ./configure
+ make -j `nproc`
+ make altinstall
+ cd ..
+ rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3
+ rm /usr/bin/lsb_release
+ python3 --version
+ make --version
+ cmake --version
+ python3 -m pip install --upgrade pip
+ - name: Checkout RedisBloom
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run:
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: cd redis && make -j `nproc`
+ - name: Build RedisBloom
+ run: make -j `nproc`
+ - name: Run tests
+ run: make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ debian:
+ runs-on: ubuntu-latest
+ container: ${{ matrix.builder-container }}
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ builder-container: ['debian:bullseye']
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install build dependencies
+ run: |
+ apt-get update
+ apt-get install -y build-essential make cmake autoconf automake libtool lcov git wget zlib1g-dev lsb-release libssl-dev openssl ca-certificates python3 python3-pip
+ python3 --version
+ make --version
+ cmake --version
+ python3 -m pip install --upgrade pip
+ - name: Checkout RedisBloom
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run:
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: cd redis && make -j `nproc`
+ - name: Build RedisBloom
+ run: make -j `nproc`
+ - name: Run tests
+ run: make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ test-valgrind:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install build dependencies
+ run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool cmake lcov valgrind
+ - name: Setup Python for testing
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.9'
+ architecture: 'x64'
+ - name: Install Python dependencies
+ run:
+ python -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: cd redis && make valgrind -j 4
+ - name: Build
+ run: make -j 4
+ - name: Run tests
+ run: make test VG=1 REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ test-address-sanitizer:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["7.2.4", "unstable"]
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install build dependencies
+ run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool cmake lcov valgrind
+ - name: Setup Python for testing
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.9'
+ architecture: 'x64'
+ - name: Install Python dependencies
+ run:
+ python -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: cd redis && make SANITIZER=address -j 4
+ - name: Build
+ run: make -j 4
+ - name: Run tests
+ run: make test SAN=addr REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ centos:
+ runs-on: ubuntu-latest
+ container: ${{ matrix.builder-container }}
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.2.14", "7.2.4", "unstable"]
+ builder-container: ['centos:7']
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install build dependencies
+ run: |
+ yum -y install epel-release
+ yum -y install http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm
+ yum -y install gcc make cmake3 git python-pip openssl-devel bzip2-devel libffi-devel zlib-devel wget centos-release-scl scl-utils
+ yum groupinstall "Development Tools" -y
+ yum install -y devtoolset-11
+ . scl_source enable devtoolset-11 || true
+ make --version
+ gcc --version
+ git --version
+ wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz
+ tar -xvf Python-3.9.6.tgz
+ cd Python-3.9.6
+ ./configure
+ make -j `nproc`
+ make altinstall
+ cd ..
+ rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3
+ ln -s `which cmake3` /usr/bin/cmake
+ python3 --version
+ - name: Checkout sources
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run: |
+ scl enable devtoolset-11 bash
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: |
+ cd redis && make -j `nproc`
+ - name: Build RedisBloom
+ run: |
+ . scl_source enable devtoolset-11 || true
+ make -j `nproc`
+ - name: Run tests
+ run: |
+ . scl_source enable devtoolset-11 || true
+ make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ amazon-linux:
+ runs-on: ubuntu-latest
+ container: ${{ matrix.builder-container }}
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ builder-container: ['amazonlinux:2']
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install build dependencies
+ run: |
+ amazon-linux-extras install epel -y
+ yum -y install epel-release yum-utils
+ yum-config-manager --add-repo http://mirror.centos.org/centos/7/sclo/x86_64/rh/
+ yum -y install gcc make cmake3 git python-pip openssl-devel bzip2-devel libffi-devel zlib-devel wget centos-release-scl scl-utils which tar
+ yum -y install devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-make --nogpgcheck
+ . scl_source enable devtoolset-11 || true
+ make --version
+ git --version
+ wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz
+ tar -xvf Python-3.9.6.tgz
+ cd Python-3.9.6
+ ./configure
+ make -j `nproc`
+ make altinstall
+ cd ..
+ rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3
+ ln -s `which cmake3` /usr/bin/cmake
+ python3 --version
+ - name: Checkout sources
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run: |
+ scl enable devtoolset-11 bash
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: |
+ cd redis && make -j `nproc`
+ - name: Build RedisBloom
+ run: |
+ . scl_source enable devtoolset-11 || true
+ make -j `nproc`
+ - name: Run tests
+ run: |
+ . scl_source enable devtoolset-11 || true
+ make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ rocky-linux:
+ runs-on: ubuntu-latest
+ container: ${{ matrix.builder-container }}
+ strategy:
+ fail-fast: false
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ builder-container: ['rockylinux:8', 'rockylinux:9']
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install build dependencies
+ run: |
+ yum -y install epel-release
+ yum -y install http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm
+ yum -y install gcc make cmake3 git openssl-devel bzip2-devel libffi-devel zlib-devel wget scl-utils gcc-toolset-13 which
+ yum groupinstall "Development Tools" -y
+ . scl_source enable gcc-toolset-13 || true
+ make --version
+ gcc --version
+ git --version
+ wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz
+ tar -xvf Python-3.9.6.tgz
+ cd Python-3.9.6
+ ./configure
+ make -j `nproc`
+ make altinstall
+ cd ..
+ rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3
+ cmake --version
+ python3 --version
+ - name: Checkout sources
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run: |
+ . scl_source enable gcc-toolset-13 || true
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: |
+ cd redis && make -j `nproc`
+ - name: Build RedisBloom
+ run: |
+ . scl_source enable gcc-toolset-13 || true
+ make -j `nproc`
+ - name: Run tests
+ run: |
+ . scl_source enable gcc-toolset-13 || true
+ make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
+
+ macos-x86_64:
+ runs-on: macos-13
+ strategy:
+ fail-fast: false
+ # TODO: figure out the version we need to test on.
+ matrix:
+ redis-version: ["6.0.20", "7.2.4", "unstable"]
+ defaults:
+ run:
+ shell: bash -l -eo pipefail {0}
+ steps:
+ - name: Install prerequisites
+ run: |
+ brew install make
+ - name: Checkout sources
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Install Python dependencies
+ run: |
+ python3 -m pip install --upgrade pip setuptools wheel
+ python3 -m pip install -r tests/flow/requirements.txt
+ - name: Checkout Redis
+ uses: actions/checkout@v3
+ with:
+ repository: 'redis/redis'
+ ref: ${{ matrix.redis-version }}
+ path: 'redis'
+ - name: Build Redis
+ run: |
+ cd redis && make -j `sysctl -n hw.logicalcpu`
+ - name: Build RedisBloom
+ run: |
+ gmake -j `sysctl -n hw.logicalcpu`
+ - name: Run tests
+ run: |
+ gmake test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server
diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml
deleted file mode 100644
index f6286962..00000000
--- a/.github/workflows/sanitizer.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-name: CLang Sanitizer
-
-on:
- push:
- workflow_dispatch:
-
-jobs:
- clang-sanitizer:
- runs-on: ubuntu-latest
- defaults:
- run:
- shell: bash -l -eo pipefail {0}
- container:
- image: redisfab/clang:16-x64-bullseye
- options: --cpus 2
- steps:
- - uses: actions/checkout@v3
- with:
- submodules: true
- - name: Setup
- run: ./sbin/setup
- - name: Build
- run: make SAN=addr
- - name: Unit tests
- run: make unit-tests SAN=addr
- - name: Flow tests
- run: make flow-tests SAN=addr
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 00000000..d70acb19
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,42 @@
+# Security Policy
+
+## Supported Versions
+
+RedisBloom is generally backwards compatible with very few exceptions, so we
+recommend users to always use the latest version to experience stability,
+performance and security.
+
+We generally backport security issues to a single previous major version,
+unless this is not possible or feasible with a reasonable effort.
+
+| Version | Supported |
+| ------- | ------------------ |
+| 2.6 | :white_check_mark: |
+| 2.4 | :white_check_mark: |
+| < 2.4 | :x: |
+
+## Reporting a Vulnerability
+
+If you believe you've discovered a serious vulnerability, please contact the
+Redis core team at redis@redis.io. We will evaluate your report and if
+necessary issue a fix and an advisory. If the issue was previously undisclosed,
+we'll also mention your name in the credits.
+
+## Responsible Disclosure
+
+In some cases, we may apply a responsible disclosure process to reported or
+otherwise discovered vulnerabilities. We will usually do that for a critical
+vulnerability, and only if we have a good reason to believe information about
+it is not yet public.
+
+This process involves providing an early notification about the vulnerability,
+its impact and mitigations to a short list of vendors under a time-limited
+embargo on public disclosure.
+
+Vendors on the list are individuals or organizations that maintain Redis
+distributions or provide Redis as a service, who have third party users who
+will benefit from the vendor's ability to prepare for a new version or deploy a
+fix early.
+
+If you believe you should be on the list, please contact us and we will
+consider your request based on the above criteria.
diff --git a/deps/bloom/bloom.c b/deps/bloom/bloom.c
index 77040ea7..6d89cd90 100644
--- a/deps/bloom/bloom.c
+++ b/deps/bloom/bloom.c
@@ -42,6 +42,8 @@ extern void * (*RedisModule_Calloc)(size_t nmemb, size_t size);
#define MODE_READ 0
#define MODE_WRITE 1
+#define LN2 (0.693147180559945)
+
inline static int test_bit_set_bit(unsigned char *buf, uint64_t x, int mode) {
uint64_t byte = x >> 3;
uint8_t mask = 1 << (x % 8);
@@ -147,7 +149,13 @@ int bloom_init(struct bloom *bloom, uint64_t entries, double error, unsigned opt
} else if (options & BLOOM_OPT_NOROUND) {
// Don't perform any rounding. Conserve memory instead
- bits = bloom->bits = (uint64_t)(entries * bloom->bpe);
+ bits = (uint64_t)(entries * bloom->bpe);
+
+ // Guard against very small 'bpe'. Have at least one bit in the filter.
+ if (bits == 0) {
+ bits = 1;
+ }
+ bloom->bits = bits;
bloom->n2 = 0;
} else {
@@ -176,7 +184,7 @@ int bloom_init(struct bloom *bloom, uint64_t entries, double error, unsigned opt
bloom->bits = bloom->bytes * 8;
bloom->force64 = (options & BLOOM_OPT_FORCE64);
- bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2)
+ bloom->hashes = (int)ceil(LN2 * bloom->bpe); // ln(2)
bloom->bf = (unsigned char *)BLOOM_CALLOC(bloom->bytes, sizeof(unsigned char));
if (bloom->bf == NULL) {
return 1;
@@ -220,3 +228,15 @@ int bloom_add(struct bloom *bloom, const void *buffer, int len) {
void bloom_free(struct bloom *bloom) { BLOOM_FREE(bloom->bf); }
const char *bloom_version() { return MAKESTRING(BLOOM_VERSION); }
+
+// Returns 0 on success
+int bloom_validate_integrity(struct bloom *bloom) {
+ if (bloom->error <= 0 || bloom->error >= 1.0 ||
+ (bloom->n2 != 0 && bloom->bits < (1ULL << bloom->n2)) ||
+ bloom->bits == 0 || bloom->bits != bloom->bytes * 8 ||
+ bloom->hashes != (int)ceil(LN2 * bloom->bpe)) {
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/deps/bloom/bloom.h b/deps/bloom/bloom.h
index e3001a56..a6c5750d 100644
--- a/deps/bloom/bloom.h
+++ b/deps/bloom/bloom.h
@@ -160,6 +160,13 @@ void bloom_free(struct bloom *bloom);
*/
const char *bloom_version();
+/**
+ * Validates filter state
+ *
+ * Return: 0 on success
+ */
+int bloom_validate_integrity(struct bloom *bloom);
+
#ifdef __cplusplus
}
#endif
diff --git a/docs/commands/cf.reserve.md b/docs/commands/cf.reserve.md
index 75936340..9f2adec4 100644
--- a/docs/commands/cf.reserve.md
+++ b/docs/commands/cf.reserve.md
@@ -24,25 +24,40 @@ is key name for the the cuckoo filter to be created.
capacity
-Estimated capacity for the filter. Capacity is rounded to the next `2^n` number. The filter will likely not fill up to 100% of it's capacity.
-Make sure to reserve extra capacity if you want to avoid expansions.
+Estimated capacity for the filter.
+
+Capacity is rounded to the next `2^n` number.
+
+The filter will likely not fill up to 100% of it's capacity. Make sure to reserve extra capacity if you want to avoid expansions.
## Optional arguments
BUCKETSIZE bucketsize
-Number of items in each bucket. A higher bucket size value improves the fill rate but also causes a higher error rate and slightly slower performance. The default value is 2.
+Number of items in each bucket.
+
+A higher bucket size value improves the fill rate but also causes a higher error rate and slightly slower performance.
+
+`bucketsize` is an integer between 1 and 255. The default value is 2.
MAXITERATIONS maxiterations
-Number of attempts to swap items between buckets before declaring filter as full and creating an additional filter. A low value is better for performance and a higher number is better for filter fill rate. The default value is 20.
+Number of attempts to swap items between buckets before declaring filter as full and creating an additional filter.
+
+A low value is better for performance and a higher number is better for filter fill rate.
+
+`maxiterations` is an integer between 1 and 65535. The default value is 20.
EXPANSION expansion
-When a new filter is created, its size is the size of the current filter multiplied by `expansion`, specified as a non-negative integer. Expansion is rounded to the next `2^n` number. The default value is `1`.
+When a new filter is created, its size is the size of the current filter multiplied by `expansion`.
+
+`expansion` is an integer between 0 and 32768. The default value is 1.
+
+Expansion is rounded to the next `2^n` number.
## Return value
diff --git a/sbin/memcheck-summary b/sbin/memcheck-summary
index ed884823..752b5ecd 100755
--- a/sbin/memcheck-summary
+++ b/sbin/memcheck-summary
@@ -11,79 +11,55 @@ cd $HERE
#----------------------------------------------------------------------------------------------
valgrind_check() {
- echo -n "${NOCOLOR}"
- if grep -l "$1" $logdir/*.valgrind.log &> /dev/null; then
- echo
- echo "${LIGHTRED}### Valgrind: ${TYPE} detected:${RED}"
- grep -l "$1" $logdir/*.valgrind.log
- echo -n "${NOCOLOR}"
- E=1
- fi
-}
-
-valgrind_summary() {
local logdir="$ROOT/tests/$DIR/logs"
- local leaks_head=0
- for file in $(ls $logdir/*.valgrind.log 2>/dev/null); do
- # If the last "definitely lost: " line of a logfile has a nonzero value, print the file name
- if tac "$file" | grep -a -m 1 "definitely lost: " | grep "definitely lost: [1-9][0-9,]* bytes" &> /dev/null; then
- if [[ $leaks_head == 0 ]]; then
- echo
- echo "${LIGHTRED}### Valgrind: leaks detected:${RED}"
- leaks_head=1
- fi
- echo "$file"
+ for file in "$logdir"/*.valgrind.log; do
+ if grep -q -e "definitely lost" -e "0x" -e "Invalid" -e "Mismatched" \
+ -e "uninitialized" -e "has a fishy" -e "overlap" "$file"; then
+ echo
+ echo "### Valgrind error in $file:"
+ cat "$file"
E=1
fi
done
-
- TYPE="invalid reads" valgrind_check "Invalid read"
- TYPE="invalid writes" valgrind_check "Invalid write"
}
#----------------------------------------------------------------------------------------------
sanitizer_check() {
- if grep -l "$1" $logdir/*.asan.log* &> /dev/null; then
- echo
- echo "${LIGHTRED}### Sanitizer: ${TYPE} detected:${RED}"
- grep -l "$1" $logdir/*.asan.log*
- echo "${NOCOLOR}"
- E=1
- fi
-}
-
-sanitizer_summary() {
local logdir="$ROOT/tests/$DIR/logs"
- if ! TYPE="leaks" sanitizer_check "Direct leak"; then
- TYPE="leaks" sanitizer_check "detected memory leaks"
- fi
- TYPE="buffer overflow" sanitizer_check "dynamic-stack-buffer-overflow"
- TYPE="memory errors" sanitizer_check "memcpy-param-overlap"
- TYPE="stack use after scope" sanitizer_check "stack-use-after-scope"
- TYPE="use after free" sanitizer_check "heap-use-after-free"
+
+ for file in "$logdir"/*.asan.log*; do
+ if grep -q -e "runtime error" -e "Sanitizer" "$file"; then
+ echo
+ echo "### Sanitizer error in $file:"
+ cat "$file"
+ E=1
+ fi
+ done
}
+
+
#----------------------------------------------------------------------------------------------
E=0
DIRS=
-#if [[ $UNIT == 1 ]]; then
-# DIRS+=" ctests cpptests"
-#fi
+if [[ $UNIT == 1 ]]; then
+ DIRS+=" unit"
+fi
if [[ $FLOW == 1 ]]; then
- DIRS+=" pytests"
+ DIRS+=" flow"
fi
if [[ $VG == 1 ]]; then
for dir in $DIRS; do
- DIR="$dir" valgrind_summary
+ DIR="$dir" valgrind_check
done
elif [[ -n $SAN ]]; then
for dir in $DIRS; do
- DIR="$dir" sanitizer_summary
+ DIR="$dir" sanitizer_check
done
fi
diff --git a/src/cf.c b/src/cf.c
index ca7878c5..a9e82a3f 100644
--- a/src/cf.c
+++ b/src/cf.c
@@ -84,7 +84,7 @@ const char *CF_GetEncodedChunk(const CuckooFilter *cf, long long *pos, size_t *b
}
int CF_LoadEncodedChunk(const CuckooFilter *cf, long long pos, const char *data, size_t datalen) {
- if (datalen == 0) {
+ if (datalen == 0 || pos <= 0 || (size_t)(pos - 1) < datalen) {
return REDISMODULE_ERR;
}
@@ -102,6 +102,12 @@ int CF_LoadEncodedChunk(const CuckooFilter *cf, long long pos, const char *data,
offset -= currentSize;
}
+ // Boundary check before memcpy()
+ if (!filter || ((size_t)offset > SIZE_MAX - datalen) ||
+ filter->bucketSize * filter->numBuckets < offset + datalen) {
+ return REDISMODULE_ERR;
+ }
+
// copy data to filter
memcpy(filter->data + offset, data, datalen);
return REDISMODULE_OK;
@@ -116,7 +122,12 @@ CuckooFilter *CFHeader_Load(const CFHeader *header) {
filter->bucketSize = header->bucketSize;
filter->maxIterations = header->maxIterations;
filter->expansion = header->expansion;
- filter->filters = RedisModule_Alloc(sizeof(*filter->filters) * header->numFilters);
+ filter->filters = RedisModule_Calloc(sizeof(*filter->filters), filter->numFilters);
+
+ if (CuckooFilter_ValidateIntegrity(filter) != 0) {
+ goto error;
+ }
+
for (size_t ii = 0; ii < filter->numFilters; ++ii) {
SubCF *cur = filter->filters + ii;
cur->bucketSize = header->bucketSize;
@@ -125,6 +136,11 @@ CuckooFilter *CFHeader_Load(const CFHeader *header) {
RedisModule_Calloc((size_t)cur->numBuckets * filter->bucketSize, sizeof(CuckooBucket));
}
return filter;
+
+error:
+ CuckooFilter_Free(filter);
+ RedisModule_Free(filter);
+ return NULL;
}
void fillCFHeader(CFHeader *header, const CuckooFilter *cf) {
diff --git a/src/cuckoo.c b/src/cuckoo.c
index 15cb7803..ba52b49b 100644
--- a/src/cuckoo.c
+++ b/src/cuckoo.c
@@ -397,3 +397,15 @@ void CuckooFilter_GetInfo(const CuckooFilter *cf, CuckooHash hash, CuckooKey *ou
assert(getAltHash(params.fp, out->h1, cf->numBuckets) == out->h2);
assert(getAltHash(params.fp, out->h2, cf->numBuckets) == out->h1);
}*/
+
+// Returns 0 on success
+int CuckooFilter_ValidateIntegrity(const CuckooFilter *cf) {
+ if (cf->bucketSize == 0 || cf->bucketSize > CF_MAX_BUCKET_SIZE ||
+ cf->numBuckets == 0 || cf->numBuckets > CF_MAX_NUM_BUCKETS ||
+ cf->numFilters == 0 || cf->numFilters > CF_MAX_NUM_FILTERS ||
+ cf->maxIterations == 0 || !isPower2(cf->numBuckets) ) {
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/src/cuckoo.h b/src/cuckoo.h
index 529b15f9..68d34367 100644
--- a/src/cuckoo.h
+++ b/src/cuckoo.h
@@ -24,6 +24,15 @@ typedef uint64_t CuckooHash;
typedef uint8_t CuckooBucket[1];
typedef uint8_t MyCuckooBucket;
+#define CF_DEFAULT_MAX_ITERATIONS 20
+#define CF_DEFAULT_BUCKETSIZE 2
+#define CF_DEFAULT_EXPANSION 1
+#define CF_MAX_EXPANSION 32768
+#define CF_MAX_ITERATIONS 65535
+#define CF_MAX_BUCKET_SIZE 255 // 8 bits, see struct SubCF
+#define CF_MAX_NUM_BUCKETS (0x00FFFFFFFFFFFFFFULL) // 56 bits, see struct SubCF
+#define CF_MAX_NUM_FILTERS (UINT16_MAX) // 16 bits, see struct CuckooFilter
+
typedef struct {
uint64_t numBuckets : 56;
uint64_t bucketSize : 8;
@@ -72,3 +81,4 @@ int CuckooFilter_Check(const CuckooFilter *filter, CuckooHash hash);
uint64_t CuckooFilter_Count(const CuckooFilter *filter, CuckooHash);
void CuckooFilter_Compact(CuckooFilter *filter, bool cont);
void CuckooFilter_GetInfo(const CuckooFilter *cf, CuckooHash hash, CuckooKey *out);
+int CuckooFilter_ValidateIntegrity(const CuckooFilter *cf);
diff --git a/src/rebloom.c b/src/rebloom.c
index 8a4fd429..f876087e 100644
--- a/src/rebloom.c
+++ b/src/rebloom.c
@@ -19,14 +19,12 @@
#include // strncasecmp
#include
#include
+#include
#ifndef REDISBLOOM_GIT_SHA
#define REDISBLOOM_GIT_SHA "unknown"
#endif
-#define CF_MAX_ITERATIONS 20
-#define CF_DEFAULT_BUCKETSIZE 2
-#define CF_DEFAULT_EXPANSION 1
#define BF_DEFAULT_EXPANSION 2
////////////////////////////////////////////////////////////////////////////////
@@ -107,8 +105,8 @@ static SBChain *bfCreateChain(RedisModuleKey *key, double error_rate, size_t cap
return sb;
}
-static CuckooFilter *cfCreate(RedisModuleKey *key, size_t capacity, size_t bucketSize,
- size_t maxIterations, size_t expansion) {
+static CuckooFilter *cfCreate(RedisModuleKey *key, size_t capacity, uint16_t bucketSize,
+ uint16_t maxIterations, uint16_t expansion) {
if (capacity < bucketSize * 2)
return NULL;
@@ -418,8 +416,10 @@ static int BFDebug_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, i
for (size_t ii = 0; ii < sb->nfilters; ++ii) {
const SBLink *lb = sb->filters + ii;
info_s = RedisModule_CreateStringPrintf(
- ctx, "bytes:%lu bits:%llu hashes:%u hashwidth:%u capacity:%lu size:%lu ratio:%g",
- lb->inner.bytes, lb->inner.bits ? lb->inner.bits : 1LLU << lb->inner.n2,
+ ctx,
+ "bytes:%" PRIu64 " bits:%" PRIu64 " hashes:%u hashwidth:%u capacity:%" PRIu64
+ " size:%zu ratio:%g",
+ lb->inner.bytes, lb->inner.bits ? lb->inner.bits : UINT64_C(1) << lb->inner.n2,
lb->inner.hashes, sb->options & BLOOM_OPT_FORCE64 ? 64 : 32, lb->inner.entries,
lb->size, lb->inner.error);
RedisModule_ReplyWithString(ctx, info_s);
@@ -529,14 +529,14 @@ static int CFReserve_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
return RedisModule_ReplyWithError(ctx, "Bad capacity");
}
- long long maxIterations = CF_MAX_ITERATIONS;
+ long long maxIterations = CF_DEFAULT_MAX_ITERATIONS;
int mi_loc = RMUtil_ArgIndex("MAXITERATIONS", argv, argc);
if (mi_loc != -1) {
if (RedisModule_StringToLongLong(argv[mi_loc + 1], &maxIterations) != REDISMODULE_OK) {
return RedisModule_ReplyWithError(ctx, "Couldn't parse MAXITERATIONS");
- } else if (maxIterations <= 0) {
+ } else if (maxIterations <= 0 || maxIterations > CF_MAX_ITERATIONS) {
return RedisModule_ReplyWithError(
- ctx, "MAXITERATIONS parameter needs to be a positive integer");
+ ctx, "MAXITERATIONS: value must be an integer between 1 and 65535, inclusive.");
}
}
@@ -545,9 +545,9 @@ static int CFReserve_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
if (bs_loc != -1) {
if (RedisModule_StringToLongLong(argv[bs_loc + 1], &bucketSize) != REDISMODULE_OK) {
return RedisModule_ReplyWithError(ctx, "Couldn't parse BUCKETSIZE");
- } else if (bucketSize <= 0) {
+ } else if (bucketSize <= 0 || bucketSize > CF_MAX_BUCKET_SIZE) {
return RedisModule_ReplyWithError(
- ctx, "BUCKETSIZE parameter needs to be a positive integer");
+ ctx, "BUCKETSIZE: value must be an integer between 1 and 255, inclusive.");
}
}
@@ -556,9 +556,9 @@ static int CFReserve_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
if (ex_loc != -1) {
if (RedisModule_StringToLongLong(argv[ex_loc + 1], &expansion) != REDISMODULE_OK) {
return RedisModule_ReplyWithError(ctx, "Couldn't parse EXPANSION");
- } else if (expansion < 0) {
+ } else if (expansion < 0 || expansion > CF_MAX_EXPANSION) {
return RedisModule_ReplyWithError(
- ctx, "EXPANSION parameter needs to be a non-negative integer");
+ ctx, "EXPANSION: value must be an integer between 0 and 32768, inclusive.");
}
}
@@ -596,7 +596,7 @@ static int cfInsertCommon(RedisModuleCtx *ctx, RedisModuleString *keystr, RedisM
int status = cfGetFilter(key, &cf);
if (status == SB_EMPTY && options->autocreate) {
- if ((cf = cfCreate(key, options->capacity, CF_DEFAULT_BUCKETSIZE, CF_MAX_ITERATIONS,
+ if ((cf = cfCreate(key, options->capacity, CF_DEFAULT_BUCKETSIZE, CF_DEFAULT_MAX_ITERATIONS,
CF_DEFAULT_EXPANSION)) == NULL) {
return RedisModule_ReplyWithError(ctx, "Could not create filter"); // LCOV_EXCL_LINE
}
@@ -850,7 +850,7 @@ static int CFScanDump_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv
}
long long pos;
- if (RedisModule_StringToLongLong(argv[2], &pos) != REDISMODULE_OK) {
+ if (RedisModule_StringToLongLong(argv[2], &pos) != REDISMODULE_OK || pos < 0) {
return RedisModule_ReplyWithError(ctx, "Invalid position");
}
@@ -1092,7 +1092,8 @@ static int CFDebug_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, i
RedisModuleString *resp = RedisModule_CreateStringPrintf(
ctx,
- "bktsize:%u buckets:%lu items:%lu deletes:%lu filters:%u max_iterations:%u expansion:%u",
+ "bktsize:%u buckets:%" PRIu64 " items:%" PRIu64 " deletes:%" PRIu64
+ " filters:%u max_iterations:%u expansion:%u",
cf->bucketSize, cf->numBuckets, cf->numItems, cf->numDeletes, cf->numFilters,
cf->maxIterations, cf->expansion);
return RedisModule_ReplyWithString(ctx, resp);
@@ -1252,7 +1253,7 @@ static void *CFRdbLoad(RedisModuleIO *io, int encver) {
if (encver < CF_MIN_EXPANSION_VERSION) { // CF_ENCODING_VERSION when added
cf->numDeletes = 0; // Didn't exist earlier. bug fix
cf->bucketSize = CF_DEFAULT_BUCKETSIZE;
- cf->maxIterations = CF_MAX_ITERATIONS;
+ cf->maxIterations = CF_DEFAULT_MAX_ITERATIONS;
cf->expansion = CF_DEFAULT_EXPANSION;
} else {
cf->numDeletes = RedisModule_LoadUnsigned(io);
diff --git a/src/sb.c b/src/sb.c
index ddfe30bc..fb6ac7f8 100644
--- a/src/sb.c
+++ b/src/sb.c
@@ -26,15 +26,14 @@ bloom_hashval bloom_calc_hash64(const void *buffer, int len);
#define CUR_FILTER(sb) ((sb)->filters + ((sb)->nfilters - 1))
static int SBChain_AddLink(SBChain *chain, uint64_t size, double error_rate) {
- if (!chain->filters) {
- chain->filters = RedisModule_Calloc(1, sizeof(*chain->filters));
- } else {
- chain->filters =
- RedisModule_Realloc(chain->filters, sizeof(*chain->filters) * (chain->nfilters + 1));
- }
+ chain->filters =
+ RedisModule_Realloc(chain->filters, sizeof(*chain->filters) * (chain->nfilters + 1));
SBLink *newlink = chain->filters + chain->nfilters;
- newlink->size = 0;
+ *newlink = (SBLink){
+ .size = 0,
+ };
+
chain->nfilters++;
return bloom_init(&newlink->inner, size, error_rate, chain->options);
}
@@ -150,7 +149,9 @@ typedef struct __attribute__((packed)) {
} dumpedChainHeader;
static SBLink *getLinkPos(const SBChain *sb, long long curIter, size_t *offset) {
- // printf("Requested %lld\n", curIter);
+ if (curIter < 1) {
+ return NULL;
+ }
curIter--;
SBLink *link = NULL;
@@ -218,6 +219,28 @@ char *SBChain_GetEncodedHeader(const SBChain *sb, size_t *hdrlen) {
void SB_FreeEncodedHeader(char *s) { RedisModule_Free(s); }
+// Returns 0 on success
+int SB_ValidateIntegrity(const SBChain *sb) {
+ if (sb->options &
+ ~(BLOOM_OPT_NOROUND | BLOOM_OPT_ENTS_IS_BITS | BLOOM_OPT_FORCE64 | BLOOM_OPT_NO_SCALING)) {
+ return 1;
+ }
+
+ size_t total = 0;
+ for (size_t i = 0; i < sb->nfilters; i++) {
+ if (sb->filters[i].size > SIZE_MAX - total) {
+ return 1;
+ }
+ total += sb->filters[i].size;
+ }
+
+ if (sb->size != total) {
+ return 1;
+ }
+
+ return 0;
+}
+
SBChain *SB_NewChainFromHeader(const char *buf, size_t bufLen, const char **errmsg) {
const dumpedChainHeader *header = (const void *)buf;
if (bufLen < sizeof(dumpedChainHeader)) {
@@ -243,17 +266,34 @@ SBChain *SB_NewChainFromHeader(const char *buf, size_t bufLen, const char **errm
#define X(encfld, dstfld) dstfld = encfld;
X_ENCODED_LINK(X, srclink, dstlink)
#undef X
- dstlink->inner.bf = RedisModule_Alloc(dstlink->inner.bytes);
+
+ if (bloom_validate_integrity(&dstlink->inner) != 0) {
+ SBChain_Free(sb);
+ *errmsg = "ERR received bad data";
+ return NULL;
+ }
+
+ dstlink->inner.bf = RedisModule_Calloc(1, dstlink->inner.bytes);
if (sb->options & BLOOM_OPT_FORCE64) {
dstlink->inner.force64 = 1;
}
}
+ if (SB_ValidateIntegrity(sb) != 0) {
+ SBChain_Free(sb);
+ *errmsg = "ERR received bad data";
+ return NULL;
+ }
+
return sb;
}
int SBChain_LoadEncodedChunk(SBChain *sb, long long iter, const char *buf, size_t bufLen,
const char **errmsg) {
+ if (!buf || iter <= 0 || iter < bufLen) {
+ *errmsg = "ERR received bad data";
+ return -1;
+ }
// Load the chunk
size_t offset;
iter -= bufLen;
diff --git a/src/sb.h b/src/sb.h
index 75db51b9..c3b180c5 100644
--- a/src/sb.h
+++ b/src/sb.h
@@ -104,6 +104,9 @@ SBChain *SB_NewChainFromHeader(const char *buf, size_t bufLen, const char **errm
*/
int SBChain_LoadEncodedChunk(SBChain *sb, long long iter, const char *buf, size_t bufLen,
const char **errmsg);
+
+int SB_ValidateIntegrity(const SBChain *sb);
+
#ifdef __cplusplus
}
#endif
diff --git a/tests/flow/requirements.txt b/tests/flow/requirements.txt
index 899f8acd..56d4f282 100644
--- a/tests/flow/requirements.txt
+++ b/tests/flow/requirements.txt
@@ -1,2 +1,2 @@
-RLTest ~= 0.7.2
+RLTest == 0.7.5
numpy
diff --git a/tests/flow/test_cuckoo.py b/tests/flow/test_cuckoo.py
index b5041383..1d193693 100644
--- a/tests/flow/test_cuckoo.py
+++ b/tests/flow/test_cuckoo.py
@@ -1,3 +1,4 @@
+import random
from common import *
@@ -346,6 +347,19 @@ def test_params(self):
self.assertRaises(ResponseError, self.cmd, 'CF.LOADCHUNK err iterator') # missing data
self.assertRaises(ResponseError, self.cmd, 'CF.SCANDUMP err')
+ def test_reserve_limits(self):
+ self.cmd('FLUSHALL')
+ self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE cf 100 BUCKETSIZE 33554432')
+ self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE cf 100 MAXITERATIONS 165536')
+ self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE cf 100 EXPANSION 327695')
+ self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE CF 67108864 BUCKETSIZE 33554432 MAXITERATIONS 1337 EXPANSION 1337')
+
+ self.cmd('CF.RESERVE cf 67108864 BUCKETSIZE 255 MAXITERATIONS 65535 EXPANSION 32768')
+ info = self.cmd('CF.INFO cf')
+ self.assertEqual(info[info.index('Bucket size') + 1], 255)
+ self.assertEqual(info[info.index('Expansion rate') + 1], 32768)
+ self.assertEqual(info[info.index('Max iterations') + 1], 65535)
+
class testCuckooNoCodec():
def __init__(self):
self.env = Env(decodeResponses=False)
@@ -454,3 +468,169 @@ def test_scandump_huge(self):
# check loaded filter
for x in range(6):
self.assertEqual(1, self.cmd('cf.exists', 'cf', 'foo'))
+
+ def test_scandump_with_content(self):
+ # Basic success scenario with content validation
+
+ self.cmd('FLUSHALL')
+ self.cmd('cf.reserve', 'cf', 1024 * 1024 * 64)
+
+ for x in range(1000):
+ self.cmd('cf.add', 'cf', 'foo' + str(x))
+ for x in range(1000):
+ self.assertEqual(1, self.cmd('cf.exists', 'cf', 'foo' + str(x)))
+
+ chunks = []
+ while True:
+ last_pos = chunks[-1][0] if chunks else 0
+ chunk = self.cmd('cf.scandump', 'cf', last_pos)
+ if not chunk[0]:
+ break
+ chunks.append(chunk)
+
+ for chunk in chunks:
+ self.cmd('cf.loadchunk', 'cf2', *chunk)
+
+ # check loaded filter
+ for x in range(1000):
+ self.assertEqual(1, self.cmd('cf.exists', 'cf2', 'foo' + str(x)))
+
+
+ def test_scandump_invalid(self):
+ self.cmd('FLUSHALL')
+ self.cmd('cf.reserve', 'cf', 4)
+ self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '-9223372036854775808', '1')
+ self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '922337203685477588', '1')
+ self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '4', 'kdoasdksaodsadsadsadsadsadadsadadsdad')
+ self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '4', 'abcd')
+ self.cmd('cf.add', 'cf', 'x')
+ self.assertRaises(ResponseError, self.cmd, 'cf.scandump', 'cf', '-1')
+
+
+ def test_scandump_invalid_header(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+
+ env.cmd('cf.reserve', 'cf', 100)
+ for x in range(50):
+ env.cmd('cf.add', 'cf', 'foo' + str(x))
+
+ chunk = env.cmd('cf.scandump', 'cf', 0)
+ env.cmd('del', 'cf')
+
+ arr = bytearray(chunk[1])
+
+ # It corrupts first 8 bytes in the response. See struct CFHeader
+ # for internals.
+ for i in range(9):
+ arr[i] = 0
+
+ thrown = None
+ try:
+ env.cmd('cf.loadchunk', 'cf', 1, bytes(arr))
+ except Exception as e:
+ thrown = e
+
+ if thrown is None or str(thrown) != "Couldn't create filter!":
+ print("Exception was: " + str(thrown))
+ assert False
+
+
+ def test_scandump_random_scan_small(self):
+ self.cmd('FLUSHALL')
+ self.cmd('cf.reserve', 'cf', 50)
+
+ for i in range(0, 10000):
+ try:
+ self.cmd('cf.add', 'cf', 'x' + str(i))
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = self.cmd('CF.INFO', 'cf')
+ size = info[info.index(b'Size') + 1]
+
+ for i in range(0, size + 1024):
+ self.cmd('cf.scandump', 'cf', i)
+
+
+ def test_scandump_scan_big(self):
+ self.cmd('FLUSHALL')
+ self.cmd('cf.reserve', 'cf', 1024, 'EXPANSION', 30000)
+
+ for i in range(0, 100):
+ arr = []
+ for j in range(0, 10000):
+ arr.append('x' + str(i) + str(j))
+
+ try:
+ self.cmd('cf.insert', 'cf', 'ITEMS', *arr)
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = self.cmd('CF.INFO', 'cf')
+ size = info[info.index(b'Size') + 1]
+
+ for i in range(0, 100):
+ self.cmd('cf.scandump', 'cf', random.randint(0, size * 2))
+
+
+ def test_scandump_load_small(self):
+ self.cmd('FLUSHALL')
+ self.cmd('cf.reserve', 'cf', 10)
+
+ for i in range(0, 100):
+ arr = []
+ for j in range(0, 1000):
+ arr.append('x' + str(i) + str(j))
+
+ try:
+ self.cmd('cf.insert', 'cf', 'ITEMS', *arr)
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = self.cmd('CF.INFO', 'cf')
+ size = info[info.index(b'Size') + 1]
+
+ for i in range (0, size + 100):
+ b = bytearray(os.urandom(random.randint(0, 100)))
+ try:
+ self.cmd('cf.loadchunk', 'cf', random.randint(0, 10000), bytes(b))
+ except Exception as e:
+ if (str(e) != "Couldn't load chunk!" and
+ str(e) != "Invalid position" and
+ str(e) != "item exists"):
+ raise e
+
+
+ def test_scandump_load_big(self):
+ self.cmd('FLUSHALL')
+ self.cmd('cf.reserve', 'cf', 1024, 'EXPANSION', 30000)
+
+ for i in range(0, 100):
+ arr = []
+ for j in range(0, 1000):
+ arr.append('x' + str(i) + str(j))
+
+ try:
+ self.cmd('cf.insert', 'cf', 'ITEMS', *arr)
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = self.cmd('CF.INFO', 'cf')
+ size = info[info.index(b'Size') + 1]
+
+ for i in range (0, 100):
+ b = bytearray(os.urandom(random.randint(1024, 36 * 1024 * 1024)))
+ try:
+ self.cmd('cf.loadchunk', 'cf', random.randint(2, size), bytes(b))
+ except Exception as e:
+ if str(e) != "Couldn't load chunk!":
+ raise e
diff --git a/tests/flow/test_overall.py b/tests/flow/test_overall.py
index d29f4e79..0f5ac741 100644
--- a/tests/flow/test_overall.py
+++ b/tests/flow/test_overall.py
@@ -1,3 +1,4 @@
+import random
from common import *
@@ -425,6 +426,20 @@ def test_issue178(self):
env.assertEqual(info["Capacity"], 300000000)
env.assertEqual(info["Size"], 1132420232)
+ def test_very_high_error_rate(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+
+ env.cmd('bf.reserve', 'bf1', 0.99, 3, "NONSCALING")
+ env.cmd('bf.add', 'bf1', 1)
+
+ env.cmd('bf.reserve', 'bf2', 0.95, 8, "NONSCALING")
+ env.cmd('bf.add', 'bf2', 1)
+
+ env.cmd('bf.reserve', 'bf3', 0.9999999999999999, 100, "NONSCALING")
+ env.cmd('bf.add', 'bf3', 1)
+
+
class testRedisBloomNoCodec():
def __init__(self):
self.env = Env(decodeResponses=False)
@@ -521,3 +536,225 @@ def test_scandump_huge(self):
# check loaded filter
for x in range(6):
env.assertEqual(1, env.cmd('bf.exists', 'bf', 'foo'))
+
+
+ def test_scandump_with_content(self):
+ # Basic success scenario with content validation
+
+ env = self.env
+ env.cmd('FLUSHALL')
+
+ env.cmd('bf.reserve', 'bf', 0.01, 1024 * 1024 * 64)
+ for x in range(1000):
+ env.cmd('bf.add', 'bf', 'foo' + str(x))
+ for x in range(1000):
+ env.assertEqual(1, env.cmd('bf.exists', 'bf', 'foo' + str(x)))
+
+ chunks = []
+ while True:
+ last_pos = chunks[-1][0] if chunks else 0
+ chunk = env.cmd('bf.scandump', 'bf', last_pos)
+ if not chunk[0]:
+ break
+ chunks.append(chunk)
+
+ env.cmd('del', 'bf')
+
+ for chunk in chunks:
+ env.cmd('bf.loadchunk', 'bf2', *chunk)
+
+ # Validate items in the loaded filter
+ for x in range(1000):
+ env.assertEqual(1, env.cmd('bf.exists', 'bf2', 'foo' + str(x)))
+
+
+ def test_scandump_invalid(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+ env.cmd('bf.reserve', 'bf', 0.1, 4)
+ env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '-9223372036854775808', '1')
+ env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '922337203685477588', '1')
+ env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '4', 'kdoasdksaodsadsadsadsadsadadsadadsdad')
+ env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '4', 'abcd')
+ env.cmd('bf.add', 'bf', 'x')
+ env.cmd('bf.add', 'bf', 'y')
+
+
+ def test_scandump_invalid_header(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+
+ env.cmd('bf.reserve', 'bf', 0.01, 5)
+ for x in range(50):
+ env.cmd('bf.add', 'bf', 'foo' + str(x))
+
+ chunk = env.cmd('bf.scandump', 'bf', 0)
+
+ env.cmd('del', 'bf')
+ arr = bytearray(chunk[1])
+
+ # See 'struct dumpedChainHeader' for internals.
+ # It corrupts second link in the response.
+ for i in range(8):
+ arr[72 + i] = 0
+
+ thrown = None
+ try:
+ env.cmd('bf.loadchunk', 'bf', 1, bytes(arr))
+ except Exception as e:
+ thrown = e
+
+ if thrown is None or str(thrown) != "received bad data":
+ raise thrown
+
+ # It corrupts 'options' field in the response.
+ arr = bytearray(chunk[1])
+ for i in range(4):
+ arr[12 + i] = 255
+
+ thrown = None
+ try:
+ env.cmd('bf.loadchunk', 'bf', 1, bytes(arr))
+ except Exception as e:
+ thrown = e
+
+ if thrown is None or str(thrown) != "received bad data":
+ raise thrown
+
+ # It corrupts first field in the response.
+ arr = bytearray(chunk[1])
+ for i in range(4):
+ arr[i] = 255
+
+ thrown = None
+ try:
+ env.cmd('bf.loadchunk', 'bf', 1, bytes(arr))
+ except Exception as e:
+ thrown = e
+
+ if thrown is None or str(thrown) != "received bad data":
+ raise thrown
+
+ # It corrupts second link in the response.
+ arr = bytearray(chunk[1])
+ for i in range(8):
+ arr[36 + i] = 255
+ arr[0 + i] = 255
+
+ thrown = None
+ try:
+ env.cmd('bf.loadchunk', 'bf', 1, bytes(arr))
+ except Exception as e:
+ thrown = e
+
+ if thrown is None or str(thrown) != "received bad data":
+ raise thrown
+
+
+ def test_scandump_scan_small(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+ env.cmd('bf.reserve', 'bf', 0.1, 50)
+
+ for i in range(0, 1500):
+ try:
+ env.cmd('bf.add', 'bf', 'x' + str(i))
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = env.cmd('BF.INFO', 'bf')
+ size = info[info.index(b'Size') + 1]
+
+ # Verify random scandump does not cause any problem
+ for i in range(0, size + 1024):
+ env.cmd('bf.scandump', 'bf', i)
+
+
+ def test_scandump_scan_big(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+ env.cmd('bf.reserve', 'bf', 0.001, 1024, 'EXPANSION', 30000)
+
+ for i in range(0, 100):
+ arr = []
+ for j in range(0, 10000):
+ arr.append('x' + str(i) + str(j))
+
+ try:
+ env.cmd('bf.insert', 'bf', 'ITEMS', *arr)
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = env.cmd('bf.INFO', 'bf')
+ size = info[info.index(b'Size') + 1]
+
+ # Verify random scandump does not cause any problem
+ for i in range(0, 100):
+ env.cmd('bf.scandump', 'bf', random.randint(0, size * 2))
+
+
+ def test_scandump_load_small(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+ env.cmd('bf.reserve', 'bf', 0.01, 10)
+
+ for i in range(0, 100):
+ arr = []
+ for j in range(0, 1000):
+ arr.append('x' + str(i) + str(j))
+
+ try:
+ env.cmd('bf.insert', 'bf', 'ITEMS', *arr)
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = env.cmd('BF.INFO', 'bf')
+ size = info[info.index(b'Size') + 1]
+
+ # Try loading chunks with random size and content
+ for i in range (0, 100):
+ b = bytearray(os.urandom(random.randint(0, 4096)))
+ try:
+ env.cmd('bf.loadchunk', 'bf', random.randint(0, size * 2), bytes(b))
+ except Exception as e:
+ if (str(e) != "invalid offset - no link found" and
+ str(e) != "invalid chunk - Too big for current filter" and
+ str(e) != "received bad data"):
+ raise e
+
+
+ def test_scandump_load_big(self):
+ env = self.env
+ env.cmd('FLUSHALL')
+ env.cmd('bf.reserve', 'bf', 0.01, 1024, 'EXPANSION', 30000)
+
+ for i in range(0, 100):
+ arr = []
+ for j in range(0, 1000):
+ arr.append('x' + str(i) + str(j))
+
+ try:
+ env.cmd('bf.insert', 'bf', 'ITEMS', *arr)
+ except ResponseError as e:
+ if str(e) == "Maximum expansions reached":
+ break
+ raise e
+
+ info = env.cmd('BF.INFO', 'bf')
+ size = info[info.index(b'Size') + 1]
+
+ # Try loading chunks with random size and content
+ for i in range (0, 100):
+ b = bytearray(os.urandom(random.randint(1024, 36 * 1024 * 1024)))
+ try:
+ env.cmd('bf.loadchunk', 'bf', random.randint(0, size), bytes(b))
+ except Exception as e:
+ if (str(e) != "invalid offset - no link found" and
+ str(e) != "received bad data"):
+ raise e
diff --git a/tests/flow/tests.sh b/tests/flow/tests.sh
index 3e8f874f..4173c926 100755
--- a/tests/flow/tests.sh
+++ b/tests/flow/tests.sh
@@ -18,9 +18,9 @@ cd $HERE
help() {
cat <<-'END'
Run flow tests
-
+
[ARGVARS...] tests.sh [--help|help] []
-
+
Argument variables:
MODULE=path Path to redisbloom.so
MODARGS=args RediSearch module arguments
@@ -32,7 +32,7 @@ help() {
SLAVES=1 Tests with --test-slaves
CLUSTER=1 Test with OSS cluster, one shard
QUICK=1 Perform only GEN=1 test variant
-
+
TEST=name Run specific test (e.g. test.py:test_name)
TESTFILE=file Run tests listed in `file`
FAILEDFILE=file Write failed tests into `file`
@@ -55,7 +55,7 @@ help() {
COV=1 Run with coverage analysis
VG=1 Run with Valgrind
VG_LEAKS=0 Do not detect leaks
- SAN=type Use LLVM sanitizer (type=address|memory|leak|thread)
+ SAN=type Use LLVM sanitizer (type=address|memory|leak|thread)
BB=1 Enable Python debugger (break using BB() in tests)
GDB=1 Enable interactive gdb debugging (in single-test mode)
@@ -159,42 +159,14 @@ setup_rltest() {
#----------------------------------------------------------------------------------------------
-setup_clang_sanitizer() {
- local ignorelist=$ROOT/tests/memcheck/redis.san-ignorelist
- if ! grep THPIsEnabled $ignorelist &> /dev/null; then
- echo "fun:THPIsEnabled" >> $ignorelist
- fi
-
- # for RediSearch module
- export RS_GLOBAL_DTORS=1
-
+setup_sanitizer() {
# for RLTest
export SANITIZER="$SAN"
export SHORT_READ_BYTES_DELTA=512
-
+
+ export ASAN_OPTIONS="detect_odr_violation=0:halt_on_error=0:detect_leaks=1"
# --no-output-catch --exit-on-failure --check-exitcode
RLTEST_SAN_ARGS="--sanitizer $SAN"
-
- if [[ $SAN == addr || $SAN == address ]]; then
- REDIS_SERVER=${REDIS_SERVER:-redis-server-asan-$SAN_REDIS_VER}
- if ! command -v $REDIS_SERVER > /dev/null; then
- echo Building Redis for clang-asan ...
- $READIES/bin/getredis --force -v $SAN_REDIS_VER --own-openssl --no-run \
- --suffix asan --clang-asan --clang-san-blacklist $ignorelist
- fi
-
- export ASAN_OPTIONS="detect_odr_violation=0:halt_on_error=0:detect_leaks=1"
- export LSAN_OPTIONS="suppressions=$ROOT/tests/memcheck/asan.supp"
-
- elif [[ $SAN == mem || $SAN == memory ]]; then
- REDIS_SERVER=${REDIS_SERVER:-redis-server-msan-$SAN_REDIS_VER}
- if ! command -v $REDIS_SERVER > /dev/null; then
- echo Building Redis for clang-msan ...
- $READIES/bin/getredis --force -v $SAN_REDIS_VER --no-run --own-openssl \
- --suffix msan --clang-msan --llvm-dir /opt/llvm-project/build-msan \
- --clang-san-blacklist $ignorelist
- fi
- fi
}
#----------------------------------------------------------------------------------------------
@@ -211,12 +183,6 @@ setup_redis_server() {
#----------------------------------------------------------------------------------------------
setup_valgrind() {
- REDIS_SERVER=${REDIS_SERVER:-redis-server-vg}
- if ! is_command $REDIS_SERVER; then
- echo Building Redis for Valgrind ...
- $READIES/bin/getredis -v $VALGRIND_REDIS_VER --valgrind --suffix vg
- fi
-
if [[ $VG_LEAKS == 0 ]]; then
VG_LEAK_CHECK=no
RLTEST_VG_NOLEAKS="--vg-no-leakcheck"
@@ -232,7 +198,7 @@ setup_valgrind() {
--track-origins=yes \
--show-possibly-lost=no"
- VALGRIND_SUPRESSIONS=$ROOT/tests/memcheck/valgrind.supp
+ VALGRIND_SUPRESSIONS=$ROOT/tests/memcheck/redis_valgrind.sup
RLTEST_VG_ARGS+="\
--use-valgrind \
@@ -356,7 +322,7 @@ run_tests() {
fi
[[ $RLEC == 1 ]] && export RLEC_CLUSTER=1
-
+
local E=0
if [[ $NOP != 1 ]]; then
{ $OP python3 -m RLTest @$rltest_config; (( E |= $? )); } || true
@@ -441,12 +407,6 @@ fi
[[ $SAN == addr ]] && SAN=address
[[ $SAN == mem ]] && SAN=memory
-if [[ -n $TEST ]]; then
- [[ $LOG != 1 ]] && RLTEST_LOG=0
- # export BB=${BB:-1}
- export RUST_BACKTRACE=1
-fi
-
#-------------------------------------------------------------------------------- Platform Mode
if [[ $PLATFORM_MODE == 1 ]]; then
@@ -525,7 +485,7 @@ fi
setup_rltest
if [[ -n $SAN ]]; then
- setup_clang_sanitizer
+ setup_sanitizer
fi
if [[ $VG == 1 ]]; then
@@ -549,13 +509,13 @@ if [[ $GEN == 1 ]]; then
{ (run_tests "general"); (( E |= $? )); } || true
fi
if [[ $VG != 1 && $SLAVES == 1 ]]; then
- { (RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves"); (( E |= $? )); } || true
+ { (RLTEST_ARGS+=" --use-slaves --enable-debug-command" run_tests "--use-slaves"); (( E |= $? )); } || true
fi
if [[ $AOF == 1 ]]; then
- { (RLTEST_ARGS+=" --use-aof" run_tests "--use-aof"); (( E |= $? )); } || true
+ { (RLTEST_ARGS+=" --use-aof --enable-debug-command" run_tests "--use-aof"); (( E |= $? )); } || true
fi
if [[ $CLUSTER == 1 ]]; then
- { (RLTEST_ARGS+=" --env oss-cluster --shards-count 1" run_tests "--env oss-cluster"); (( E |= $? )); } || true
+ { (RLTEST_ARGS+=" --env oss-cluster --shards-count 1 --enable-debug-command" run_tests "--env oss-cluster"); (( E |= $? )); } || true
fi
#-------------------------------------------------------------------------------------- Summary
@@ -564,7 +524,7 @@ if [[ $NO_SUMMARY == 1 ]]; then
exit 0
fi
-if [[ $NOP != 1 && -n $SAN ]]; then
+if [[ $NOP != 1 ]]; then
if [[ -n $SAN || $VG == 1 ]]; then
{ FLOW=1 $ROOT/sbin/memcheck-summary; (( E |= $? )); } || true
fi
diff --git a/tests/memcheck/redis_valgrind.sup b/tests/memcheck/redis_valgrind.sup
index a44aedf6..b05843d8 100644
--- a/tests/memcheck/redis_valgrind.sup
+++ b/tests/memcheck/redis_valgrind.sup
@@ -1,152 +1,17 @@
{
-
+
Memcheck:Cond
fun:lzf_compress
}
{
-
+
Memcheck:Value4
fun:lzf_compress
}
{
-
+
Memcheck:Value8
fun:lzf_compress
}
-
-{
-
- Memcheck:Value8
- fun:crcspeed64little
- fun:crcspeed64native
- fun:crc64
- fun:createDumpPayload
- fun:dumpCommand
-}
-
-
-{
-
- Memcheck:Value8
- fun:crcspeed64little
- fun:crcspeed64native
- fun:crc64
- fun:rioGenericUpdateChecksum
- fun:rioWrite
-}
-
-
-{
-
- Memcheck:Value8
- fun:crcspeed64little
- fun:createDumpPayload
- fun:dumpCommand
-}
-
-{
-
- Memcheck:Param
- write(buf)
- fun:__libc_write
- fun:write
- fun:connSocketWrite
- fun:connWrite
-}
-
-{
-
- Memcheck:Addr8
- fun:appendBits
- fun:appendFloat
- fun:Compressed_Append
- fun:Compressed_AddSample
-}
-
-{
-
- Memcheck:Addr8
- fun:appendBits
- fun:appendInteger
- fun:Compressed_Append
- fun:Compressed_AddSample
-}
-
-{
-
- Memcheck:Addr1
- fun:raxLowWalk
- fun:raxSeek
- fun:RM_DictIteratorStartC
-}
-
-{
-
- Memcheck:Addr1
- fun:raxSeek
- fun:RM_DictIteratorStartC
-}
-
-{
-
- Memcheck:Param
- write(buf)
- fun:__libc_write
- fun:write
- fun:_IO_file_write@@GLIBC_2.2.5
- fun:new_do_write
- fun:_IO_new_do_write
- fun:_IO_do_write@@GLIBC_2.2.5
- fun:_IO_new_file_xsputn
- fun:_IO_file_xsputn@@GLIBC_2.2.5
- fun:__vfprintf_internal
- fun:__fprintf_chk
- fun:fprintf
- fun:serverLogRaw
- fun:RM_LogRaw
- fun:RM_Log
-}
-
-{
-
- Memcheck:Cond
- fun:strlen
- fun:__vfprintf_internal
- fun:__vsnprintf_internal
- fun:RM_LogRaw
- fun:RM_Log
-}
-
-{
-
- Memcheck:Cond
- fun:strlen
- fun:__vfprintf_internal
- fun:__vsnprintf_internal
- fun:vsnprintf
- fun:RM_LogRaw
- fun:RM_Log
-}
-
-{
-
- Memcheck:Cond
- fun:strlen
- fun:vfprintf
- fun:fprintf
- fun:serverLogRaw
- fun:RM_LogRaw
- fun:RM_Log
-}
-
-{
-
- Memcheck:Cond
- fun:strlen
- fun:vfprintf
- fun:vsnprintf
- fun:RM_LogRaw
- fun:RM_Log
-}