diff --git a/.github/wordlist.txt b/.github/wordlist.txt index f5df9ca1..652a293a 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -64,3 +64,5 @@ vertx wjso wq yadazula +backport +mitigations diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..6ada7478 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,426 @@ +name: CI + +on: [push, pull_request] + +jobs: + test-recent-ubuntu: + runs-on: ${{ matrix.builder-os }} + strategy: + fail-fast: false + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + # jammy and bionic + builder-os: ['ubuntu-22.04', 'ubuntu-20.04'] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool cmake lcov valgrind + - name: Setup Python for testing + uses: actions/setup-python@v5 + with: + python-version: '3.9' + architecture: 'x64' + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: cd redis && make -j 4 + - name: Build + run: make -j 4 + - name: Run tests + run: make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + test-old-ubuntu: + runs-on: ubuntu-latest + container: ${{ matrix.builder-container }} + strategy: + fail-fast: false + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + builder-container: ['ubuntu:xenial', 'ubuntu:bionic'] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install build dependencies + run: | + apt-get update && apt-get install -y software-properties-common + add-apt-repository ppa:git-core/ppa -y + apt-get update + apt-get install -y build-essential make autoconf automake libtool lcov git wget zlib1g-dev lsb-release libssl-dev openssl ca-certificates + wget https://cmake.org/files/v3.28/cmake-3.28.0.tar.gz + tar -xzvf cmake-3.28.0.tar.gz + cd cmake-3.28.0 + ./configure + make -j `nproc` + make install + cd .. + ln -s /usr/local/bin/cmake /usr/bin/cmake + wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz + tar -xvf Python-3.9.6.tgz + cd Python-3.9.6 + ./configure + make -j `nproc` + make altinstall + cd .. + rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3 + rm /usr/bin/lsb_release + python3 --version + make --version + cmake --version + python3 -m pip install --upgrade pip + - name: Checkout RedisBloom + uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: cd redis && make -j `nproc` + - name: Build RedisBloom + run: make -j `nproc` + - name: Run tests + run: make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + debian: + runs-on: ubuntu-latest + container: ${{ matrix.builder-container }} + strategy: + fail-fast: false + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + builder-container: ['debian:bullseye'] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install build dependencies + run: | + apt-get update + apt-get install -y build-essential make cmake autoconf automake libtool lcov git wget zlib1g-dev lsb-release libssl-dev openssl ca-certificates python3 python3-pip + python3 --version + make --version + cmake --version + python3 -m pip install --upgrade pip + - name: Checkout RedisBloom + uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: cd redis && make -j `nproc` + - name: Build RedisBloom + run: make -j `nproc` + - name: Run tests + run: make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + test-valgrind: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool cmake lcov valgrind + - name: Setup Python for testing + uses: actions/setup-python@v5 + with: + python-version: '3.9' + architecture: 'x64' + - name: Install Python dependencies + run: + python -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: cd redis && make valgrind -j 4 + - name: Build + run: make -j 4 + - name: Run tests + run: make test VG=1 REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + test-address-sanitizer: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + redis-version: ["7.2.4", "unstable"] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool cmake lcov valgrind + - name: Setup Python for testing + uses: actions/setup-python@v5 + with: + python-version: '3.9' + architecture: 'x64' + - name: Install Python dependencies + run: + python -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: cd redis && make SANITIZER=address -j 4 + - name: Build + run: make -j 4 + - name: Run tests + run: make test SAN=addr REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + centos: + runs-on: ubuntu-latest + container: ${{ matrix.builder-container }} + strategy: + fail-fast: false + matrix: + redis-version: ["6.2.14", "7.2.4", "unstable"] + builder-container: ['centos:7'] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install build dependencies + run: | + yum -y install epel-release + yum -y install http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum -y install gcc make cmake3 git python-pip openssl-devel bzip2-devel libffi-devel zlib-devel wget centos-release-scl scl-utils + yum groupinstall "Development Tools" -y + yum install -y devtoolset-11 + . scl_source enable devtoolset-11 || true + make --version + gcc --version + git --version + wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz + tar -xvf Python-3.9.6.tgz + cd Python-3.9.6 + ./configure + make -j `nproc` + make altinstall + cd .. + rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3 + ln -s `which cmake3` /usr/bin/cmake + python3 --version + - name: Checkout sources + uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: | + scl enable devtoolset-11 bash + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: | + cd redis && make -j `nproc` + - name: Build RedisBloom + run: | + . scl_source enable devtoolset-11 || true + make -j `nproc` + - name: Run tests + run: | + . scl_source enable devtoolset-11 || true + make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + amazon-linux: + runs-on: ubuntu-latest + container: ${{ matrix.builder-container }} + strategy: + fail-fast: false + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + builder-container: ['amazonlinux:2'] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install build dependencies + run: | + amazon-linux-extras install epel -y + yum -y install epel-release yum-utils + yum-config-manager --add-repo http://mirror.centos.org/centos/7/sclo/x86_64/rh/ + yum -y install gcc make cmake3 git python-pip openssl-devel bzip2-devel libffi-devel zlib-devel wget centos-release-scl scl-utils which tar + yum -y install devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-make --nogpgcheck + . scl_source enable devtoolset-11 || true + make --version + git --version + wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz + tar -xvf Python-3.9.6.tgz + cd Python-3.9.6 + ./configure + make -j `nproc` + make altinstall + cd .. + rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3 + ln -s `which cmake3` /usr/bin/cmake + python3 --version + - name: Checkout sources + uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: | + scl enable devtoolset-11 bash + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: | + cd redis && make -j `nproc` + - name: Build RedisBloom + run: | + . scl_source enable devtoolset-11 || true + make -j `nproc` + - name: Run tests + run: | + . scl_source enable devtoolset-11 || true + make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + rocky-linux: + runs-on: ubuntu-latest + container: ${{ matrix.builder-container }} + strategy: + fail-fast: false + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + builder-container: ['rockylinux:8', 'rockylinux:9'] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install build dependencies + run: | + yum -y install epel-release + yum -y install http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum -y install gcc make cmake3 git openssl-devel bzip2-devel libffi-devel zlib-devel wget scl-utils gcc-toolset-13 which + yum groupinstall "Development Tools" -y + . scl_source enable gcc-toolset-13 || true + make --version + gcc --version + git --version + wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz + tar -xvf Python-3.9.6.tgz + cd Python-3.9.6 + ./configure + make -j `nproc` + make altinstall + cd .. + rm /usr/bin/python3 && ln -s `which python3.9` /usr/bin/python3 + cmake --version + python3 --version + - name: Checkout sources + uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: | + . scl_source enable gcc-toolset-13 || true + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: | + cd redis && make -j `nproc` + - name: Build RedisBloom + run: | + . scl_source enable gcc-toolset-13 || true + make -j `nproc` + - name: Run tests + run: | + . scl_source enable gcc-toolset-13 || true + make test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server + + macos-x86_64: + runs-on: macos-13 + strategy: + fail-fast: false + # TODO: figure out the version we need to test on. + matrix: + redis-version: ["6.0.20", "7.2.4", "unstable"] + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install prerequisites + run: | + brew install make + - name: Checkout sources + uses: actions/checkout@v3 + with: + submodules: true + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install -r tests/flow/requirements.txt + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ matrix.redis-version }} + path: 'redis' + - name: Build Redis + run: | + cd redis && make -j `sysctl -n hw.logicalcpu` + - name: Build RedisBloom + run: | + gmake -j `sysctl -n hw.logicalcpu` + - name: Run tests + run: | + gmake test REDIS_SERVER=$GITHUB_WORKSPACE/redis/src/redis-server diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml deleted file mode 100644 index f6286962..00000000 --- a/.github/workflows/sanitizer.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: CLang Sanitizer - -on: - push: - workflow_dispatch: - -jobs: - clang-sanitizer: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l -eo pipefail {0} - container: - image: redisfab/clang:16-x64-bullseye - options: --cpus 2 - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup - run: ./sbin/setup - - name: Build - run: make SAN=addr - - name: Unit tests - run: make unit-tests SAN=addr - - name: Flow tests - run: make flow-tests SAN=addr diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..d70acb19 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,42 @@ +# Security Policy + +## Supported Versions + +RedisBloom is generally backwards compatible with very few exceptions, so we +recommend users to always use the latest version to experience stability, +performance and security. + +We generally backport security issues to a single previous major version, +unless this is not possible or feasible with a reasonable effort. + +| Version | Supported | +| ------- | ------------------ | +| 2.6 | :white_check_mark: | +| 2.4 | :white_check_mark: | +| < 2.4 | :x: | + +## Reporting a Vulnerability + +If you believe you've discovered a serious vulnerability, please contact the +Redis core team at redis@redis.io. We will evaluate your report and if +necessary issue a fix and an advisory. If the issue was previously undisclosed, +we'll also mention your name in the credits. + +## Responsible Disclosure + +In some cases, we may apply a responsible disclosure process to reported or +otherwise discovered vulnerabilities. We will usually do that for a critical +vulnerability, and only if we have a good reason to believe information about +it is not yet public. + +This process involves providing an early notification about the vulnerability, +its impact and mitigations to a short list of vendors under a time-limited +embargo on public disclosure. + +Vendors on the list are individuals or organizations that maintain Redis +distributions or provide Redis as a service, who have third party users who +will benefit from the vendor's ability to prepare for a new version or deploy a +fix early. + +If you believe you should be on the list, please contact us and we will +consider your request based on the above criteria. diff --git a/deps/bloom/bloom.c b/deps/bloom/bloom.c index 77040ea7..6d89cd90 100644 --- a/deps/bloom/bloom.c +++ b/deps/bloom/bloom.c @@ -42,6 +42,8 @@ extern void * (*RedisModule_Calloc)(size_t nmemb, size_t size); #define MODE_READ 0 #define MODE_WRITE 1 +#define LN2 (0.693147180559945) + inline static int test_bit_set_bit(unsigned char *buf, uint64_t x, int mode) { uint64_t byte = x >> 3; uint8_t mask = 1 << (x % 8); @@ -147,7 +149,13 @@ int bloom_init(struct bloom *bloom, uint64_t entries, double error, unsigned opt } else if (options & BLOOM_OPT_NOROUND) { // Don't perform any rounding. Conserve memory instead - bits = bloom->bits = (uint64_t)(entries * bloom->bpe); + bits = (uint64_t)(entries * bloom->bpe); + + // Guard against very small 'bpe'. Have at least one bit in the filter. + if (bits == 0) { + bits = 1; + } + bloom->bits = bits; bloom->n2 = 0; } else { @@ -176,7 +184,7 @@ int bloom_init(struct bloom *bloom, uint64_t entries, double error, unsigned opt bloom->bits = bloom->bytes * 8; bloom->force64 = (options & BLOOM_OPT_FORCE64); - bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2) + bloom->hashes = (int)ceil(LN2 * bloom->bpe); // ln(2) bloom->bf = (unsigned char *)BLOOM_CALLOC(bloom->bytes, sizeof(unsigned char)); if (bloom->bf == NULL) { return 1; @@ -220,3 +228,15 @@ int bloom_add(struct bloom *bloom, const void *buffer, int len) { void bloom_free(struct bloom *bloom) { BLOOM_FREE(bloom->bf); } const char *bloom_version() { return MAKESTRING(BLOOM_VERSION); } + +// Returns 0 on success +int bloom_validate_integrity(struct bloom *bloom) { + if (bloom->error <= 0 || bloom->error >= 1.0 || + (bloom->n2 != 0 && bloom->bits < (1ULL << bloom->n2)) || + bloom->bits == 0 || bloom->bits != bloom->bytes * 8 || + bloom->hashes != (int)ceil(LN2 * bloom->bpe)) { + return 1; + } + + return 0; +} diff --git a/deps/bloom/bloom.h b/deps/bloom/bloom.h index e3001a56..a6c5750d 100644 --- a/deps/bloom/bloom.h +++ b/deps/bloom/bloom.h @@ -160,6 +160,13 @@ void bloom_free(struct bloom *bloom); */ const char *bloom_version(); +/** + * Validates filter state + * + * Return: 0 on success + */ +int bloom_validate_integrity(struct bloom *bloom); + #ifdef __cplusplus } #endif diff --git a/docs/commands/cf.reserve.md b/docs/commands/cf.reserve.md index 75936340..9f2adec4 100644 --- a/docs/commands/cf.reserve.md +++ b/docs/commands/cf.reserve.md @@ -24,25 +24,40 @@ is key name for the the cuckoo filter to be created.
capacity -Estimated capacity for the filter. Capacity is rounded to the next `2^n` number. The filter will likely not fill up to 100% of it's capacity. -Make sure to reserve extra capacity if you want to avoid expansions. +Estimated capacity for the filter. + +Capacity is rounded to the next `2^n` number. + +The filter will likely not fill up to 100% of it's capacity. Make sure to reserve extra capacity if you want to avoid expansions.
## Optional arguments
BUCKETSIZE bucketsize -Number of items in each bucket. A higher bucket size value improves the fill rate but also causes a higher error rate and slightly slower performance. The default value is 2. +Number of items in each bucket. + +A higher bucket size value improves the fill rate but also causes a higher error rate and slightly slower performance. + +`bucketsize` is an integer between 1 and 255. The default value is 2.
MAXITERATIONS maxiterations -Number of attempts to swap items between buckets before declaring filter as full and creating an additional filter. A low value is better for performance and a higher number is better for filter fill rate. The default value is 20. +Number of attempts to swap items between buckets before declaring filter as full and creating an additional filter. + +A low value is better for performance and a higher number is better for filter fill rate. + +`maxiterations` is an integer between 1 and 65535. The default value is 20.
EXPANSION expansion -When a new filter is created, its size is the size of the current filter multiplied by `expansion`, specified as a non-negative integer. Expansion is rounded to the next `2^n` number. The default value is `1`. +When a new filter is created, its size is the size of the current filter multiplied by `expansion`. + +`expansion` is an integer between 0 and 32768. The default value is 1. + +Expansion is rounded to the next `2^n` number.
## Return value diff --git a/sbin/memcheck-summary b/sbin/memcheck-summary index ed884823..752b5ecd 100755 --- a/sbin/memcheck-summary +++ b/sbin/memcheck-summary @@ -11,79 +11,55 @@ cd $HERE #---------------------------------------------------------------------------------------------- valgrind_check() { - echo -n "${NOCOLOR}" - if grep -l "$1" $logdir/*.valgrind.log &> /dev/null; then - echo - echo "${LIGHTRED}### Valgrind: ${TYPE} detected:${RED}" - grep -l "$1" $logdir/*.valgrind.log - echo -n "${NOCOLOR}" - E=1 - fi -} - -valgrind_summary() { local logdir="$ROOT/tests/$DIR/logs" - local leaks_head=0 - for file in $(ls $logdir/*.valgrind.log 2>/dev/null); do - # If the last "definitely lost: " line of a logfile has a nonzero value, print the file name - if tac "$file" | grep -a -m 1 "definitely lost: " | grep "definitely lost: [1-9][0-9,]* bytes" &> /dev/null; then - if [[ $leaks_head == 0 ]]; then - echo - echo "${LIGHTRED}### Valgrind: leaks detected:${RED}" - leaks_head=1 - fi - echo "$file" + for file in "$logdir"/*.valgrind.log; do + if grep -q -e "definitely lost" -e "0x" -e "Invalid" -e "Mismatched" \ + -e "uninitialized" -e "has a fishy" -e "overlap" "$file"; then + echo + echo "### Valgrind error in $file:" + cat "$file" E=1 fi done - - TYPE="invalid reads" valgrind_check "Invalid read" - TYPE="invalid writes" valgrind_check "Invalid write" } #---------------------------------------------------------------------------------------------- sanitizer_check() { - if grep -l "$1" $logdir/*.asan.log* &> /dev/null; then - echo - echo "${LIGHTRED}### Sanitizer: ${TYPE} detected:${RED}" - grep -l "$1" $logdir/*.asan.log* - echo "${NOCOLOR}" - E=1 - fi -} - -sanitizer_summary() { local logdir="$ROOT/tests/$DIR/logs" - if ! TYPE="leaks" sanitizer_check "Direct leak"; then - TYPE="leaks" sanitizer_check "detected memory leaks" - fi - TYPE="buffer overflow" sanitizer_check "dynamic-stack-buffer-overflow" - TYPE="memory errors" sanitizer_check "memcpy-param-overlap" - TYPE="stack use after scope" sanitizer_check "stack-use-after-scope" - TYPE="use after free" sanitizer_check "heap-use-after-free" + + for file in "$logdir"/*.asan.log*; do + if grep -q -e "runtime error" -e "Sanitizer" "$file"; then + echo + echo "### Sanitizer error in $file:" + cat "$file" + E=1 + fi + done } + + #---------------------------------------------------------------------------------------------- E=0 DIRS= -#if [[ $UNIT == 1 ]]; then -# DIRS+=" ctests cpptests" -#fi +if [[ $UNIT == 1 ]]; then + DIRS+=" unit" +fi if [[ $FLOW == 1 ]]; then - DIRS+=" pytests" + DIRS+=" flow" fi if [[ $VG == 1 ]]; then for dir in $DIRS; do - DIR="$dir" valgrind_summary + DIR="$dir" valgrind_check done elif [[ -n $SAN ]]; then for dir in $DIRS; do - DIR="$dir" sanitizer_summary + DIR="$dir" sanitizer_check done fi diff --git a/src/cf.c b/src/cf.c index ca7878c5..a9e82a3f 100644 --- a/src/cf.c +++ b/src/cf.c @@ -84,7 +84,7 @@ const char *CF_GetEncodedChunk(const CuckooFilter *cf, long long *pos, size_t *b } int CF_LoadEncodedChunk(const CuckooFilter *cf, long long pos, const char *data, size_t datalen) { - if (datalen == 0) { + if (datalen == 0 || pos <= 0 || (size_t)(pos - 1) < datalen) { return REDISMODULE_ERR; } @@ -102,6 +102,12 @@ int CF_LoadEncodedChunk(const CuckooFilter *cf, long long pos, const char *data, offset -= currentSize; } + // Boundary check before memcpy() + if (!filter || ((size_t)offset > SIZE_MAX - datalen) || + filter->bucketSize * filter->numBuckets < offset + datalen) { + return REDISMODULE_ERR; + } + // copy data to filter memcpy(filter->data + offset, data, datalen); return REDISMODULE_OK; @@ -116,7 +122,12 @@ CuckooFilter *CFHeader_Load(const CFHeader *header) { filter->bucketSize = header->bucketSize; filter->maxIterations = header->maxIterations; filter->expansion = header->expansion; - filter->filters = RedisModule_Alloc(sizeof(*filter->filters) * header->numFilters); + filter->filters = RedisModule_Calloc(sizeof(*filter->filters), filter->numFilters); + + if (CuckooFilter_ValidateIntegrity(filter) != 0) { + goto error; + } + for (size_t ii = 0; ii < filter->numFilters; ++ii) { SubCF *cur = filter->filters + ii; cur->bucketSize = header->bucketSize; @@ -125,6 +136,11 @@ CuckooFilter *CFHeader_Load(const CFHeader *header) { RedisModule_Calloc((size_t)cur->numBuckets * filter->bucketSize, sizeof(CuckooBucket)); } return filter; + +error: + CuckooFilter_Free(filter); + RedisModule_Free(filter); + return NULL; } void fillCFHeader(CFHeader *header, const CuckooFilter *cf) { diff --git a/src/cuckoo.c b/src/cuckoo.c index 15cb7803..ba52b49b 100644 --- a/src/cuckoo.c +++ b/src/cuckoo.c @@ -397,3 +397,15 @@ void CuckooFilter_GetInfo(const CuckooFilter *cf, CuckooHash hash, CuckooKey *ou assert(getAltHash(params.fp, out->h1, cf->numBuckets) == out->h2); assert(getAltHash(params.fp, out->h2, cf->numBuckets) == out->h1); }*/ + +// Returns 0 on success +int CuckooFilter_ValidateIntegrity(const CuckooFilter *cf) { + if (cf->bucketSize == 0 || cf->bucketSize > CF_MAX_BUCKET_SIZE || + cf->numBuckets == 0 || cf->numBuckets > CF_MAX_NUM_BUCKETS || + cf->numFilters == 0 || cf->numFilters > CF_MAX_NUM_FILTERS || + cf->maxIterations == 0 || !isPower2(cf->numBuckets) ) { + return 1; + } + + return 0; +} diff --git a/src/cuckoo.h b/src/cuckoo.h index 529b15f9..68d34367 100644 --- a/src/cuckoo.h +++ b/src/cuckoo.h @@ -24,6 +24,15 @@ typedef uint64_t CuckooHash; typedef uint8_t CuckooBucket[1]; typedef uint8_t MyCuckooBucket; +#define CF_DEFAULT_MAX_ITERATIONS 20 +#define CF_DEFAULT_BUCKETSIZE 2 +#define CF_DEFAULT_EXPANSION 1 +#define CF_MAX_EXPANSION 32768 +#define CF_MAX_ITERATIONS 65535 +#define CF_MAX_BUCKET_SIZE 255 // 8 bits, see struct SubCF +#define CF_MAX_NUM_BUCKETS (0x00FFFFFFFFFFFFFFULL) // 56 bits, see struct SubCF +#define CF_MAX_NUM_FILTERS (UINT16_MAX) // 16 bits, see struct CuckooFilter + typedef struct { uint64_t numBuckets : 56; uint64_t bucketSize : 8; @@ -72,3 +81,4 @@ int CuckooFilter_Check(const CuckooFilter *filter, CuckooHash hash); uint64_t CuckooFilter_Count(const CuckooFilter *filter, CuckooHash); void CuckooFilter_Compact(CuckooFilter *filter, bool cont); void CuckooFilter_GetInfo(const CuckooFilter *cf, CuckooHash hash, CuckooKey *out); +int CuckooFilter_ValidateIntegrity(const CuckooFilter *cf); diff --git a/src/rebloom.c b/src/rebloom.c index 8a4fd429..f876087e 100644 --- a/src/rebloom.c +++ b/src/rebloom.c @@ -19,14 +19,12 @@ #include // strncasecmp #include #include +#include #ifndef REDISBLOOM_GIT_SHA #define REDISBLOOM_GIT_SHA "unknown" #endif -#define CF_MAX_ITERATIONS 20 -#define CF_DEFAULT_BUCKETSIZE 2 -#define CF_DEFAULT_EXPANSION 1 #define BF_DEFAULT_EXPANSION 2 //////////////////////////////////////////////////////////////////////////////// @@ -107,8 +105,8 @@ static SBChain *bfCreateChain(RedisModuleKey *key, double error_rate, size_t cap return sb; } -static CuckooFilter *cfCreate(RedisModuleKey *key, size_t capacity, size_t bucketSize, - size_t maxIterations, size_t expansion) { +static CuckooFilter *cfCreate(RedisModuleKey *key, size_t capacity, uint16_t bucketSize, + uint16_t maxIterations, uint16_t expansion) { if (capacity < bucketSize * 2) return NULL; @@ -418,8 +416,10 @@ static int BFDebug_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, i for (size_t ii = 0; ii < sb->nfilters; ++ii) { const SBLink *lb = sb->filters + ii; info_s = RedisModule_CreateStringPrintf( - ctx, "bytes:%lu bits:%llu hashes:%u hashwidth:%u capacity:%lu size:%lu ratio:%g", - lb->inner.bytes, lb->inner.bits ? lb->inner.bits : 1LLU << lb->inner.n2, + ctx, + "bytes:%" PRIu64 " bits:%" PRIu64 " hashes:%u hashwidth:%u capacity:%" PRIu64 + " size:%zu ratio:%g", + lb->inner.bytes, lb->inner.bits ? lb->inner.bits : UINT64_C(1) << lb->inner.n2, lb->inner.hashes, sb->options & BLOOM_OPT_FORCE64 ? 64 : 32, lb->inner.entries, lb->size, lb->inner.error); RedisModule_ReplyWithString(ctx, info_s); @@ -529,14 +529,14 @@ static int CFReserve_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, return RedisModule_ReplyWithError(ctx, "Bad capacity"); } - long long maxIterations = CF_MAX_ITERATIONS; + long long maxIterations = CF_DEFAULT_MAX_ITERATIONS; int mi_loc = RMUtil_ArgIndex("MAXITERATIONS", argv, argc); if (mi_loc != -1) { if (RedisModule_StringToLongLong(argv[mi_loc + 1], &maxIterations) != REDISMODULE_OK) { return RedisModule_ReplyWithError(ctx, "Couldn't parse MAXITERATIONS"); - } else if (maxIterations <= 0) { + } else if (maxIterations <= 0 || maxIterations > CF_MAX_ITERATIONS) { return RedisModule_ReplyWithError( - ctx, "MAXITERATIONS parameter needs to be a positive integer"); + ctx, "MAXITERATIONS: value must be an integer between 1 and 65535, inclusive."); } } @@ -545,9 +545,9 @@ static int CFReserve_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, if (bs_loc != -1) { if (RedisModule_StringToLongLong(argv[bs_loc + 1], &bucketSize) != REDISMODULE_OK) { return RedisModule_ReplyWithError(ctx, "Couldn't parse BUCKETSIZE"); - } else if (bucketSize <= 0) { + } else if (bucketSize <= 0 || bucketSize > CF_MAX_BUCKET_SIZE) { return RedisModule_ReplyWithError( - ctx, "BUCKETSIZE parameter needs to be a positive integer"); + ctx, "BUCKETSIZE: value must be an integer between 1 and 255, inclusive."); } } @@ -556,9 +556,9 @@ static int CFReserve_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, if (ex_loc != -1) { if (RedisModule_StringToLongLong(argv[ex_loc + 1], &expansion) != REDISMODULE_OK) { return RedisModule_ReplyWithError(ctx, "Couldn't parse EXPANSION"); - } else if (expansion < 0) { + } else if (expansion < 0 || expansion > CF_MAX_EXPANSION) { return RedisModule_ReplyWithError( - ctx, "EXPANSION parameter needs to be a non-negative integer"); + ctx, "EXPANSION: value must be an integer between 0 and 32768, inclusive."); } } @@ -596,7 +596,7 @@ static int cfInsertCommon(RedisModuleCtx *ctx, RedisModuleString *keystr, RedisM int status = cfGetFilter(key, &cf); if (status == SB_EMPTY && options->autocreate) { - if ((cf = cfCreate(key, options->capacity, CF_DEFAULT_BUCKETSIZE, CF_MAX_ITERATIONS, + if ((cf = cfCreate(key, options->capacity, CF_DEFAULT_BUCKETSIZE, CF_DEFAULT_MAX_ITERATIONS, CF_DEFAULT_EXPANSION)) == NULL) { return RedisModule_ReplyWithError(ctx, "Could not create filter"); // LCOV_EXCL_LINE } @@ -850,7 +850,7 @@ static int CFScanDump_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv } long long pos; - if (RedisModule_StringToLongLong(argv[2], &pos) != REDISMODULE_OK) { + if (RedisModule_StringToLongLong(argv[2], &pos) != REDISMODULE_OK || pos < 0) { return RedisModule_ReplyWithError(ctx, "Invalid position"); } @@ -1092,7 +1092,8 @@ static int CFDebug_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, i RedisModuleString *resp = RedisModule_CreateStringPrintf( ctx, - "bktsize:%u buckets:%lu items:%lu deletes:%lu filters:%u max_iterations:%u expansion:%u", + "bktsize:%u buckets:%" PRIu64 " items:%" PRIu64 " deletes:%" PRIu64 + " filters:%u max_iterations:%u expansion:%u", cf->bucketSize, cf->numBuckets, cf->numItems, cf->numDeletes, cf->numFilters, cf->maxIterations, cf->expansion); return RedisModule_ReplyWithString(ctx, resp); @@ -1252,7 +1253,7 @@ static void *CFRdbLoad(RedisModuleIO *io, int encver) { if (encver < CF_MIN_EXPANSION_VERSION) { // CF_ENCODING_VERSION when added cf->numDeletes = 0; // Didn't exist earlier. bug fix cf->bucketSize = CF_DEFAULT_BUCKETSIZE; - cf->maxIterations = CF_MAX_ITERATIONS; + cf->maxIterations = CF_DEFAULT_MAX_ITERATIONS; cf->expansion = CF_DEFAULT_EXPANSION; } else { cf->numDeletes = RedisModule_LoadUnsigned(io); diff --git a/src/sb.c b/src/sb.c index ddfe30bc..fb6ac7f8 100644 --- a/src/sb.c +++ b/src/sb.c @@ -26,15 +26,14 @@ bloom_hashval bloom_calc_hash64(const void *buffer, int len); #define CUR_FILTER(sb) ((sb)->filters + ((sb)->nfilters - 1)) static int SBChain_AddLink(SBChain *chain, uint64_t size, double error_rate) { - if (!chain->filters) { - chain->filters = RedisModule_Calloc(1, sizeof(*chain->filters)); - } else { - chain->filters = - RedisModule_Realloc(chain->filters, sizeof(*chain->filters) * (chain->nfilters + 1)); - } + chain->filters = + RedisModule_Realloc(chain->filters, sizeof(*chain->filters) * (chain->nfilters + 1)); SBLink *newlink = chain->filters + chain->nfilters; - newlink->size = 0; + *newlink = (SBLink){ + .size = 0, + }; + chain->nfilters++; return bloom_init(&newlink->inner, size, error_rate, chain->options); } @@ -150,7 +149,9 @@ typedef struct __attribute__((packed)) { } dumpedChainHeader; static SBLink *getLinkPos(const SBChain *sb, long long curIter, size_t *offset) { - // printf("Requested %lld\n", curIter); + if (curIter < 1) { + return NULL; + } curIter--; SBLink *link = NULL; @@ -218,6 +219,28 @@ char *SBChain_GetEncodedHeader(const SBChain *sb, size_t *hdrlen) { void SB_FreeEncodedHeader(char *s) { RedisModule_Free(s); } +// Returns 0 on success +int SB_ValidateIntegrity(const SBChain *sb) { + if (sb->options & + ~(BLOOM_OPT_NOROUND | BLOOM_OPT_ENTS_IS_BITS | BLOOM_OPT_FORCE64 | BLOOM_OPT_NO_SCALING)) { + return 1; + } + + size_t total = 0; + for (size_t i = 0; i < sb->nfilters; i++) { + if (sb->filters[i].size > SIZE_MAX - total) { + return 1; + } + total += sb->filters[i].size; + } + + if (sb->size != total) { + return 1; + } + + return 0; +} + SBChain *SB_NewChainFromHeader(const char *buf, size_t bufLen, const char **errmsg) { const dumpedChainHeader *header = (const void *)buf; if (bufLen < sizeof(dumpedChainHeader)) { @@ -243,17 +266,34 @@ SBChain *SB_NewChainFromHeader(const char *buf, size_t bufLen, const char **errm #define X(encfld, dstfld) dstfld = encfld; X_ENCODED_LINK(X, srclink, dstlink) #undef X - dstlink->inner.bf = RedisModule_Alloc(dstlink->inner.bytes); + + if (bloom_validate_integrity(&dstlink->inner) != 0) { + SBChain_Free(sb); + *errmsg = "ERR received bad data"; + return NULL; + } + + dstlink->inner.bf = RedisModule_Calloc(1, dstlink->inner.bytes); if (sb->options & BLOOM_OPT_FORCE64) { dstlink->inner.force64 = 1; } } + if (SB_ValidateIntegrity(sb) != 0) { + SBChain_Free(sb); + *errmsg = "ERR received bad data"; + return NULL; + } + return sb; } int SBChain_LoadEncodedChunk(SBChain *sb, long long iter, const char *buf, size_t bufLen, const char **errmsg) { + if (!buf || iter <= 0 || iter < bufLen) { + *errmsg = "ERR received bad data"; + return -1; + } // Load the chunk size_t offset; iter -= bufLen; diff --git a/src/sb.h b/src/sb.h index 75db51b9..c3b180c5 100644 --- a/src/sb.h +++ b/src/sb.h @@ -104,6 +104,9 @@ SBChain *SB_NewChainFromHeader(const char *buf, size_t bufLen, const char **errm */ int SBChain_LoadEncodedChunk(SBChain *sb, long long iter, const char *buf, size_t bufLen, const char **errmsg); + +int SB_ValidateIntegrity(const SBChain *sb); + #ifdef __cplusplus } #endif diff --git a/tests/flow/requirements.txt b/tests/flow/requirements.txt index 899f8acd..56d4f282 100644 --- a/tests/flow/requirements.txt +++ b/tests/flow/requirements.txt @@ -1,2 +1,2 @@ -RLTest ~= 0.7.2 +RLTest == 0.7.5 numpy diff --git a/tests/flow/test_cuckoo.py b/tests/flow/test_cuckoo.py index b5041383..1d193693 100644 --- a/tests/flow/test_cuckoo.py +++ b/tests/flow/test_cuckoo.py @@ -1,3 +1,4 @@ +import random from common import * @@ -346,6 +347,19 @@ def test_params(self): self.assertRaises(ResponseError, self.cmd, 'CF.LOADCHUNK err iterator') # missing data self.assertRaises(ResponseError, self.cmd, 'CF.SCANDUMP err') + def test_reserve_limits(self): + self.cmd('FLUSHALL') + self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE cf 100 BUCKETSIZE 33554432') + self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE cf 100 MAXITERATIONS 165536') + self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE cf 100 EXPANSION 327695') + self.assertRaises(ResponseError, self.cmd, 'CF.RESERVE CF 67108864 BUCKETSIZE 33554432 MAXITERATIONS 1337 EXPANSION 1337') + + self.cmd('CF.RESERVE cf 67108864 BUCKETSIZE 255 MAXITERATIONS 65535 EXPANSION 32768') + info = self.cmd('CF.INFO cf') + self.assertEqual(info[info.index('Bucket size') + 1], 255) + self.assertEqual(info[info.index('Expansion rate') + 1], 32768) + self.assertEqual(info[info.index('Max iterations') + 1], 65535) + class testCuckooNoCodec(): def __init__(self): self.env = Env(decodeResponses=False) @@ -454,3 +468,169 @@ def test_scandump_huge(self): # check loaded filter for x in range(6): self.assertEqual(1, self.cmd('cf.exists', 'cf', 'foo')) + + def test_scandump_with_content(self): + # Basic success scenario with content validation + + self.cmd('FLUSHALL') + self.cmd('cf.reserve', 'cf', 1024 * 1024 * 64) + + for x in range(1000): + self.cmd('cf.add', 'cf', 'foo' + str(x)) + for x in range(1000): + self.assertEqual(1, self.cmd('cf.exists', 'cf', 'foo' + str(x))) + + chunks = [] + while True: + last_pos = chunks[-1][0] if chunks else 0 + chunk = self.cmd('cf.scandump', 'cf', last_pos) + if not chunk[0]: + break + chunks.append(chunk) + + for chunk in chunks: + self.cmd('cf.loadchunk', 'cf2', *chunk) + + # check loaded filter + for x in range(1000): + self.assertEqual(1, self.cmd('cf.exists', 'cf2', 'foo' + str(x))) + + + def test_scandump_invalid(self): + self.cmd('FLUSHALL') + self.cmd('cf.reserve', 'cf', 4) + self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '-9223372036854775808', '1') + self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '922337203685477588', '1') + self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '4', 'kdoasdksaodsadsadsadsadsadadsadadsdad') + self.assertRaises(ResponseError, self.cmd, 'cf.loadchunk', 'cf', '4', 'abcd') + self.cmd('cf.add', 'cf', 'x') + self.assertRaises(ResponseError, self.cmd, 'cf.scandump', 'cf', '-1') + + + def test_scandump_invalid_header(self): + env = self.env + env.cmd('FLUSHALL') + + env.cmd('cf.reserve', 'cf', 100) + for x in range(50): + env.cmd('cf.add', 'cf', 'foo' + str(x)) + + chunk = env.cmd('cf.scandump', 'cf', 0) + env.cmd('del', 'cf') + + arr = bytearray(chunk[1]) + + # It corrupts first 8 bytes in the response. See struct CFHeader + # for internals. + for i in range(9): + arr[i] = 0 + + thrown = None + try: + env.cmd('cf.loadchunk', 'cf', 1, bytes(arr)) + except Exception as e: + thrown = e + + if thrown is None or str(thrown) != "Couldn't create filter!": + print("Exception was: " + str(thrown)) + assert False + + + def test_scandump_random_scan_small(self): + self.cmd('FLUSHALL') + self.cmd('cf.reserve', 'cf', 50) + + for i in range(0, 10000): + try: + self.cmd('cf.add', 'cf', 'x' + str(i)) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = self.cmd('CF.INFO', 'cf') + size = info[info.index(b'Size') + 1] + + for i in range(0, size + 1024): + self.cmd('cf.scandump', 'cf', i) + + + def test_scandump_scan_big(self): + self.cmd('FLUSHALL') + self.cmd('cf.reserve', 'cf', 1024, 'EXPANSION', 30000) + + for i in range(0, 100): + arr = [] + for j in range(0, 10000): + arr.append('x' + str(i) + str(j)) + + try: + self.cmd('cf.insert', 'cf', 'ITEMS', *arr) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = self.cmd('CF.INFO', 'cf') + size = info[info.index(b'Size') + 1] + + for i in range(0, 100): + self.cmd('cf.scandump', 'cf', random.randint(0, size * 2)) + + + def test_scandump_load_small(self): + self.cmd('FLUSHALL') + self.cmd('cf.reserve', 'cf', 10) + + for i in range(0, 100): + arr = [] + for j in range(0, 1000): + arr.append('x' + str(i) + str(j)) + + try: + self.cmd('cf.insert', 'cf', 'ITEMS', *arr) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = self.cmd('CF.INFO', 'cf') + size = info[info.index(b'Size') + 1] + + for i in range (0, size + 100): + b = bytearray(os.urandom(random.randint(0, 100))) + try: + self.cmd('cf.loadchunk', 'cf', random.randint(0, 10000), bytes(b)) + except Exception as e: + if (str(e) != "Couldn't load chunk!" and + str(e) != "Invalid position" and + str(e) != "item exists"): + raise e + + + def test_scandump_load_big(self): + self.cmd('FLUSHALL') + self.cmd('cf.reserve', 'cf', 1024, 'EXPANSION', 30000) + + for i in range(0, 100): + arr = [] + for j in range(0, 1000): + arr.append('x' + str(i) + str(j)) + + try: + self.cmd('cf.insert', 'cf', 'ITEMS', *arr) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = self.cmd('CF.INFO', 'cf') + size = info[info.index(b'Size') + 1] + + for i in range (0, 100): + b = bytearray(os.urandom(random.randint(1024, 36 * 1024 * 1024))) + try: + self.cmd('cf.loadchunk', 'cf', random.randint(2, size), bytes(b)) + except Exception as e: + if str(e) != "Couldn't load chunk!": + raise e diff --git a/tests/flow/test_overall.py b/tests/flow/test_overall.py index d29f4e79..0f5ac741 100644 --- a/tests/flow/test_overall.py +++ b/tests/flow/test_overall.py @@ -1,3 +1,4 @@ +import random from common import * @@ -425,6 +426,20 @@ def test_issue178(self): env.assertEqual(info["Capacity"], 300000000) env.assertEqual(info["Size"], 1132420232) + def test_very_high_error_rate(self): + env = self.env + env.cmd('FLUSHALL') + + env.cmd('bf.reserve', 'bf1', 0.99, 3, "NONSCALING") + env.cmd('bf.add', 'bf1', 1) + + env.cmd('bf.reserve', 'bf2', 0.95, 8, "NONSCALING") + env.cmd('bf.add', 'bf2', 1) + + env.cmd('bf.reserve', 'bf3', 0.9999999999999999, 100, "NONSCALING") + env.cmd('bf.add', 'bf3', 1) + + class testRedisBloomNoCodec(): def __init__(self): self.env = Env(decodeResponses=False) @@ -521,3 +536,225 @@ def test_scandump_huge(self): # check loaded filter for x in range(6): env.assertEqual(1, env.cmd('bf.exists', 'bf', 'foo')) + + + def test_scandump_with_content(self): + # Basic success scenario with content validation + + env = self.env + env.cmd('FLUSHALL') + + env.cmd('bf.reserve', 'bf', 0.01, 1024 * 1024 * 64) + for x in range(1000): + env.cmd('bf.add', 'bf', 'foo' + str(x)) + for x in range(1000): + env.assertEqual(1, env.cmd('bf.exists', 'bf', 'foo' + str(x))) + + chunks = [] + while True: + last_pos = chunks[-1][0] if chunks else 0 + chunk = env.cmd('bf.scandump', 'bf', last_pos) + if not chunk[0]: + break + chunks.append(chunk) + + env.cmd('del', 'bf') + + for chunk in chunks: + env.cmd('bf.loadchunk', 'bf2', *chunk) + + # Validate items in the loaded filter + for x in range(1000): + env.assertEqual(1, env.cmd('bf.exists', 'bf2', 'foo' + str(x))) + + + def test_scandump_invalid(self): + env = self.env + env.cmd('FLUSHALL') + env.cmd('bf.reserve', 'bf', 0.1, 4) + env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '-9223372036854775808', '1') + env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '922337203685477588', '1') + env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '4', 'kdoasdksaodsadsadsadsadsadadsadadsdad') + env.assertRaises(ResponseError, env.cmd, 'bf.loadchunk', 'bf', '4', 'abcd') + env.cmd('bf.add', 'bf', 'x') + env.cmd('bf.add', 'bf', 'y') + + + def test_scandump_invalid_header(self): + env = self.env + env.cmd('FLUSHALL') + + env.cmd('bf.reserve', 'bf', 0.01, 5) + for x in range(50): + env.cmd('bf.add', 'bf', 'foo' + str(x)) + + chunk = env.cmd('bf.scandump', 'bf', 0) + + env.cmd('del', 'bf') + arr = bytearray(chunk[1]) + + # See 'struct dumpedChainHeader' for internals. + # It corrupts second link in the response. + for i in range(8): + arr[72 + i] = 0 + + thrown = None + try: + env.cmd('bf.loadchunk', 'bf', 1, bytes(arr)) + except Exception as e: + thrown = e + + if thrown is None or str(thrown) != "received bad data": + raise thrown + + # It corrupts 'options' field in the response. + arr = bytearray(chunk[1]) + for i in range(4): + arr[12 + i] = 255 + + thrown = None + try: + env.cmd('bf.loadchunk', 'bf', 1, bytes(arr)) + except Exception as e: + thrown = e + + if thrown is None or str(thrown) != "received bad data": + raise thrown + + # It corrupts first field in the response. + arr = bytearray(chunk[1]) + for i in range(4): + arr[i] = 255 + + thrown = None + try: + env.cmd('bf.loadchunk', 'bf', 1, bytes(arr)) + except Exception as e: + thrown = e + + if thrown is None or str(thrown) != "received bad data": + raise thrown + + # It corrupts second link in the response. + arr = bytearray(chunk[1]) + for i in range(8): + arr[36 + i] = 255 + arr[0 + i] = 255 + + thrown = None + try: + env.cmd('bf.loadchunk', 'bf', 1, bytes(arr)) + except Exception as e: + thrown = e + + if thrown is None or str(thrown) != "received bad data": + raise thrown + + + def test_scandump_scan_small(self): + env = self.env + env.cmd('FLUSHALL') + env.cmd('bf.reserve', 'bf', 0.1, 50) + + for i in range(0, 1500): + try: + env.cmd('bf.add', 'bf', 'x' + str(i)) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = env.cmd('BF.INFO', 'bf') + size = info[info.index(b'Size') + 1] + + # Verify random scandump does not cause any problem + for i in range(0, size + 1024): + env.cmd('bf.scandump', 'bf', i) + + + def test_scandump_scan_big(self): + env = self.env + env.cmd('FLUSHALL') + env.cmd('bf.reserve', 'bf', 0.001, 1024, 'EXPANSION', 30000) + + for i in range(0, 100): + arr = [] + for j in range(0, 10000): + arr.append('x' + str(i) + str(j)) + + try: + env.cmd('bf.insert', 'bf', 'ITEMS', *arr) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = env.cmd('bf.INFO', 'bf') + size = info[info.index(b'Size') + 1] + + # Verify random scandump does not cause any problem + for i in range(0, 100): + env.cmd('bf.scandump', 'bf', random.randint(0, size * 2)) + + + def test_scandump_load_small(self): + env = self.env + env.cmd('FLUSHALL') + env.cmd('bf.reserve', 'bf', 0.01, 10) + + for i in range(0, 100): + arr = [] + for j in range(0, 1000): + arr.append('x' + str(i) + str(j)) + + try: + env.cmd('bf.insert', 'bf', 'ITEMS', *arr) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = env.cmd('BF.INFO', 'bf') + size = info[info.index(b'Size') + 1] + + # Try loading chunks with random size and content + for i in range (0, 100): + b = bytearray(os.urandom(random.randint(0, 4096))) + try: + env.cmd('bf.loadchunk', 'bf', random.randint(0, size * 2), bytes(b)) + except Exception as e: + if (str(e) != "invalid offset - no link found" and + str(e) != "invalid chunk - Too big for current filter" and + str(e) != "received bad data"): + raise e + + + def test_scandump_load_big(self): + env = self.env + env.cmd('FLUSHALL') + env.cmd('bf.reserve', 'bf', 0.01, 1024, 'EXPANSION', 30000) + + for i in range(0, 100): + arr = [] + for j in range(0, 1000): + arr.append('x' + str(i) + str(j)) + + try: + env.cmd('bf.insert', 'bf', 'ITEMS', *arr) + except ResponseError as e: + if str(e) == "Maximum expansions reached": + break + raise e + + info = env.cmd('BF.INFO', 'bf') + size = info[info.index(b'Size') + 1] + + # Try loading chunks with random size and content + for i in range (0, 100): + b = bytearray(os.urandom(random.randint(1024, 36 * 1024 * 1024))) + try: + env.cmd('bf.loadchunk', 'bf', random.randint(0, size), bytes(b)) + except Exception as e: + if (str(e) != "invalid offset - no link found" and + str(e) != "received bad data"): + raise e diff --git a/tests/flow/tests.sh b/tests/flow/tests.sh index 3e8f874f..4173c926 100755 --- a/tests/flow/tests.sh +++ b/tests/flow/tests.sh @@ -18,9 +18,9 @@ cd $HERE help() { cat <<-'END' Run flow tests - + [ARGVARS...] tests.sh [--help|help] [] - + Argument variables: MODULE=path Path to redisbloom.so MODARGS=args RediSearch module arguments @@ -32,7 +32,7 @@ help() { SLAVES=1 Tests with --test-slaves CLUSTER=1 Test with OSS cluster, one shard QUICK=1 Perform only GEN=1 test variant - + TEST=name Run specific test (e.g. test.py:test_name) TESTFILE=file Run tests listed in `file` FAILEDFILE=file Write failed tests into `file` @@ -55,7 +55,7 @@ help() { COV=1 Run with coverage analysis VG=1 Run with Valgrind VG_LEAKS=0 Do not detect leaks - SAN=type Use LLVM sanitizer (type=address|memory|leak|thread) + SAN=type Use LLVM sanitizer (type=address|memory|leak|thread) BB=1 Enable Python debugger (break using BB() in tests) GDB=1 Enable interactive gdb debugging (in single-test mode) @@ -159,42 +159,14 @@ setup_rltest() { #---------------------------------------------------------------------------------------------- -setup_clang_sanitizer() { - local ignorelist=$ROOT/tests/memcheck/redis.san-ignorelist - if ! grep THPIsEnabled $ignorelist &> /dev/null; then - echo "fun:THPIsEnabled" >> $ignorelist - fi - - # for RediSearch module - export RS_GLOBAL_DTORS=1 - +setup_sanitizer() { # for RLTest export SANITIZER="$SAN" export SHORT_READ_BYTES_DELTA=512 - + + export ASAN_OPTIONS="detect_odr_violation=0:halt_on_error=0:detect_leaks=1" # --no-output-catch --exit-on-failure --check-exitcode RLTEST_SAN_ARGS="--sanitizer $SAN" - - if [[ $SAN == addr || $SAN == address ]]; then - REDIS_SERVER=${REDIS_SERVER:-redis-server-asan-$SAN_REDIS_VER} - if ! command -v $REDIS_SERVER > /dev/null; then - echo Building Redis for clang-asan ... - $READIES/bin/getredis --force -v $SAN_REDIS_VER --own-openssl --no-run \ - --suffix asan --clang-asan --clang-san-blacklist $ignorelist - fi - - export ASAN_OPTIONS="detect_odr_violation=0:halt_on_error=0:detect_leaks=1" - export LSAN_OPTIONS="suppressions=$ROOT/tests/memcheck/asan.supp" - - elif [[ $SAN == mem || $SAN == memory ]]; then - REDIS_SERVER=${REDIS_SERVER:-redis-server-msan-$SAN_REDIS_VER} - if ! command -v $REDIS_SERVER > /dev/null; then - echo Building Redis for clang-msan ... - $READIES/bin/getredis --force -v $SAN_REDIS_VER --no-run --own-openssl \ - --suffix msan --clang-msan --llvm-dir /opt/llvm-project/build-msan \ - --clang-san-blacklist $ignorelist - fi - fi } #---------------------------------------------------------------------------------------------- @@ -211,12 +183,6 @@ setup_redis_server() { #---------------------------------------------------------------------------------------------- setup_valgrind() { - REDIS_SERVER=${REDIS_SERVER:-redis-server-vg} - if ! is_command $REDIS_SERVER; then - echo Building Redis for Valgrind ... - $READIES/bin/getredis -v $VALGRIND_REDIS_VER --valgrind --suffix vg - fi - if [[ $VG_LEAKS == 0 ]]; then VG_LEAK_CHECK=no RLTEST_VG_NOLEAKS="--vg-no-leakcheck" @@ -232,7 +198,7 @@ setup_valgrind() { --track-origins=yes \ --show-possibly-lost=no" - VALGRIND_SUPRESSIONS=$ROOT/tests/memcheck/valgrind.supp + VALGRIND_SUPRESSIONS=$ROOT/tests/memcheck/redis_valgrind.sup RLTEST_VG_ARGS+="\ --use-valgrind \ @@ -356,7 +322,7 @@ run_tests() { fi [[ $RLEC == 1 ]] && export RLEC_CLUSTER=1 - + local E=0 if [[ $NOP != 1 ]]; then { $OP python3 -m RLTest @$rltest_config; (( E |= $? )); } || true @@ -441,12 +407,6 @@ fi [[ $SAN == addr ]] && SAN=address [[ $SAN == mem ]] && SAN=memory -if [[ -n $TEST ]]; then - [[ $LOG != 1 ]] && RLTEST_LOG=0 - # export BB=${BB:-1} - export RUST_BACKTRACE=1 -fi - #-------------------------------------------------------------------------------- Platform Mode if [[ $PLATFORM_MODE == 1 ]]; then @@ -525,7 +485,7 @@ fi setup_rltest if [[ -n $SAN ]]; then - setup_clang_sanitizer + setup_sanitizer fi if [[ $VG == 1 ]]; then @@ -549,13 +509,13 @@ if [[ $GEN == 1 ]]; then { (run_tests "general"); (( E |= $? )); } || true fi if [[ $VG != 1 && $SLAVES == 1 ]]; then - { (RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves"); (( E |= $? )); } || true + { (RLTEST_ARGS+=" --use-slaves --enable-debug-command" run_tests "--use-slaves"); (( E |= $? )); } || true fi if [[ $AOF == 1 ]]; then - { (RLTEST_ARGS+=" --use-aof" run_tests "--use-aof"); (( E |= $? )); } || true + { (RLTEST_ARGS+=" --use-aof --enable-debug-command" run_tests "--use-aof"); (( E |= $? )); } || true fi if [[ $CLUSTER == 1 ]]; then - { (RLTEST_ARGS+=" --env oss-cluster --shards-count 1" run_tests "--env oss-cluster"); (( E |= $? )); } || true + { (RLTEST_ARGS+=" --env oss-cluster --shards-count 1 --enable-debug-command" run_tests "--env oss-cluster"); (( E |= $? )); } || true fi #-------------------------------------------------------------------------------------- Summary @@ -564,7 +524,7 @@ if [[ $NO_SUMMARY == 1 ]]; then exit 0 fi -if [[ $NOP != 1 && -n $SAN ]]; then +if [[ $NOP != 1 ]]; then if [[ -n $SAN || $VG == 1 ]]; then { FLOW=1 $ROOT/sbin/memcheck-summary; (( E |= $? )); } || true fi diff --git a/tests/memcheck/redis_valgrind.sup b/tests/memcheck/redis_valgrind.sup index a44aedf6..b05843d8 100644 --- a/tests/memcheck/redis_valgrind.sup +++ b/tests/memcheck/redis_valgrind.sup @@ -1,152 +1,17 @@ { - + Memcheck:Cond fun:lzf_compress } { - + Memcheck:Value4 fun:lzf_compress } { - + Memcheck:Value8 fun:lzf_compress } - -{ - - Memcheck:Value8 - fun:crcspeed64little - fun:crcspeed64native - fun:crc64 - fun:createDumpPayload - fun:dumpCommand -} - - -{ - - Memcheck:Value8 - fun:crcspeed64little - fun:crcspeed64native - fun:crc64 - fun:rioGenericUpdateChecksum - fun:rioWrite -} - - -{ - - Memcheck:Value8 - fun:crcspeed64little - fun:createDumpPayload - fun:dumpCommand -} - -{ - - Memcheck:Param - write(buf) - fun:__libc_write - fun:write - fun:connSocketWrite - fun:connWrite -} - -{ - - Memcheck:Addr8 - fun:appendBits - fun:appendFloat - fun:Compressed_Append - fun:Compressed_AddSample -} - -{ - - Memcheck:Addr8 - fun:appendBits - fun:appendInteger - fun:Compressed_Append - fun:Compressed_AddSample -} - -{ - - Memcheck:Addr1 - fun:raxLowWalk - fun:raxSeek - fun:RM_DictIteratorStartC -} - -{ - - Memcheck:Addr1 - fun:raxSeek - fun:RM_DictIteratorStartC -} - -{ - - Memcheck:Param - write(buf) - fun:__libc_write - fun:write - fun:_IO_file_write@@GLIBC_2.2.5 - fun:new_do_write - fun:_IO_new_do_write - fun:_IO_do_write@@GLIBC_2.2.5 - fun:_IO_new_file_xsputn - fun:_IO_file_xsputn@@GLIBC_2.2.5 - fun:__vfprintf_internal - fun:__fprintf_chk - fun:fprintf - fun:serverLogRaw - fun:RM_LogRaw - fun:RM_Log -} - -{ - - Memcheck:Cond - fun:strlen - fun:__vfprintf_internal - fun:__vsnprintf_internal - fun:RM_LogRaw - fun:RM_Log -} - -{ - - Memcheck:Cond - fun:strlen - fun:__vfprintf_internal - fun:__vsnprintf_internal - fun:vsnprintf - fun:RM_LogRaw - fun:RM_Log -} - -{ - - Memcheck:Cond - fun:strlen - fun:vfprintf - fun:fprintf - fun:serverLogRaw - fun:RM_LogRaw - fun:RM_Log -} - -{ - - Memcheck:Cond - fun:strlen - fun:vfprintf - fun:vsnprintf - fun:RM_LogRaw - fun:RM_Log -}