Skip to content
Permalink
Browse files Browse the repository at this point in the history
Fix malformed read/write in BufferedReader
read_strings and read affected
  • Loading branch information
xzkostyan committed Sep 11, 2020
1 parent d708ed5 commit 3e99054
Show file tree
Hide file tree
Showing 8 changed files with 1,266 additions and 394 deletions.
31 changes: 31 additions & 0 deletions .travis.yml
Expand Up @@ -65,6 +65,37 @@ after_success:

jobs:
include:
- stage: valgrind
name: Valgrind check
os: linux
language: python
python:
- "3.6"
addons:
apt:
packages:
- valgrind
install:
- docker run -e "TZ=Europe/Moscow" -d -p 127.0.0.1:9000:9000 --name test-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:$VERSION
- docker run -d --entrypoint "/bin/sh" --name test-clickhouse-client --link test-clickhouse-server:clickhouse-server yandex/clickhouse-client:$VERSION -c 'while :; do sleep 1; done'
- docker ps -a
# Faking clickhouse-client real communication with container via docker exec.
- echo -e '#!/bin/bash\n\ndocker exec -e "`env | grep ^TZ=`" test-clickhouse-client clickhouse-client "$@"' | sudo tee /usr/local/bin/clickhouse-client > /dev/null
- sudo chmod +x /usr/local/bin/clickhouse-client
# Overriding setup.cfg. Set host=clickhouse-server
- sed -i 's/^host=localhost$/host=clickhouse-server/' setup.cfg
# Make host think that clickhouse-server is localhost
- echo '127.0.0.1 clickhouse-server' | sudo tee /etc/hosts > /dev/null
- pip install --upgrade pip setuptools
- pip install cython

script:
- valgrind --error-exitcode=1 python setup.py test

env:
- VERSION=20.3.7.46
- PYTHONMALLOC=malloc

- stage: wheels
name: Wheels for Linux
os: linux
Expand Down
730 changes: 541 additions & 189 deletions clickhouse_driver/bufferedreader.c

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions clickhouse_driver/bufferedreader.pyx
Expand Up @@ -8,7 +8,7 @@ from libc.string cimport memcpy


cdef class BufferedReader(object):
cdef public Py_ssize_t position, current_buffer_size
cdef public unsigned long long position, current_buffer_size
cdef public bytearray buffer

def __init__(self, bufsize):
Expand All @@ -22,17 +22,17 @@ cdef class BufferedReader(object):
def read_into_buffer(self):
raise NotImplementedError

def read(self, Py_ssize_t unread):
def read(self, unsigned long long unread):
# When the buffer is large enough bytes read are almost
# always hit the buffer.
cdef Py_ssize_t next_position = unread + self.position
cdef unsigned long long next_position = unread + self.position
if next_position < self.current_buffer_size:
t = self.position
self.position = next_position
return bytes(self.buffer[t:self.position])

cdef char* buffer_ptr = PyByteArray_AsString(self.buffer)
cdef Py_ssize_t read_bytes
cdef unsigned long long read_bytes
rv = bytes()

while unread > 0:
Expand All @@ -59,24 +59,24 @@ cdef class BufferedReader(object):
self.position += 1
return rv

def read_strings(self, Py_ssize_t n_items, encoding=None):
def read_strings(self, unsigned long long n_items, encoding=None):
"""
Python has great overhead between function calls.
We inline strings reading logic here to avoid this overhead.
"""
items = PyTuple_New(n_items)

cdef Py_ssize_t i
cdef unsigned long long i
# Buffer vars
cdef char* buffer_ptr = PyByteArray_AsString(self.buffer)
cdef Py_ssize_t right
cdef unsigned long long right
# String length vars
cdef Py_ssize_t size, shift, bytes_read
cdef unsigned char b
cdef unsigned long long size, shift, bytes_read
cdef unsigned long long b

# String for decode vars.
cdef char *c_string = NULL
cdef Py_ssize_t c_string_size = 1024
cdef unsigned long long c_string_size = 1024
cdef char *c_encoding = NULL
if encoding:
encoding = encoding.encode('utf-8')
Expand Down

0 comments on commit 3e99054

Please sign in to comment.