Skip to content

Commit

Permalink
ORC-372: Enable valgrind for C++ travis-ci tests
Browse files Browse the repository at this point in the history 
Fixes #277

Signed-off-by: Owen O'Malley <omalley@apache.org>
  • Loading branch information
@omalley
Deepak Majeti authored and omalley committed Jun 5, 2018
1 parent 0a815b5 commit e159f05
Show file tree
Hide file tree
Showing 10 changed files with 128 additions and 84 deletions.
11 changes: 10 additions & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,21 @@
language: cpp
sudo: false
dist: trusty
addons:
apt:
packages:
- valgrind

matrix:
fast_finish: true
include:
- compiler: gcc
os: linux
script:
- mkdir build
- cd build
- cmake -DANALYZE_JAVA=ON -DTEST_VALGRIND_MEMCHECK=ON ..
- travis_wait 35 make package test-out
- compiler: clang
os: linux
addons:
Expand Down Expand Up @@ -44,4 +53,4 @@ script:
- mkdir build
- cd build
- cmake -DANALYZE_JAVA=ON ..
- make package test-out
- make package test-out
4 changes: 4 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ option(BUILD_TOOLS
"Build the tools"
ON)

option(TEST_VALGRIND_MEMCHECK
"Run the test suite using valgrind --tool=memcheck"
OFF)

option(INSTALL_VENDORED_LIBS
"Install vendored thirdparty libraries along with liborc"
ON)
Expand Down
9 changes: 9 additions & 0 deletions c++/src/Compression.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,13 +199,18 @@ namespace orc {
uint64_t blockSize,
MemoryPool& pool);

virtual ~ZlibCompressionStream() override {
end();
}

virtual std::string getName() const override;

protected:
virtual uint64_t doStreamingCompression() override;

private:
void init();
void end();
z_stream strm;
};

Expand Down Expand Up @@ -282,6 +287,10 @@ DIAGNOSTIC_PUSH
}
}

void ZlibCompressionStream::end() {
(void)deflateEnd(&strm);
}

DIAGNOSTIC_PUSH

enum DecompressState { DECOMPRESS_HEADER,
Expand Down
131 changes: 71 additions & 60 deletions c++/src/TypeImpl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,31 +258,34 @@ namespace orc {

case STRUCT: {
StructVectorBatch *result = new StructVectorBatch(capacity, memoryPool);
std::unique_ptr<ColumnVectorBatch> return_value = std::unique_ptr<ColumnVectorBatch>(result);
for(uint64_t i=0; i < getSubtypeCount(); ++i) {
result->fields.push_back(getSubtype(i)->
createRowBatch(capacity,
memoryPool).release());
}
return std::unique_ptr<ColumnVectorBatch>(result);
return return_value;
}

case LIST: {
ListVectorBatch* result = new ListVectorBatch(capacity, memoryPool);
std::unique_ptr<ColumnVectorBatch> return_value = std::unique_ptr<ColumnVectorBatch>(result);
if (getSubtype(0) != nullptr) {
result->elements = getSubtype(0)->createRowBatch(capacity, memoryPool);
}
return std::unique_ptr<ColumnVectorBatch>(result);
return return_value;
}

case MAP: {
MapVectorBatch* result = new MapVectorBatch(capacity, memoryPool);
std::unique_ptr<ColumnVectorBatch> return_value = std::unique_ptr<ColumnVectorBatch>(result);
if (getSubtype(0) != nullptr) {
result->keys = getSubtype(0)->createRowBatch(capacity, memoryPool);
}
if (getSubtype(1) != nullptr) {
result->elements = getSubtype(1)->createRowBatch(capacity, memoryPool);
}
return std::unique_ptr<ColumnVectorBatch>(result);
return return_value;
}

case DECIMAL: {
Expand All @@ -297,12 +300,13 @@ namespace orc {

case UNION: {
UnionVectorBatch *result = new UnionVectorBatch(capacity, memoryPool);
std::unique_ptr<ColumnVectorBatch> return_value = std::unique_ptr<ColumnVectorBatch>(result);
for(uint64_t i=0; i < getSubtypeCount(); ++i) {
result->children.push_back(getSubtype(i)->createRowBatch(capacity,
memoryPool)
.release());
}
return std::unique_ptr<ColumnVectorBatch>(result);
return return_value;
}

default:
Expand Down Expand Up @@ -330,16 +334,18 @@ namespace orc {

std::unique_ptr<Type> createListType(std::unique_ptr<Type> elements) {
TypeImpl* result = new TypeImpl(LIST);
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(result);
result->addChildType(std::move(elements));
return std::unique_ptr<Type>(result);
return return_value;
}

std::unique_ptr<Type> createMapType(std::unique_ptr<Type> key,
std::unique_ptr<Type> value) {
TypeImpl* result = new TypeImpl(MAP);
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(result);
result->addChildType(std::move(key));
result->addChildType(std::move(value));
return std::unique_ptr<Type>(result);
return return_value;
}

std::unique_ptr<Type> createUnionType() {
Expand Down Expand Up @@ -379,6 +385,7 @@ namespace orc {
case proto::Type_Kind_MAP:
case proto::Type_Kind_UNION: {
TypeImpl* result = new TypeImpl(static_cast<TypeKind>(type.kind()));
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(result);
if (type.kind() == proto::Type_Kind_LIST && type.subtypes_size() != 1)
throw ParseError("Illegal LIST type that doesn't contain one subtype");
if (type.kind() == proto::Type_Kind_MAP && type.subtypes_size() != 2)
Expand All @@ -390,11 +397,12 @@ namespace orc {
(type.subtypes(i))),
footer));
}
return std::unique_ptr<Type>(result);
return return_value;
}

case proto::Type_Kind_STRUCT: {
TypeImpl* result = new TypeImpl(STRUCT);
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(result);
uint64_t size = static_cast<uint64_t>(type.subtypes_size());
std::vector<Type*> typeList(size);
std::vector<std::string> fieldList(size);
Expand All @@ -404,7 +412,7 @@ namespace orc {
(type.subtypes(i))),
footer));
}
return std::unique_ptr<Type>(result);
return return_value;
}
default:
throw NotImplementedYet("Unknown type kind");
Expand Down Expand Up @@ -498,76 +506,80 @@ namespace orc {
}

ORC_UNIQUE_PTR<Type> Type::buildTypeFromString(const std::string& input) {
std::vector<std::pair<std::string, Type*> > res =
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type> > > res =
TypeImpl::parseType(input, 0, input.size());
if (res.size() != 1) {
throw std::logic_error("Invalid type string.");
}
return ORC_UNIQUE_PTR<Type>(res[0].second);
return std::move(res[0].second);
}

Type* TypeImpl::parseArrayType(const std::string &input,
size_t start,
size_t end) {
std::unique_ptr<Type> TypeImpl::parseArrayType(const std::string &input,
size_t start,
size_t end) {
TypeImpl* arrayType = new TypeImpl(LIST);
std::vector<std::pair<std::string, Type*> > v =
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(arrayType);
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type> > > v =
TypeImpl::parseType(input, start, end);
if (v.size() != 1) {
throw std::logic_error("Array type must contain exactly one sub type.");
}
arrayType->addChildType(ORC_UNIQUE_PTR<Type>(v[0].second));
return arrayType;
arrayType->addChildType(std::move(v[0].second));
return return_value;
}

Type* TypeImpl::parseMapType(const std::string &input,
size_t start,
size_t end) {
std::unique_ptr<Type> TypeImpl::parseMapType(const std::string &input,
size_t start,
size_t end) {
TypeImpl * mapType = new TypeImpl(MAP);
std::vector<std::pair<std::string, Type*> > v =
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(mapType);
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type> > > v =
TypeImpl::parseType(input, start, end);
if (v.size() != 2) {
throw std::logic_error(
"Map type must contain exactly two sub types.");
}
mapType->addChildType(ORC_UNIQUE_PTR<Type>(v[0].second));
mapType->addChildType(ORC_UNIQUE_PTR<Type>(v[1].second));
return mapType;
mapType->addChildType(std::move(v[0].second));
mapType->addChildType(std::move(v[1].second));
return return_value;
}

Type* TypeImpl::parseStructType(const std::string &input,
size_t start,
size_t end) {
std::unique_ptr<Type> TypeImpl::parseStructType(const std::string &input,
size_t start,
size_t end) {
TypeImpl* structType = new TypeImpl(STRUCT);
std::vector<std::pair<std::string, Type*> > v =
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(structType);
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type>> > v =
TypeImpl::parseType(input, start, end);
if (v.size() == 0) {
throw std::logic_error(
"Struct type must contain at least one sub type.");
}
for (size_t i = 0; i < v.size(); ++i) {
structType->addStructField(v[i].first, ORC_UNIQUE_PTR<Type>(v[i].second));
structType->addStructField(v[i].first, std::move(v[i].second));
}
return structType;
return return_value;
}

Type* TypeImpl::parseUnionType(const std::string &input,
size_t start,
size_t end) {
std::unique_ptr<Type> TypeImpl::parseUnionType(const std::string &input,
size_t start,
size_t end) {
TypeImpl* unionType = new TypeImpl(UNION);
std::vector<std::pair<std::string, Type*> > v =
std::unique_ptr<Type> return_value = std::unique_ptr<Type>(unionType);
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type> > > v =
TypeImpl::parseType(input, start, end);
if (v.size() == 0) {
throw std::logic_error("Union type must contain at least one sub type.");
}
for (size_t i = 0; i < v.size(); ++i) {
unionType->addChildType(ORC_UNIQUE_PTR<Type>(v[i].second));
unionType->addChildType(std::move(v[i].second));
}
return unionType;
return return_value;
}

Type* TypeImpl::parseDecimalType(const std::string &input,
size_t start,
size_t end) {
std::unique_ptr<Type> TypeImpl::parseDecimalType(const std::string &input,
size_t start,
size_t end) {
size_t sep = input.find(',', start);
if (sep + 1 >= end || sep == std::string::npos) {
throw std::logic_error("Decimal type must specify precision and scale.");
Expand All @@ -576,33 +588,33 @@ namespace orc {
static_cast<uint64_t>(atoi(input.substr(start, sep - start).c_str()));
uint64_t scale =
static_cast<uint64_t>(atoi(input.substr(sep + 1, end - sep - 1).c_str()));
return new TypeImpl(DECIMAL, precision, scale);
return std::unique_ptr<Type>(new TypeImpl(DECIMAL, precision, scale));
}

Type* TypeImpl::parseCategory(std::string category,
const std::string &input,
size_t start,
size_t end) {
std::unique_ptr<Type> TypeImpl::parseCategory(std::string category,
const std::string &input,
size_t start,
size_t end) {
if (category == "boolean") {
return new TypeImpl(BOOLEAN);
return std::unique_ptr<Type>(new TypeImpl(BOOLEAN));
} else if (category == "tinyint") {
return new TypeImpl(BYTE);
return std::unique_ptr<Type>(new TypeImpl(BYTE));
} else if (category == "smallint") {
return new TypeImpl(SHORT);
return std::unique_ptr<Type>(new TypeImpl(SHORT));
} else if (category == "int") {
return new TypeImpl(INT);
return std::unique_ptr<Type>(new TypeImpl(INT));
} else if (category == "bigint") {
return new TypeImpl(LONG);
return std::unique_ptr<Type>(new TypeImpl(LONG));
} else if (category == "float") {
return new TypeImpl(FLOAT);
return std::unique_ptr<Type>(new TypeImpl(FLOAT));
} else if (category == "double") {
return new TypeImpl(DOUBLE);
return std::unique_ptr<Type>(new TypeImpl(DOUBLE));
} else if (category == "string") {
return new TypeImpl(STRING);
return std::unique_ptr<Type>(new TypeImpl(STRING));
} else if (category == "binary") {
return new TypeImpl(BINARY);
return std::unique_ptr<Type>(new TypeImpl(BINARY));
} else if (category == "timestamp") {
return new TypeImpl(TIMESTAMP);
return std::unique_ptr<Type>(new TypeImpl(TIMESTAMP));
} else if (category == "array") {
return parseArrayType(input, start, end);
} else if (category == "map") {
Expand All @@ -614,26 +626,26 @@ namespace orc {
} else if (category == "decimal") {
return parseDecimalType(input, start, end);
} else if (category == "date") {
return new TypeImpl(DATE);
return std::unique_ptr<Type>(new TypeImpl(DATE));
} else if (category == "varchar") {
uint64_t maxLength = static_cast<uint64_t>(
atoi(input.substr(start, end - start).c_str()));
return new TypeImpl(VARCHAR, maxLength);
return std::unique_ptr<Type>(new TypeImpl(VARCHAR, maxLength));
} else if (category == "char") {
uint64_t maxLength = static_cast<uint64_t>(
atoi(input.substr(start, end - start).c_str()));
return new TypeImpl(CHAR, maxLength);
return std::unique_ptr<Type>(new TypeImpl(CHAR, maxLength));
} else {
throw std::logic_error("Unknown type " + category);
}
}

std::vector<std::pair<std::string, Type *> > TypeImpl::parseType(
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type> > > TypeImpl::parseType(
const std::string &input,
size_t start,
size_t end) {
std::string types = input.substr(start, end - start);
std::vector<std::pair<std::string, Type *> > res;
std::vector<std::pair<std::string, ORC_UNIQUE_PTR<Type> > > res;
size_t pos = 0;

while (pos < types.size()) {
Expand Down Expand Up @@ -680,8 +692,7 @@ namespace orc {
}

std::string category = types.substr(pos, endPos - pos);
Type* type = parseCategory(category, types, endPos + 1, nextPos);
res.push_back(std::make_pair(fieldName, type));
res.push_back(std::make_pair(fieldName, parseCategory(category, types, endPos + 1, nextPos)));

if (nextPos < types.size() && (types[nextPos] == ')' || types[nextPos] == '>')) {
pos = nextPos + 2;
Expand Down
Loading

0 comments on commit e159f05

Please sign in to comment.