Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Fix NULL handling (mostly) 
* DATETIME 0000-00-00 00:00:00 is now valid and different from NULL
* DATETIME valid date range is now the same as MariaDB's
* VARCHAR '' is now separate from NULL

Known issue: VARCHAR "IS NULL" comparison does not work yet.
  • Loading branch information
Andrew Hutchings committed Aug 24, 2016
1 parent 91d2304 commit 0434612
Show file tree
Hide file tree
Showing 28 changed files with 249 additions and 264 deletions.
4 changes: 2 additions & 2 deletions dbcon/dmlpackage/calpontdmlfactory.cpp
View file
Expand Up @@ -174,15 +174,15 @@ dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysql
{
case DML_INSERT:
packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID());
(void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows());
(void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues());
break;
case DML_COMMAND:
packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID() );
break;
case DML_DELETE:
packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(),
vpackage.get_DMLStatement(), vpackage.get_SessionID() );
(void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows());
(void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues());
break;
default:
cerr << "makeCalpontDMLPackage: invalid statement type" << endl;
Expand Down
2 changes: 1 addition & 1 deletion dbcon/dmlpackage/calpontdmlpackage.h
View file
Expand Up @@ -94,7 +94,7 @@ namespace dmlpackage
* @param columns number of columns in the table
* @param rows number of rows to be touched
*/
virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows) = 0;
virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues) = 0;

/** @brief get the table object
*/
Expand Down
2 changes: 1 addition & 1 deletion dbcon/dmlpackage/commanddmlpackage.h
View file
Expand Up @@ -86,7 +86,7 @@ namespace dmlpackage
* @param colNameList, tableValuesMap
* @param rows the number of rows in the buffer
*/
int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows)
int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues)
{
return 1;
};
Expand Down
2 changes: 1 addition & 1 deletion dbcon/dmlpackage/deletedmlpackage.cpp
View file
Expand Up @@ -189,7 +189,7 @@ int DeleteDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows

}

int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows )
int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset &nullValues )
{
int retval = 1;

Expand Down
2 changes: 1 addition & 1 deletion dbcon/dmlpackage/deletedmlpackage.h
View file
Expand Up @@ -92,7 +92,7 @@ namespace dmlpackage
* @param colNameList, tableValuesMap
* @param rows the number of rows in the buffer
*/
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows);
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues);

protected:

Expand Down
2 changes: 2 additions & 0 deletions dbcon/dmlpackage/dmlpkg.h
View file
Expand Up @@ -29,6 +29,7 @@
#include <map>
#include <utility>
#include <iostream>
#include <bitset>
#include <stdint.h>

namespace dmlpackage
Expand Down Expand Up @@ -71,6 +72,7 @@ typedef std::vector<char*> QueryBuffer;
typedef std::vector<std::string> ColValuesList;
typedef std::vector<std::string> ColNameList;
typedef std::map<uint32_t, ColValuesList> TableValuesMap;
typedef std::bitset<4096> NullValuesBitset;

std::ostream& operator<<(std::ostream& os, const SqlStatementList& ct);
std::ostream& operator<<(std::ostream& os, const SqlStatement& stmt);
Expand Down
4 changes: 2 additions & 2 deletions dbcon/dmlpackage/insertdmlpackage.cpp
View file
Expand Up @@ -151,7 +151,7 @@ int InsertDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows
return retval;
}

int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows )
int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues )
{
int retval = 1;

Expand All @@ -166,7 +166,7 @@ int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValues

colValList = tableValuesMap[j];

DMLColumn* aColumn = new DMLColumn(colName, colValList, false);
DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]);
(aRowPtr->get_ColumnList()).push_back(aColumn);
}
//build a row list for a table
Expand Down
2 changes: 1 addition & 1 deletion dbcon/dmlpackage/insertdmlpackage.h
View file
Expand Up @@ -88,7 +88,7 @@ class InsertDMLPackage : public CalpontDMLPackage
* @param columns number of columns in the table
* @param rows number of rows to be touched
*/
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows);
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues);

/** @brief build a InsertDMLPackage from a InsertSqlStatement
*
Expand Down
4 changes: 2 additions & 2 deletions dbcon/dmlpackage/updatedmlpackage.cpp
View file
Expand Up @@ -206,7 +206,7 @@ int UpdateDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows

return retval;
}
int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows )
int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues )
{
int retval = 1;

Expand All @@ -221,7 +221,7 @@ int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValues

colValList = tableValuesMap[j];

DMLColumn* aColumn = new DMLColumn(colName, colValList, false);
DMLColumn* aColumn = new DMLColumn(colName, colValList, nullValues[j]);
(aRowPtr->get_ColumnList()).push_back(aColumn);
}
//build a row list for a table
Expand Down
2 changes: 1 addition & 1 deletion dbcon/dmlpackage/updatedmlpackage.h
View file
Expand Up @@ -92,7 +92,7 @@ namespace dmlpackage
* @param colNameList, tableValuesMap
* @param rows the number of rows in the buffer
*/
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows);
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues);
void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt );


Expand Down
4 changes: 2 additions & 2 deletions dbcon/dmlpackage/vendordmlstatement.cpp
View file
Expand Up @@ -48,10 +48,10 @@ namespace dmlpackage
{}

VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns,
ColNameList& colNameList, TableValuesMap& tableValuesMap, int sessionID)
ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID)
:fDMLStatement(dmlstatement), fDMLStatementType(stmttype),
fTableName(tName), fSchema(schema), fRows(rows), fColumns(columns),
fColNameList(colNameList), fTableValuesMap(tableValuesMap), fSessionID(sessionID), fLogging(true),fLogending(true)
fColNameList(colNameList), fTableValuesMap(tableValuesMap), fNullValues(nullValues), fSessionID(sessionID), fLogging(true),fLogending(true)
{}

VendorDMLStatement::~VendorDMLStatement()
Expand Down
8 changes: 6 additions & 2 deletions dbcon/dmlpackage/vendordmlstatement.h
View file
Expand Up @@ -26,6 +26,7 @@
#include <string>
#include <vector>
#include <map>
#include <bitset>
#include <stdint.h>

#if defined(_MSC_VER) && defined(xxxVENDORDMLSTATEMENT_DLLEXPORT)
Expand All @@ -38,6 +39,7 @@ namespace dmlpackage
typedef std::vector<std::string> ColValuesList;
typedef std::vector<std::string> ColNameList;
typedef std::map<uint32_t, ColValuesList> TableValuesMap;
typedef std::bitset<4096> NullValuesBitset;

/** @brief describes the general interface
* and implementation of a Vendor DML Statement
Expand All @@ -63,7 +65,7 @@ namespace dmlpackage
/** @brief ctor for mysql
*/
EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns,
ColNameList& colNameList, TableValuesMap& tableValuesMap, int sessionID);
ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID);

/** @brief destructor
*/
Expand Down Expand Up @@ -130,6 +132,8 @@ namespace dmlpackage

/** @brief Set the session ID
*/

inline NullValuesBitset& get_nullValues() { return fNullValues; }
inline void set_SessionID( int value ) { fSessionID = value; }

inline ColNameList& get_ColNames() { return fColNameList; }
Expand Down Expand Up @@ -172,10 +176,10 @@ namespace dmlpackage
std::string fDataBuffer;
ColNameList fColNameList;
TableValuesMap fTableValuesMap;
NullValuesBitset fNullValues;
int fSessionID;
bool fLogging;
bool fLogending;

};

}
Expand Down
3 changes: 0 additions & 3 deletions dbcon/execplan/aggregatecolumn.cpp
View file
Expand Up @@ -342,9 +342,6 @@ void AggregateColumn::evaluate(Row& row, bool& isNull)
isNull = true;
else
fResult.strVal = row.getStringField(fInputIndex);
// stringColVal is padded with '\0' to colWidth so can't use str.length()
if (strlen(fResult.strVal.c_str()) == 0)
isNull = true;
break;
}
if (fResultType.colDataType == CalpontSystemCatalog::STRINT)
Expand Down
7 changes: 7 additions & 0 deletions dbcon/execplan/constantcolumn.cpp
View file
Expand Up @@ -58,6 +58,13 @@ ConstantColumn::ConstantColumn(const string& sql, TYPE type) :
fResult.uintVal = uint64ToStr(*((uint64_t *) tmp));
fResult.intVal = (int64_t)fResult.uintVal;
}
else if ((type == NULLDATA) && sql.length() > 0 && sql.length() < 9)
{
memcpy(tmp, sql.c_str(), sql.length());
memset(tmp+sql.length(), 0, 8);
fResult.uintVal = uint64ToStr(*((uint64_t *) tmp));
fResult.intVal = (int64_t)fResult.uintVal;
}
else
{
fResult.intVal = atoll(sql.c_str());
Expand Down
3 changes: 0 additions & 3 deletions dbcon/execplan/windowfunctioncolumn.cpp
View file
Expand Up @@ -399,9 +399,6 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull)
isNull = true;
else
fResult.strVal = row.getStringField(fInputIndex);
// stringColVal is padded with '\0' to colWidth so can't use str.length()
if (strlen(fResult.strVal.c_str()) == 0)
isNull = true;
break;
}
if (fResultType.colDataType == CalpontSystemCatalog::STRINT)
Expand Down
4 changes: 2 additions & 2 deletions dbcon/joblist/jlf_execplantojoblist.cpp
View file
Expand Up @@ -234,8 +234,8 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct)
}
else
{
WriteEngine::Token t = boost::any_cast<WriteEngine::Token>(anyVal);
n = *(uint64_t*)&t;
const string &i = boost::any_cast<string>(anyVal);
n = *((uint64_t*) i.data());
}
break;

Expand Down
2 changes: 1 addition & 1 deletion dbcon/joblist/tupleconstantstep.cpp
View file
Expand Up @@ -193,7 +193,7 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo)
if (types[*i] == CalpontSystemCatalog::CHAR ||
types[*i] == CalpontSystemCatalog::VARCHAR)
{
fRowConst.setStringField("", *i);
fRowConst.setUintField(fRowConst.getNullValue(*i), *i);
}
else if (isUnsigned(types[*i]))
{
Expand Down
5 changes: 4 additions & 1 deletion dbcon/mysql/ha_calpont_dml.cpp
View file
Expand Up @@ -187,15 +187,18 @@ uint32_t buildValueList (TABLE* table, cal_connection_info& ci )
uint32_t size=0;
int columnPos = 0;
double dbval;
ci.nullValuesBitset.reset();
for (Field** field = table->field; *field; field++)
{
if((*field)->is_null())
{
ci.tableValuesMap[columnPos].push_back (""); //currently, empty string is treated as null.
ci.nullValuesBitset[columnPos] = true;
}
else
{
bitmap_set_bit(table->read_set, (*field)->field_index);
ci.nullValuesBitset[columnPos] = false;
// @bug 3798 get real value for float/double type
if ((*field)->result_type() == REAL_RESULT)
{
Expand Down Expand Up @@ -330,7 +333,7 @@ int doProcessInsertValues ( TABLE* table, uint32_t size, cal_connection_info& ci

VendorDMLStatement dmlStmts(idb_mysql_query_str(thd), DML_INSERT, table->s->table_name.str,
table->s->db.str, size, ci.colNameList.size(), ci.colNameList,
ci.tableValuesMap, sessionID);
ci.tableValuesMap, ci.nullValuesBitset, sessionID);

CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStmts);
//@Bug 2466 Move the clean up earlier to avoid the second insert in another session to get the data
Expand Down
31 changes: 29 additions & 2 deletions dbcon/mysql/ha_calpont_execplan.cpp
View file
Expand Up @@ -1455,15 +1455,42 @@ bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip
SimpleFilter *sf = new SimpleFilter();
boost::shared_ptr<Operator> sop(new PredicateOperator(ifp->func_name()));
ConstantColumn *lhs = 0;
string value;

if ((rhs->resultType().colDataType == CalpontSystemCatalog::CHAR ||
rhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR ||
rhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
rhs->resultType().colDataType == CalpontSystemCatalog::BLOB ||
rhs->resultType().colDataType == CalpontSystemCatalog::CLOB))
{
switch (rhs->resultType().colWidth) {
case 1: value = joblist::CHAR1NULL; break;
case 2: value.assign((char*)&joblist::CHAR2NULL, 2); break;
case 3:
case 4: value.assign((char*)&joblist::CHAR4NULL, 4); break;
case 5:
case 6:
case 7:
case 8: value.assign((char*)&joblist::CHAR8NULL, 8);
break;
default:
value = joblist::CPNULLSTRMARK;
break;
}
}
else
{
value = "";
}

if (ifp->functype() == Item_func::ISNULL_FUNC)
{
lhs = new ConstantColumn("", ConstantColumn::NULLDATA);
lhs = new ConstantColumn(value, ConstantColumn::NULLDATA);
sop.reset(new PredicateOperator("isnull"));
}
else if (ifp->functype() == Item_func::ISNOTNULL_FUNC)
{
lhs = new ConstantColumn("", ConstantColumn::NULLDATA);
lhs = new ConstantColumn(value, ConstantColumn::NULLDATA);
sop.reset(new PredicateOperator("isnotnull"));
}
else //if (ifp->functype() == Item_func::NOT_FUNC)
Expand Down
2 changes: 2 additions & 0 deletions dbcon/mysql/ha_calpont_impl_if.h
View file
Expand Up @@ -195,6 +195,7 @@ typedef std::tr1::unordered_map<TABLE*, cal_table_info> CalTableMap;
typedef std::vector<std::string> ColValuesList;
typedef std::vector<std::string> ColNameList;
typedef std::map<uint32_t, ColValuesList> TableValuesMap;
typedef std::bitset<4096> NullValuesBitset;
struct cal_connection_info
{
enum AlterTableState { NOT_ALTER, ALTER_SECOND_RENAME, ALTER_FIRST_RENAME };
Expand Down Expand Up @@ -257,6 +258,7 @@ struct cal_connection_info
ha_rows rowsHaveInserted;
ColNameList colNameList;
TableValuesMap tableValuesMap;
NullValuesBitset nullValuesBitset;
int rc;
uint32_t tableOid;
querystats::QueryStats stats;
Expand Down
13 changes: 11 additions & 2 deletions primitives/linux-port/column.cpp
View file
Expand Up @@ -345,7 +345,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival)
case CalpontSystemCatalog::VARBINARY:
//@bug 339 might be a token here
//TODO: what's up with the second const here?
return (*val == joblist::CHAR8NULL || 0xFFFFFFFFFFFFFFFELL == *val);
return (*val == joblist::CHAR8NULL || 0xFFFFFFFFFFFFFFFELL == *val || *val == (*((uint64_t *) joblist::CPNULLSTRMARK.c_str())));
case CalpontSystemCatalog::UBIGINT:
return (joblist::UBIGINTNULL == *val);
default:
Expand Down Expand Up @@ -517,7 +517,16 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int
else {
bool val2Null = isNullVal(width, type, (uint8_t *) &val2);
if (isNull == val2Null || (val2Null && COP == COMPARE_NE))
return colCompare_(val1, val2, COP, rf);
{
if (type == CalpontSystemCatalog::CHAR || type == CalpontSystemCatalog::VARCHAR)
{
return colCompare_(order_swap(val1), order_swap(val2), COP);
}
else
{
return colCompare_(val1, val2, COP, rf);
}
}
else
return false;
}
Expand Down
8 changes: 4 additions & 4 deletions utils/common/nullvaluemanip.cpp
View file
Expand Up @@ -58,8 +58,8 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
default:
throw logic_error("getNullValue() Can't return the NULL string");
default:
return (*((uint64_t *) joblist::CPNULLSTRMARK.c_str()));
}
break;
}
Expand Down Expand Up @@ -129,8 +129,8 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
default:
throw logic_error("getSignedNullValue() Can't return the NULL string");
default:
return (*((int64_t *) joblist::CPNULLSTRMARK.c_str()));
}
break;
}
Expand Down

0 comments on commit 0434612

Please sign in to comment.