Merge pull request #512 from cwida/rigger2

Rigger Test Fixes 2

src/execution/index/art/art.cpp

@@ -20,6 +20,7 @@ ART::ART(DataTable &table, vector<column_t> column_ids, vector<unique_ptr<Expres
 		is_little_endian = false;
 	}
 	switch (types[0]) {
+	case TypeId::BOOL:
 	case TypeId::INT8:
 	case TypeId::INT16:
 	case TypeId::INT32:

src/execution/index/art/art_key.cpp

@@ -79,6 +79,12 @@ uint64_t Key::EncodeDouble(double x) {
 Key::Key(unique_ptr<data_t[]> data, idx_t len) : len(len), data(move(data)) {
 }
 
+template <> unique_ptr<data_t[]> Key::CreateData(bool value, bool is_little_endian) {
+	auto data = unique_ptr<data_t[]>(new data_t[sizeof(value)]);
+	data[0] = value ? 1 : 0;
+	return data;
+}
+
 template <> unique_ptr<data_t[]> Key::CreateData(int8_t value, bool is_little_endian) {
 	auto data = unique_ptr<data_t[]>(new data_t[sizeof(value)]);
 	reinterpret_cast<uint8_t *>(data.get())[0] = value;

src/execution/join_hashtable.cpp

@@ -403,6 +403,11 @@ unique_ptr<ScanStructure> JoinHashTable::Probe(DataChunk &keys) {
 	// set up the scan structure
 	auto ss = make_unique<ScanStructure>(*this);
 
+	if (join_type != JoinType::INNER) {
+		ss->found_match = unique_ptr<bool[]>(new bool[STANDARD_VECTOR_SIZE]);
+		memset(ss->found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
+	}
+
 	// first prepare the keys for probing
 	const SelectionVector *current_sel;
 	ss->count = PrepareKeys(keys, ss->key_data, current_sel, ss->sel_vector);
@@ -417,11 +422,6 @@ unique_ptr<ScanStructure> JoinHashTable::Probe(DataChunk &keys) {
 	// now initialize the pointers of the scan structure based on the hashes
 	ApplyBitmask(hashes, *current_sel, ss->count, ss->pointers);
 
-	if (join_type != JoinType::INNER) {
-		ss->found_match = unique_ptr<bool[]>(new bool[STANDARD_VECTOR_SIZE]);
-		memset(ss->found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
-	}
-
 	// create the selection vector linking to only non-empty entries
 	idx_t count = 0;
 	auto pointers = FlatVector::GetData<data_ptr_t>(ss->pointers);

src/function/scalar/string/regexp.cpp

@@ -31,6 +31,19 @@ static inline re2::StringPiece CreateStringPiece(string_t &input) {
 	return re2::StringPiece(input.GetData(), input.GetSize());
 }
 
+struct RegexPartialMatch {
+	static inline bool Operation(const re2::StringPiece &input, RE2 &re) {
+		return RE2::PartialMatch(input, re);
+	}
+};
+
+struct RegexFullMatch {
+	static inline bool Operation(const re2::StringPiece &input, RE2 &re) {
+		return RE2::FullMatch(input, re);
+	}
+};
+
+template<class OP>
 static void regexp_matches_function(DataChunk &args, ExpressionState &state, Vector &result) {
 	auto &strings = args.data[0];
 	auto &patterns = args.data[1];
@@ -44,7 +57,7 @@ static void regexp_matches_function(DataChunk &args, ExpressionState &state, Vec
 	if (info.constant_pattern) {
 		// FIXME: this should be a unary loop
 		UnaryExecutor::Execute<string_t, bool, true>(strings, result, args.size(), [&](string_t input) {
-			return RE2::PartialMatch(CreateStringPiece(input), *info.constant_pattern);
+			return OP::Operation(CreateStringPiece(input), *info.constant_pattern);
 		});
 	} else {
 		BinaryExecutor::Execute<string_t, string_t, bool, true>(
@@ -53,7 +66,7 @@ static void regexp_matches_function(DataChunk &args, ExpressionState &state, Vec
 			    if (!re.ok()) {
 				    throw Exception(re.error());
 			    }
-			    return RE2::PartialMatch(CreateStringPiece(input), re);
+			    return OP::Operation(CreateStringPiece(input), re);
 		    });
 	}
 }
@@ -105,8 +118,10 @@ static void regexp_replace_function(DataChunk &args, ExpressionState &state, Vec
 }
 
 void RegexpFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction(ScalarFunction("regexp_full_match", {SQLType::VARCHAR, SQLType::VARCHAR}, SQLType::BOOLEAN,
+	                               regexp_matches_function<RegexFullMatch>, false, regexp_matches_get_bind_function));
 	set.AddFunction(ScalarFunction("regexp_matches", {SQLType::VARCHAR, SQLType::VARCHAR}, SQLType::BOOLEAN,
-	                               regexp_matches_function, false, regexp_matches_get_bind_function));
+	                               regexp_matches_function<RegexPartialMatch>, false, regexp_matches_get_bind_function));
 	set.AddFunction(ScalarFunction("regexp_replace", {SQLType::VARCHAR, SQLType::VARCHAR, SQLType::VARCHAR},
 	                               SQLType::VARCHAR, regexp_replace_function));
 }

src/include/duckdb/execution/index/art/art_key.hpp

@@ -49,6 +49,7 @@ class Key {
 	}
 };
 
+template <> unique_ptr<data_t[]> Key::CreateData(bool value, bool is_little_endian);
 template <> unique_ptr<data_t[]> Key::CreateData(int8_t value, bool is_little_endian);
 template <> unique_ptr<data_t[]> Key::CreateData(int16_t value, bool is_little_endian);
 template <> unique_ptr<data_t[]> Key::CreateData(int32_t value, bool is_little_endian);

src/optimizer/join_order_optimizer.cpp

@@ -711,15 +711,16 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 						query_graph.CreateEdge(filter_info->left_set, filter_info->right_set, filter_info);
 						query_graph.CreateEdge(filter_info->right_set, filter_info->left_set, filter_info);
 					} else {
+						continue;
 						// the sets are not disjoint, we create two sets of edges
-						auto left_difference = set_manager.Difference(filter_info->left_set, filter_info->right_set);
-						auto right_difference = set_manager.Difference(filter_info->right_set, filter_info->left_set);
-						// -> LEFT <-> RIGHT \ LEFT
-						query_graph.CreateEdge(filter_info->left_set, right_difference, filter_info);
-						query_graph.CreateEdge(right_difference, filter_info->left_set, filter_info);
-						// -> RIGHT <-> LEFT \ RIGHT
-						query_graph.CreateEdge(left_difference, filter_info->right_set, filter_info);
-						query_graph.CreateEdge(filter_info->right_set, left_difference, filter_info);
+						// auto left_difference = set_manager.Difference(filter_info->left_set, filter_info->right_set);
+						// auto right_difference = set_manager.Difference(filter_info->right_set, filter_info->left_set);
+						// // -> LEFT <-> RIGHT \ LEFT
+						// query_graph.CreateEdge(filter_info->left_set, right_difference, filter_info);
+						// query_graph.CreateEdge(right_difference, filter_info->left_set, filter_info);
+						// // -> RIGHT <-> LEFT \ RIGHT
+						// query_graph.CreateEdge(left_difference, filter_info->right_set, filter_info);
+						// query_graph.CreateEdge(filter_info->right_set, left_difference, filter_info);
 					}
 					continue;
 				}

src/parser/transform/expression/transform_operator.cpp

@@ -47,10 +47,10 @@ unique_ptr<ParsedExpression> Transformer::TransformBinaryOperator(string op, uni
 	children.push_back(move(right));
 
 	if (op == "~" || op == "!~") {
-		// rewrite SIMILAR TO into regexp_matches('asdf', '.*sd.*')
+		// rewrite 'asdf' SIMILAR TO '.*sd.*' into regexp_full_match('asdf', '.*sd.*')
 		bool invert_similar = op == "!~";
 
-		auto result = make_unique<FunctionExpression>(schema, "regexp_matches", children);
+		auto result = make_unique<FunctionExpression>(schema, "regexp_full_match", children);
 		if (invert_similar) {
 			return make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, move(result));
 		} else {
@@ -132,7 +132,7 @@ unique_ptr<ParsedExpression> Transformer::TransformAExpr(PGAExpr *root) {
 			return make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, move(compare_between));
 		}
 	} break;
-	// rewrite SIMILAR TO into regexp_matches('asdf', '.*sd.*')
+	// rewrite SIMILAR TO into regexp_full_match('asdf', '.*sd.*')
 	case PG_AEXPR_SIMILAR: {
 		auto left_expr = TransformExpression(root->lexpr);
 		auto right_expr = TransformExpression(root->rexpr);
@@ -160,7 +160,7 @@ unique_ptr<ParsedExpression> Transformer::TransformAExpr(PGAExpr *root) {
 			invert_similar = true;
 		}
 		const auto schema = DEFAULT_SCHEMA;
-		const auto regex_function = "regexp_matches";
+		const auto regex_function = "regexp_full_match";
 		auto result = make_unique<FunctionExpression>(schema, regex_function, children);
 
 		if (invert_similar) {

src/parser/transform/statement/transform_create_table.cpp

@@ -29,8 +29,9 @@ unique_ptr<CreateStatement> Transformer::TransformCreateTable(PGNode *node) {
 	    stmt->oncommit != PGOnCommitAction::PG_ONCOMMIT_NOOP) {
 		throw NotImplementedException("Only ON COMMIT PRESERVE ROWS is supported");
 	}
-
-	assert(stmt->tableElts);
+	if (!stmt->tableElts) {
+		throw ParserException("Table must have at least one column!");
+	}
 
 	for (auto c = stmt->tableElts->head; c != NULL; c = lnext(c)) {
 		auto node = reinterpret_cast<PGNode *>(c->data.ptr_value);

src/parser/transform/tableref/transform_join.cpp

@@ -22,6 +22,10 @@ unique_ptr<TableRef> Transformer::TransformJoin(PGJoinExpr *root) {
 		result->type = JoinType::OUTER;
 		break;
 	}
+	case PG_JOIN_RIGHT: {
+		result->type = JoinType::RIGHT;
+		break;
+	}
 	case PG_JOIN_SEMI: {
 		result->type = JoinType::SEMI;
 		break;

src/planner/binder/tableref/plan_joinref.cpp

@@ -143,6 +143,10 @@ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, uni
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 	auto left = CreatePlan(*ref.left);
 	auto right = CreatePlan(*ref.right);
+	if (ref.type == JoinType::RIGHT) {
+		ref.type = JoinType::LEFT;
+		std::swap(left, right);
+	}
 
 	if (ref.type == JoinType::INNER) {
 		// inner join, generate a cross product + filter

test/rigger/test_rigger.cpp

@@ -38,8 +38,7 @@ TEST_CASE("Test queries found by Rigger that cause problems in other systems", "
 	SECTION("#8 Query with RIGHT JOIN causes a server panic") {
 		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 INT);"));
 		REQUIRE_NO_FAIL(con.Query("CREATE VIEW v0(c0) AS SELECT 0 FROM t0 ORDER BY -t0.c0;"));
-		// FIXME: right join not supported
-		REQUIRE_FAIL(con.Query("SELECT * FROM v0 RIGHT JOIN t0 ON false;"));
+		REQUIRE_NO_FAIL(con.Query("SELECT * FROM v0 RIGHT JOIN t0 ON false;"));
 	}
 	// SQLite
 	SECTION("#15 './' LIKE './' does not match") {
@@ -195,4 +194,63 @@ TEST_CASE("Tests found by Rigger", "[rigger]") {
 		result = con.Query("SELECT t0.c0 FROM t0, t1 WHERE t1.c0 < t0.c0;");
 		REQUIRE(CHECK_COLUMN(result, 0, {}));
 	}
+	SECTION("503") {
+		// RIGHT JOIN with a predicate that compares two integer columns results in an "Unhandled type" error
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t1(c0 INT);"));
+		result = con.Query("SELECT * FROM t0 RIGHT JOIN t1 ON t0.c0!=t1.c0;");
+		REQUIRE(CHECK_COLUMN(result, 0, {}));
+		REQUIRE(CHECK_COLUMN(result, 1, {}));
+	}
+	SECTION("504") {
+		// INSERT results in an error "Not implemented: Cannot create data from this type"
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 BOOLEAN, c1 INT, PRIMARY KEY(c0, c1));"));
+		REQUIRE_NO_FAIL(con.Query("INSERT INTO t0(c1, c0) VALUES (0, 0);"));
+		result = con.Query("SELECT * FROM t0;");
+		REQUIRE(CHECK_COLUMN(result, 0, {false}));
+		REQUIRE(CHECK_COLUMN(result, 1, {0}));
+	}
+	SECTION("505") {
+		// A RIGHT JOIN unexpectedly fetches rows
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t1(c1 BOOLEAN);"));
+		REQUIRE_NO_FAIL(con.Query("INSERT INTO t0(c0) VALUES (1);"));
+		result = con.Query("SELECT * FROM t0 RIGHT JOIN t1 on true;");
+		REQUIRE(CHECK_COLUMN(result, 0, {}));
+		REQUIRE(CHECK_COLUMN(result, 1, {}));
+	}
+	SECTION("506") {
+		// Query results in an error "INTERNAL: Failed to bind column reference "c0" [5.0] (bindings: [6.0])"
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t1(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("SELECT * FROM t1 JOIN t0 ON t1.c0 < t1.c0 - t0.c0 WHERE t0.c0 <= t1.c0;"));
+		REQUIRE_NO_FAIL(con.Query("SELECT * FROM t1 JOIN t0 ON t0.c0 + t1.c0 < t1.c0 - t0.c0;"));
+	}
+	SECTION("507") {
+		// Creating an empty table results in a crash
+		REQUIRE_FAIL(con.Query("CREATE TABLE t0();"));
+	}
+	SECTION("508") {
+		// LEFT JOIN on column with NULL value results in a segmentation fault
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t1(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("INSERT INTO t0(c0) VALUES (0);"));
+		REQUIRE_NO_FAIL(con.Query("INSERT INTO t1(c0) VALUES (NULL);"));
+		result = con.Query("SELECT * FROM t1 LEFT JOIN t0 ON t0.c0=t1.c0;");
+		REQUIRE(CHECK_COLUMN(result, 0, {Value()}));
+		REQUIRE(CHECK_COLUMN(result, 1, {Value()}));
+	}
+	SECTION("510") {
+		// SIMILAR TO results in an incorrect result
+		REQUIRE_NO_FAIL(con.Query("CREATE TABLE t0(c0 INT);"));
+		REQUIRE_NO_FAIL(con.Query("INSERT INTO t0(c0) VALUES (-10);"));
+		result = con.Query("SELECT '-10' SIMILAR TO '0';");
+		REQUIRE(CHECK_COLUMN(result, 0, {false}));
+		result = con.Query("SELECT t0.c0 SIMILAR TO 0 FROM t0;");
+		REQUIRE(CHECK_COLUMN(result, 0, {false}));
+		result = con.Query("SELECT t0.c0 NOT SIMILAR TO 0 FROM t0;");
+		REQUIRE(CHECK_COLUMN(result, 0, {true}));
+		result = con.Query("SELECT * FROM t0 WHERE t0.c0 NOT SIMILAR TO 0;");
+		REQUIRE(CHECK_COLUMN(result, 0, {-10}));
+	}
 }

test/sql/constraints/test_primarykey.cpp

@@ -449,3 +449,19 @@ TEST_CASE("PRIMARY KEY constraint on multiple string columns with overlapping va
 	//! this should  work since it won't cause a duplicate
 	REQUIRE_NO_FAIL(con.Query("UPDATE tst SET b='hell' WHERE b='hel'"));
 }
+
+TEST_CASE("Multi-column boolean PRIMARY KEY constraint", "[constraints]") {
+	unique_ptr<QueryResult> result;
+	DuckDB db(nullptr);
+	Connection con(db);
+
+	REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i INTEGER, j BOOLEAN, PRIMARY KEY(i, j))"));
+	REQUIRE_NO_FAIL(con.Query("INSERT INTO integers VALUES (1, false), (1, true), (2, false)"));
+	// duplicate value!
+	REQUIRE_FAIL(con.Query("INSERT INTO integers VALUES (1, false)"));
+	REQUIRE_NO_FAIL(con.Query("INSERT INTO integers VALUES (2, true)"));
+
+	result = con.Query("SELECT * FROM integers ORDER BY 1, 2");
+	REQUIRE(CHECK_COLUMN(result, 0, {1, 1, 2, 2}));
+	REQUIRE(CHECK_COLUMN(result, 1, {false, true, false, true}));
+}

test/sql/function/test_regex.cpp

@@ -23,6 +23,11 @@ TEST_CASE("regex search test", "[regex]") {
 	// partial matches okay
 	result = con.Query("SELECT regexp_matches('asdf', 'sd')");
 	REQUIRE(CHECK_COLUMN(result, 0, {true}));
+	// full match requires entire match
+	result = con.Query("SELECT regexp_full_match('asdf', 'sd')");
+	REQUIRE(CHECK_COLUMN(result, 0, {false}));
+	result = con.Query("SELECT regexp_full_match('asdf', '.sd.')");
+	REQUIRE(CHECK_COLUMN(result, 0, {true}));
 
 	result = con.Query("SELECT regexp_matches('asdf', '^sdf$')");
 	REQUIRE(CHECK_COLUMN(result, 0, {false}));

test/sql/index/test_art_index.cpp

@@ -1746,13 +1746,8 @@ TEST_CASE("Index Exceptions", "[art]") {
 	REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i integer, j integer, k BOOLEAN)"));
 
 	REQUIRE_FAIL(con.Query("CREATE INDEX ON integers(i)"));
-
 	REQUIRE_FAIL(con.Query("CREATE INDEX i_index ON integers(i COLLATE \"de_DE\")"));
-
 	REQUIRE_FAIL(con.Query("CREATE INDEX i_index ON integers using blabla(i)"));
-
-	REQUIRE_FAIL(con.Query("CREATE INDEX i_index ON integers(k)"));
-
 	REQUIRE_FAIL(con.Query("CREATE INDEX i_index ON integers(f)"));
 }
 

test/sql/join/test_left_outer_join.cpp

@@ -24,6 +24,15 @@ TEST_CASE("Test LEFT OUTER JOIN", "[join]") {
 	REQUIRE(CHECK_COLUMN(result, 2, {1, 2, Value()}));
 	REQUIRE(CHECK_COLUMN(result, 3, {10, 20, Value()}));
 
+	// RIGHT OUTER JOIN is just LEFT OUTER JOIN but with arguments reversed
+	// with one caveat: SELECT * will project the columns of the LHS first!
+	result = con.Query("SELECT * FROM integers2 RIGHT OUTER JOIN integers ON "
+	                   "integers.i=integers2.k ORDER BY i");
+	REQUIRE(CHECK_COLUMN(result, 0, {1, 2, Value()}));
+	REQUIRE(CHECK_COLUMN(result, 1, {10, 20, Value()}));
+	REQUIRE(CHECK_COLUMN(result, 2, {1, 2, 3}));
+	REQUIRE(CHECK_COLUMN(result, 3, {2, 3, 4}));
+
 	// WHERE happens AFTER the join, thus [where k IS NOT NULL] filters out any tuples with generated NULL values from
 	// the LEFT OUTER JOIN. Because of this, this join is equivalent to an inner join.
 	result = con.Query("SELECT * FROM integers LEFT OUTER JOIN integers2 ON "

test/sql/simple/test_join.cpp

@@ -513,3 +513,35 @@ TEST_CASE("Test joins with various columns that are only used in the join", "[jo
 	result = con.Query("SELECT (TRUE OR a1.a=a2.b) FROM test a1, test a2 WHERE a1.a=11 AND a2.a>=10");
 	REQUIRE(CHECK_COLUMN(result, 0, {true, true, true}));
 }
+
+TEST_CASE("Test joins with comparisons involving both sides of the join", "[joins]") {
+	DuckDB db(nullptr);
+	Connection con(db);
+	unique_ptr<QueryResult> result;
+	con.EnableQueryVerification();
+
+	// create tables
+	REQUIRE_NO_FAIL(con.Query("CREATE TABLE test (a INTEGER, b INTEGER);"));
+	REQUIRE_NO_FAIL(con.Query("INSERT INTO test VALUES (4, 1), (2, 2)"));
+
+	REQUIRE_NO_FAIL(con.Query("CREATE TABLE test2 (b INTEGER, c INTEGER);"));
+	REQUIRE_NO_FAIL(con.Query("INSERT INTO test2 VALUES (1, 2), (3, 0)"));
+
+	result = con.Query("SELECT * FROM test JOIN test2 ON test.a+test2.c=test.b+test2.b");
+	REQUIRE(CHECK_COLUMN(result, 0, {4}));
+	REQUIRE(CHECK_COLUMN(result, 1, {1}));
+	REQUIRE(CHECK_COLUMN(result, 2, {3}));
+	REQUIRE(CHECK_COLUMN(result, 3, {0}));
+
+	result = con.Query("SELECT * FROM test LEFT JOIN test2 ON test.a+test2.c=test.b+test2.b ORDER BY 1");
+	REQUIRE(CHECK_COLUMN(result, 0, {2, 4}));
+	REQUIRE(CHECK_COLUMN(result, 1, {2, 1}));
+	REQUIRE(CHECK_COLUMN(result, 2, {Value(), 3}));
+	REQUIRE(CHECK_COLUMN(result, 3, {Value(), 0}));
+
+	result = con.Query("SELECT * FROM test RIGHT JOIN test2 ON test.a+test2.c=test.b+test2.b ORDER BY 1");
+	REQUIRE(CHECK_COLUMN(result, 0, {Value(), 4}));
+	REQUIRE(CHECK_COLUMN(result, 1, {Value(), 1}));
+	REQUIRE(CHECK_COLUMN(result, 2, {1, 3}));
+	REQUIRE(CHECK_COLUMN(result, 3, {2, 0}));
+}

test/sql/simple/test_similar.cpp

@@ -54,10 +54,13 @@ TEST_CASE("Test scalar SIMILAR TO statement", "[similar]") {
 	result = connection.Query("SELECT 'aaa' !~ 'bbb'");
 	REQUIRE(CHECK_COLUMN(result, 0, {Value::BOOLEAN(true)}));
 
+	// similar to must match entire expression
 	result = connection.Query("SELECT 'aaa' ~ '^a'");
+	REQUIRE(CHECK_COLUMN(result, 0, {Value::BOOLEAN(false)}));
+	result = connection.Query("SELECT 'aaa' ~ '^a+'");
 	REQUIRE(CHECK_COLUMN(result, 0, {Value::BOOLEAN(true)}));
 
-	result = connection.Query("SELECT 'aaa' ~ '(a|b)'");
+	result = connection.Query("SELECT 'aaa' ~ '(a|b)*'");
 	REQUIRE(CHECK_COLUMN(result, 0, {Value::BOOLEAN(true)}));
 
 	result = connection.Query("SELECT 'abc' ~ '^(b|c)'");