Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ADBC] Add support for windows. #9357

Merged
merged 42 commits into from Dec 6, 2023
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
85084db
Adding support for Windows - ADBC
pdet Oct 13, 2023
8727f81
Merge remote-tracking branch 'origin/feature' into adbc_windows
pdet Oct 13, 2023
7099408
Adjust python tests
pdet Oct 13, 2023
4f9fa42
format tests
pdet Oct 16, 2023
1195fa0
run pwsh
pdet Oct 16, 2023
e0c8e2c
can we just run this?
pdet Oct 17, 2023
b3390c5
finding path? what breaks?
pdet Oct 17, 2023
51b7a74
set env directly
pdet Oct 18, 2023
fd6cdb9
Check if this variable is set
pdet Oct 18, 2023
87da024
try set test for each os
pdet Oct 18, 2023
414bdca
avoid ubsan from different types on parameters
pdet Oct 18, 2023
43dd5f0
move statement and cleanup to functions that need it, try to pring wi…
pdet Oct 18, 2023
f6011fe
try top status code def
pdet Oct 20, 2023
871306c
fix release error
pdet Oct 20, 2023
a8f1993
restricting duckdb_adbc namespacking
pdet Oct 23, 2023
b2444fc
ubsan on pointer mismatch
pdet Oct 23, 2023
0e96849
get schema function signature
pdet Oct 23, 2023
ff9f85e
remove ze wrapper
pdet Oct 23, 2023
1db4267
grr
pdet Oct 24, 2023
2a26be2
function signature
pdet Oct 25, 2023
036e09a
Move to outside namespace
pdet Oct 26, 2023
9b7857c
move code def
pdet Oct 26, 2023
7268663
Merge branch 'main' into adbc_windows
pdet Nov 29, 2023
a2d815b
change ptr to ref
pdet Nov 29, 2023
c8afb4b
Fixing memory leaks
pdet Nov 30, 2023
dd8eec3
More on memory leak fixes
pdet Nov 30, 2023
3e90956
format
pdet Nov 30, 2023
7a33fa7
ubsan fix
pdet Nov 30, 2023
ef44b4e
merge
pdet Nov 30, 2023
4476a81
fix c-api test
pdet Nov 30, 2023
ece0dc9
another sneaky ubsan
pdet Nov 30, 2023
4c6440e
ubsan
pdet Nov 30, 2023
9723c96
fix windows test
pdet Nov 30, 2023
d179447
Add patch
pdet Nov 30, 2023
b58f184
more ubsan
pdet Nov 30, 2023
7f3190b
print the test_dir on windows and ubsan
pdet Nov 30, 2023
2f36ec9
test fix and ubsan
pdet Nov 30, 2023
56751ff
check substring match
pdet Nov 30, 2023
c3cc8c1
grr
pdet Nov 30, 2023
2e3d3fe
whitelist
pdet Dec 1, 2023
e09ca39
merge
pdet Dec 4, 2023
26ae748
somehow scope of function was lost in merge
pdet Dec 4, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/Windows.yml
Expand Up @@ -70,7 +70,7 @@ jobs:
shell: bash
pdet marked this conversation as resolved.
Show resolved Hide resolved
if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
run: |
echo "DUCKDB_INSTALL_LIB=D:\a\duckdb\duckdb\src\Release\duckdb.dll" >> $env:GITHUB_ENV
echo "DUCKDB_INSTALL_LIB=$((Get-ChildItem -Recurse -Filter "duckdb.dll" | Select-Object -First 1).FullName)" >> $GITHUB_ENV
test/Release/unittest.exe

- name: Tools Test
Expand Down
5 changes: 3 additions & 2 deletions src/include/duckdb/common/adbc/adbc-init.hpp
Expand Up @@ -20,15 +20,16 @@
#ifndef DUCKDB_ADBC_INIT
#define DUCKDB_ADBC_INIT

#include "duckdb.h"
#include "duckdb/common/adbc/adbc.hpp"

#ifdef __cplusplus
extern "C" {
#endif

//! We gotta leak the symbols of the init function
duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::AdbcDriver *driver,
struct duckdb_adbc::AdbcError *error);
DUCKDB_API duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::AdbcDriver *driver,
struct duckdb_adbc::AdbcError *error);

#ifdef __cplusplus
}
Expand Down
112 changes: 51 additions & 61 deletions test/api/adbc/test_adbc.cpp
Expand Up @@ -13,12 +13,15 @@ bool SUCCESS(AdbcStatusCode status) {
return status == ADBC_STATUS_OK;
}
const char *duckdb_lib = std::getenv("DUCKDB_INSTALL_LIB");

class ADBCTestDatabase {
public:
explicit ADBCTestDatabase(const string &path_parameter = ":memory:") {
duckdb_adbc::InitializeADBCError(&adbc_error);
path = TestCreatePath(path_parameter);
if (path_parameter != ":memory:") {
path = TestCreatePath(path_parameter);
} else {
path = path_parameter;
}
REQUIRE(duckdb_lib);
REQUIRE(SUCCESS(AdbcDatabaseNew(&adbc_database, &adbc_error)));
REQUIRE(SUCCESS(AdbcDatabaseSetOption(&adbc_database, "driver", duckdb_lib, &adbc_error)));
Expand All @@ -43,13 +46,17 @@ class ADBCTestDatabase {
}

bool QueryAndCheck(const string &query) {
Query(query);
DuckDB db(path);
Connection con(db);
return ArrowTestHelper::RunArrowComparison(con, query, arrow_stream);
QueryArrow(query);
auto cconn = (duckdb::Connection *)adbc_connection.private_data;
return ArrowTestHelper::RunArrowComparison(*cconn, query, arrow_stream);
}

ArrowArrayStream &Query(const string &query) {
std::unique_ptr<MaterializedQueryResult> Query(const string &query) {
auto cconn = (duckdb::Connection *)adbc_connection.private_data;
return cconn->Query(query);
}

ArrowArrayStream &QueryArrow(const string &query) {
if (arrow_stream.release) {
arrow_stream.release(&arrow_stream);
arrow_stream.release = nullptr;
Expand Down Expand Up @@ -100,7 +107,7 @@ TEST_CASE("ADBC - Test ingestion", "[adbc]") {
ADBCTestDatabase db;

// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");

// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);
Expand All @@ -115,7 +122,7 @@ TEST_CASE("ADBC - Test ingestion - Lineitem", "[adbc]") {
ADBCTestDatabase db;

// Create Arrow Result
auto input_data = db.Query("SELECT * FROM read_csv_auto(\'data/csv/lineitem-carriage.csv\')");
auto input_data = db.QueryArrow("SELECT * FROM read_csv_auto(\'data/csv/lineitem-carriage.csv\')");

// Create Table 'my_table' from the Arrow Result
db.CreateTable("lineitem", input_data);
Expand Down Expand Up @@ -162,9 +169,7 @@ TEST_CASE("Test Invalid Path", "[adbc]") {

REQUIRE(!SUCCESS(AdbcDatabaseInit(&adbc_database, &adbc_error)));

REQUIRE(std::strcmp(
adbc_error.message,
"IO Error: Cannot open file \"/this/path/is/imaginary/hopefully/\": No such file or directory") == 0);
REQUIRE(std::strstr(adbc_error.message, "Cannot open file"));
}

TEST_CASE("Error Release", "[adbc]") {
Expand Down Expand Up @@ -381,7 +386,7 @@ TEST_CASE("Test ADBC Statement Bind (unhappy)", "[adbc]") {
REQUIRE(SUCCESS(AdbcStatementPrepare(&adbc_statement, &adbc_error)));

ADBCTestDatabase db;
auto &input_data = db.Query("SELECT 42, true, 'this is a string'");
auto &input_data = db.QueryArrow("SELECT 42, true, 'this is a string'");
ArrowArray prepared_array;
ArrowSchema prepared_schema;
input_data.get_next(&input_data, &prepared_array);
Expand Down Expand Up @@ -435,7 +440,7 @@ TEST_CASE("Test ADBC Statement Bind", "[adbc]") {
ADBCTestDatabase db;

// Create prepared parameter array
auto &input_data = db.Query("SELECT 42, true, 'this is a string'");
auto &input_data = db.QueryArrow("SELECT 42, true, 'this is a string'");
string query = "select ?, ?, ?";

AdbcDatabase adbc_database;
Expand Down Expand Up @@ -503,7 +508,7 @@ TEST_CASE("Test ADBC Transactions", "[adbc]") {
ADBCTestDatabase db;

// Create Arrow Result
auto &input_data = db.Query("SELECT 42");
auto &input_data = db.QueryArrow("SELECT 42");
string table_name = "test";
string query = "select count(*) from test";

Expand Down Expand Up @@ -559,7 +564,7 @@ TEST_CASE("Test ADBC Transactions", "[adbc]") {
// Now lets insert with Auto-Commit Off
REQUIRE(SUCCESS(AdbcConnectionSetOption(&adbc_connection, ADBC_CONNECTION_OPTION_AUTOCOMMIT,
ADBC_OPTION_VALUE_DISABLED, &adbc_error)));
input_data = db.Query("SELECT 42;");
input_data = db.QueryArrow("SELECT 42;");

REQUIRE(SUCCESS(AdbcStatementNew(&adbc_connection, &adbc_statement, &adbc_error)));

Expand Down Expand Up @@ -606,7 +611,7 @@ TEST_CASE("Test ADBC Transactions", "[adbc]") {
arrow_stream.release(&arrow_stream);

// Lets do a rollback
input_data = db.Query("SELECT 42;");
input_data = db.QueryArrow("SELECT 42;");

REQUIRE(SUCCESS(AdbcStatementNew(&adbc_connection, &adbc_statement, &adbc_error)));

Expand Down Expand Up @@ -656,7 +661,7 @@ TEST_CASE("Test ADBC Transactions", "[adbc]") {
arrow_stream.release(&arrow_stream);

// Let's change the Auto commit config mid-transaction
input_data = db.Query("SELECT 42;");
input_data = db.QueryArrow("SELECT 42;");

REQUIRE(SUCCESS(AdbcStatementNew(&adbc_connection, &adbc_statement, &adbc_error)));

Expand Down Expand Up @@ -692,7 +697,7 @@ TEST_CASE("Test ADBC Transactions", "[adbc]") {
arrow_array.release(&arrow_array);
arrow_stream.release(&arrow_stream);

input_data = db.Query("SELECT 42;");
input_data = db.QueryArrow("SELECT 42;");

REQUIRE(SUCCESS(AdbcStatementNew(&adbc_connection, &adbc_statement, &adbc_error)));

Expand Down Expand Up @@ -873,7 +878,7 @@ TEST_CASE("Test ADBC Substrait", "[adbc]") {
}
// Insert Data
ADBCTestDatabase db;
auto &input_data = db.Query("SELECT 'Push Ups' as exercise, 3 as difficulty_level;");
auto &input_data = db.QueryArrow("SELECT 'Push Ups' as exercise, 3 as difficulty_level;");
string table_name = "crossfit";
REQUIRE(SUCCESS(AdbcStatementNew(&adbc_connection, &adbc_statement, &adbc_error)));

Expand Down Expand Up @@ -960,26 +965,26 @@ TEST_CASE("Test AdbcConnectionGetTableTypes", "[adbc]") {
if (!duckdb_lib) {
return;
}
string path_db;

ADBCTestDatabase db("AdbcConnectionGetTableTypes.db");

// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");
// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);

ArrowArrayStream arrow_stream;
duckdb_adbc::AdbcError adbc_error;
duckdb_adbc::InitializeADBCError(&adbc_error);
AdbcConnectionGetTableTypes(&db.adbc_connection, &arrow_stream, &adbc_error);

db.CreateTable("result", arrow_stream);
path_db = db.path;
db.arrow_stream.release = nullptr;

DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->ColumnCount() == 1);
REQUIRE(res->GetValue(0, 0).ToString() == "BASE TABLE");
db.arrow_stream.release = nullptr;
}

void TestFilters(ADBCTestDatabase &db, duckdb_adbc::AdbcError &adbc_error, idx_t depth) {
Expand All @@ -988,33 +993,27 @@ void TestFilters(ADBCTestDatabase &db, duckdb_adbc::AdbcError &adbc_error, idx_t
AdbcConnectionGetObjects(&db.adbc_connection, depth, nullptr, "bla", nullptr, nullptr, nullptr, &arrow_stream,
&adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->RowCount() == 0);
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
}
{
ArrowArrayStream arrow_stream;
AdbcConnectionGetObjects(&db.adbc_connection, depth, nullptr, nullptr, "bla", nullptr, nullptr, &arrow_stream,
&adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->RowCount() == 0);
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
}
{
ArrowArrayStream arrow_stream;
AdbcConnectionGetObjects(&db.adbc_connection, depth, nullptr, nullptr, nullptr, nullptr, "bla", &arrow_stream,
&adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->RowCount() == 0);
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
}
}

Expand All @@ -1025,11 +1024,10 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") {

// Lets first try what works
// 1. Test ADBC_OBJECT_DEPTH_DB_SCHEMAS

{
ADBCTestDatabase db("ADBC_OBJECT_DEPTH_DB_SCHEMAS.db");
// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");
// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);

Expand All @@ -1040,20 +1038,18 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") {
AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_DB_SCHEMAS, nullptr, nullptr, nullptr, nullptr,
nullptr, &arrow_stream, &adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->ColumnCount() == 1);
REQUIRE(res->GetValue(0, 0).ToString() == "main");
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
TestFilters(db, adbc_error, ADBC_OBJECT_DEPTH_DB_SCHEMAS);
}

// 2. Test ADBC_OBJECT_DEPTH_TABLES
{
ADBCTestDatabase db("test_table_depth");
// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");
// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);

Expand All @@ -1063,21 +1059,19 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") {
AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_TABLES, nullptr, nullptr, nullptr, nullptr,
nullptr, &arrow_stream, &adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->ColumnCount() == 2);
REQUIRE(res->GetValue(0, 0).ToString() == "main");
REQUIRE(res->GetValue(1, 0).ToString() == "[{'table_name': my_table}]");
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
TestFilters(db, adbc_error, ADBC_OBJECT_DEPTH_TABLES);
}

// 3.Test ADBC_OBJECT_DEPTH_COLUMNS
{
ADBCTestDatabase db("test_column_depth");
// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");
// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);

Expand All @@ -1087,22 +1081,20 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") {
AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, nullptr, nullptr,
nullptr, &arrow_stream, &adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->ColumnCount() == 2);
REQUIRE(res->GetValue(0, 0).ToString() == "main");
REQUIRE(
res->GetValue(1, 0).ToString() ==
"[{'table_name': my_table, 'table_columns': [{'column_name': 42, 'ordinal_position': 2, 'remarks': }]}]");
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
TestFilters(db, adbc_error, ADBC_OBJECT_DEPTH_COLUMNS);
}
// 4.Test ADBC_OBJECT_DEPTH_ALL
{
ADBCTestDatabase db("test_all_depth");
// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");
// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);

Expand All @@ -1112,22 +1104,20 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") {
AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_ALL, nullptr, nullptr, nullptr, nullptr,
nullptr, &arrow_stream, &adbc_error);
db.CreateTable("result", arrow_stream);
DuckDB db_check(db.path);
Connection con(db_check);
auto res = con.Query("Select * from result");
auto res = db.Query("Select * from result");
REQUIRE(res->ColumnCount() == 2);
REQUIRE(res->GetValue(0, 0).ToString() == "main");
REQUIRE(
res->GetValue(1, 0).ToString() ==
"[{'table_name': my_table, 'table_columns': [{'column_name': 42, 'ordinal_position': 2, 'remarks': }]}]");
db.Query("Drop table result;");
db.QueryArrow("Drop table result;");
TestFilters(db, adbc_error, ADBC_OBJECT_DEPTH_ALL);
}
// Now lets test some errors
// Now lets test some errors
{
ADBCTestDatabase db("test_errors");
// Create Arrow Result
auto input_data = db.Query("SELECT 42");
auto input_data = db.QueryArrow("SELECT 42");
// Create Table 'my_table' from the Arrow Result
db.CreateTable("my_table", input_data);

Expand Down
3 changes: 2 additions & 1 deletion test/helpers/test_helpers.cpp
Expand Up @@ -105,7 +105,8 @@ string TestDirectoryPath() {
string path;
if (custom_test_directory.empty()) {
// add the PID to the test directory - but only if it was not specified explicitly by the user
path = StringUtil::Format(test_directory + "/%d", getpid());
auto pid = getpid();
path = fs->JoinPath(test_directory, to_string(pid));
} else {
path = test_directory;
}
Expand Down
2 changes: 1 addition & 1 deletion tools/odbc/statement.cpp
Expand Up @@ -512,7 +512,7 @@ static SQLRETURN GetColAttribute(SQLHSTMT statement_handle, SQLUSMALLINT column_
return ret;
}

if (field_identifier != SQL_DESC_COUNT &&
if (field_identifier != SQL_DESC_COUNT && hstmt->res &&
hstmt->res->properties.return_type != duckdb::StatementReturnType::QUERY_RESULT) {
return duckdb::SetDiagnosticRecord(hstmt, SQL_ERROR, "SQLColAttribute(s)",
"Prepared statement not a cursor-specification", SQLStateType::ST_07005,
Expand Down
10 changes: 10 additions & 0 deletions tools/odbc/test/tests/col_attribute.cpp
Expand Up @@ -308,6 +308,16 @@ TEST_CASE("Test SQLColAttribute (descriptor information for a column)", "[odbc]"
ConvertToSQLCHAR("CREATE TABLE test (a INTEGER, b INTEGER)"), SQL_NTS);
ExpectError(hstmt, SQL_DESC_BASE_TABLE_NAME);

// Prepare a statement and call SQLColAttribute, succeeds but is undefined
EXECUTE_AND_CHECK("SQLExecDirect", SQLExecDirect, hstmt,
ConvertToSQLCHAR("create table colattrfoo(col1 int, col2 varchar(20))"), SQL_NTS);

EXECUTE_AND_CHECK("SQLPrepare", SQLPrepare, hstmt, ConvertToSQLCHAR("select * From colattrfoo"), SQL_NTS);

SQLLEN fixed_prec_scale;
EXECUTE_AND_CHECK("SQLColAttribute", SQLColAttribute, hstmt, 1, SQL_DESC_FIXED_PREC_SCALE, nullptr, 0, nullptr,
&fixed_prec_scale);

// Free the statement handle
EXECUTE_AND_CHECK("SQLFreeStmt (HSTMT)", SQLFreeStmt, hstmt, SQL_CLOSE);
EXECUTE_AND_CHECK("SQLFreeHandle (HSTMT)", SQLFreeHandle, SQL_HANDLE_STMT, hstmt);
Expand Down