From fe57f2380d31a483f142faeaee1cc81fe561ed90 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Thu, 9 Oct 2025 09:02:47 +0530 Subject: [PATCH 1/2] working xml --- main.py | 81 +++++++++++++++++++++++--- mssql_python/pybind/ddbc_bindings.cpp | 22 +++++-- tests/test_004_cursor.py | 84 +++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index b45b88d7..32cf156c 100644 --- a/main.py +++ b/main.py @@ -5,17 +5,80 @@ setup_logging('stdout') -conn_str = os.getenv("DB_CONNECTION_STRING") -conn = connect(conn_str) +# conn_str = os.getenv("DB_CONNECTION_STRING") +# conn = connect(conn_str) + +# # conn.autocommit = True + +# cursor = conn.cursor() +# cursor.execute("SELECT database_id, name from sys.databases;") +# rows = cursor.fetchall() + +# for row in rows: +# print(f"Database ID: {row[0]}, Name: {row[1]}") + +# cursor.close() +# conn.close() -# conn.autocommit = True +# import pyodbc +# --- Connection --- +conn_str = "DRIVER={ODBC Driver 18 for SQL Server};Server=Saumya;DATABASE=master;UID=sa;PWD=HappyPass1234;Trust_Connection=yes;TrustServerCertificate=yes;" +conn = connect(conn_str) cursor = conn.cursor() -cursor.execute("SELECT database_id, name from sys.databases;") -rows = cursor.fetchall() -for row in rows: - print(f"Database ID: {row[0]}, Name: {row[1]}") +# --- Create a test table --- +# cursor.execute(""" +# IF OBJECT_ID('dbo.TestXML', 'U') IS NOT NULL DROP TABLE dbo.TestXML; +# CREATE TABLE dbo.TestXML ( +# id INT IDENTITY PRIMARY KEY, +# xml_col XML +# ) +# """) +# conn.commit() + +# # --- Test short XML insertion --- +# short_xml = "123" +# cursor.execute("INSERT INTO dbo.TestXML (xml_col) VALUES (?)", short_xml) +# conn.commit() + +# # --- Test long XML insertion (simulate DAE / large XML) --- +# long_xml = "" + "".join(f"{i}" for i in range(10000)) + "" +# cursor.execute("INSERT INTO dbo.TestXML (xml_col) VALUES (?)", long_xml) +# conn.commit() + +# # --- Fetch and verify --- +# cursor.execute("SELECT id, xml_col FROM dbo.TestXML ORDER BY id") +# rows = cursor.fetchall() + +# for row in rows: +# print(f"ID: {row.id}, XML Length: {len(str(row.xml_col))}") + +# --- Clean up --- +# cursor.execute("DROP TABLE dbo.TestXML") +# conn.commit() +# cursor.close() +# conn.close() + + + +SMALL_XML = "1" +LARGE_XML = "" + "".join(f"{i}" for i in range(10000)) + "" +EMPTY_XML = "" +INVALID_XML = "" # malformed + + +cursor.execute("CREATE TABLE #pytest_xml_empty_null (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") +conn.commit() + +cursor.execute("INSERT INTO #pytest_xml_empty_null (xml_col) VALUES (?);", EMPTY_XML) +cursor.execute("INSERT INTO #pytest_xml_empty_null (xml_col) VALUES (?);", None) +conn.commit() + +rows = [r[0] for r in cursor.execute("SELECT xml_col FROM #pytest_xml_empty_null ORDER BY id;").fetchall()] +print(rows) +assert rows[0] == EMPTY_XML +assert rows[1] is None -cursor.close() -conn.close() \ No newline at end of file +cursor.execute("DROP TABLE IF EXISTS #pytest_xml_empty_null;") +conn.commit() diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 41478797..ba8bfab9 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -21,6 +21,7 @@ #define SQL_SS_TIMESTAMPOFFSET (-155) #define SQL_C_SS_TIMESTAMPOFFSET (0x4001) #define MAX_DIGITS_IN_NUMERIC 64 +#define SQL_SS_XML (-152) #define STRINGIFY_FOR_CASE(x) \ case x: \ @@ -2525,6 +2526,12 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p } break; } + case SQL_SS_XML: + { + LOG("Streaming XML for column {}", i); + row.append(FetchLobColumnData(hStmt, i, SQL_C_WCHAR, true, false)); + break; + } case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { @@ -2982,6 +2989,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column buffers.indicators[col - 1].data()); break; } + case SQL_SS_XML: case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { @@ -3184,6 +3192,10 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum } break; } + case SQL_SS_XML: { + row.append(FetchLobColumnData(hStmt, col, SQL_C_WCHAR, true, false)); + break; + } case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { @@ -3397,6 +3409,7 @@ size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols) { case SQL_LONGVARCHAR: rowSize += columnSize; break; + case SQL_SS_XML: case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: @@ -3501,12 +3514,13 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR || - dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY) && + dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML) && (columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) { + std::cout<<"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"< SQL_MAX_LOB_SIZE)) { lobColumns.push_back(i + 1); // 1-based } diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 97600c17..028b3c13 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -11403,6 +11403,90 @@ def test_datetime_string_parameter_binding(cursor, db_connection): drop_table_if_exists(cursor, table_name) db_connection.commit() +SMALL_XML = "1" +LARGE_XML = "" + "".join(f"{i}" for i in range(10000)) + "" +EMPTY_XML = "" +INVALID_XML = "" # malformed + +def test_xml_basic_insert_fetch(cursor, db_connection): + """Test insert and fetch of a small XML value.""" + try: + cursor.execute("CREATE TABLE #pytest_xml_basic (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") + db_connection.commit() + + cursor.execute("INSERT INTO #pytest_xml_basic (xml_col) VALUES (?);", SMALL_XML) + db_connection.commit() + + row = cursor.execute("SELECT xml_col FROM #pytest_xml_basic;").fetchone() + assert row[0] == SMALL_XML + finally: + cursor.execute("DROP TABLE IF EXISTS #pytest_xml_basic;") + db_connection.commit() + + +def test_xml_empty_and_null(cursor, db_connection): + """Test insert and fetch of empty XML and NULL values.""" + try: + cursor.execute("CREATE TABLE #pytest_xml_empty_null (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") + db_connection.commit() + + cursor.execute("INSERT INTO #pytest_xml_empty_null (xml_col) VALUES (?);", EMPTY_XML) + cursor.execute("INSERT INTO #pytest_xml_empty_null (xml_col) VALUES (?);", None) + db_connection.commit() + + rows = [r[0] for r in cursor.execute("SELECT xml_col FROM #pytest_xml_empty_null ORDER BY id;").fetchall()] + assert rows[0] == EMPTY_XML + assert rows[1] is None + finally: + cursor.execute("DROP TABLE IF EXISTS #pytest_xml_empty_null;") + db_connection.commit() + + +def test_xml_large_insert(cursor, db_connection): + """Test insert and fetch of a large XML value to verify streaming/DAE.""" + try: + cursor.execute("CREATE TABLE #pytest_xml_large (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") + db_connection.commit() + + cursor.execute("INSERT INTO #pytest_xml_large (xml_col) VALUES (?);", LARGE_XML) + db_connection.commit() + + row = cursor.execute("SELECT xml_col FROM #pytest_xml_large;").fetchone() + assert row[0] == LARGE_XML + finally: + cursor.execute("DROP TABLE IF EXISTS #pytest_xml_large;") + db_connection.commit() + + +def test_xml_batch_insert(cursor, db_connection): + """Test batch insert (executemany) of multiple XML values.""" + try: + cursor.execute("CREATE TABLE #pytest_xml_batch (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") + db_connection.commit() + + xmls = [f"{i}" for i in range(5)] + cursor.executemany("INSERT INTO #pytest_xml_batch (xml_col) VALUES (?);", [(x,) for x in xmls]) + db_connection.commit() + + rows = [r[0] for r in cursor.execute("SELECT xml_col FROM #pytest_xml_batch ORDER BY id;").fetchall()] + assert rows == xmls + finally: + cursor.execute("DROP TABLE IF EXISTS #pytest_xml_batch;") + db_connection.commit() + + +def test_xml_malformed_input(cursor, db_connection): + """Verify driver raises error for invalid XML input.""" + try: + cursor.execute("CREATE TABLE #pytest_xml_invalid (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") + db_connection.commit() + + with pytest.raises(Exception): + cursor.execute("INSERT INTO #pytest_xml_invalid (xml_col) VALUES (?);", INVALID_XML) + finally: + cursor.execute("DROP TABLE IF EXISTS #pytest_xml_invalid;") + db_connection.commit() + def test_close(db_connection): """Test closing the cursor""" try: From a86a2aacd71bf6a5631fe7bce0434e49d9eaadaa Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 14 Oct 2025 09:59:52 +0530 Subject: [PATCH 2/2] cleanup --- main.py | 81 +++------------------------ mssql_python/pybind/ddbc_bindings.cpp | 10 +--- 2 files changed, 11 insertions(+), 80 deletions(-) diff --git a/main.py b/main.py index 32cf156c..b45b88d7 100644 --- a/main.py +++ b/main.py @@ -5,80 +5,17 @@ setup_logging('stdout') -# conn_str = os.getenv("DB_CONNECTION_STRING") -# conn = connect(conn_str) - -# # conn.autocommit = True - -# cursor = conn.cursor() -# cursor.execute("SELECT database_id, name from sys.databases;") -# rows = cursor.fetchall() - -# for row in rows: -# print(f"Database ID: {row[0]}, Name: {row[1]}") - -# cursor.close() -# conn.close() - -# import pyodbc - -# --- Connection --- -conn_str = "DRIVER={ODBC Driver 18 for SQL Server};Server=Saumya;DATABASE=master;UID=sa;PWD=HappyPass1234;Trust_Connection=yes;TrustServerCertificate=yes;" +conn_str = os.getenv("DB_CONNECTION_STRING") conn = connect(conn_str) -cursor = conn.cursor() - -# --- Create a test table --- -# cursor.execute(""" -# IF OBJECT_ID('dbo.TestXML', 'U') IS NOT NULL DROP TABLE dbo.TestXML; -# CREATE TABLE dbo.TestXML ( -# id INT IDENTITY PRIMARY KEY, -# xml_col XML -# ) -# """) -# conn.commit() - -# # --- Test short XML insertion --- -# short_xml = "123" -# cursor.execute("INSERT INTO dbo.TestXML (xml_col) VALUES (?)", short_xml) -# conn.commit() -# # --- Test long XML insertion (simulate DAE / large XML) --- -# long_xml = "" + "".join(f"{i}" for i in range(10000)) + "" -# cursor.execute("INSERT INTO dbo.TestXML (xml_col) VALUES (?)", long_xml) -# conn.commit() +# conn.autocommit = True -# # --- Fetch and verify --- -# cursor.execute("SELECT id, xml_col FROM dbo.TestXML ORDER BY id") -# rows = cursor.fetchall() - -# for row in rows: -# print(f"ID: {row.id}, XML Length: {len(str(row.xml_col))}") - -# --- Clean up --- -# cursor.execute("DROP TABLE dbo.TestXML") -# conn.commit() -# cursor.close() -# conn.close() - - - -SMALL_XML = "1" -LARGE_XML = "" + "".join(f"{i}" for i in range(10000)) + "" -EMPTY_XML = "" -INVALID_XML = "" # malformed - - -cursor.execute("CREATE TABLE #pytest_xml_empty_null (id INT PRIMARY KEY IDENTITY(1,1), xml_col XML NULL);") -conn.commit() - -cursor.execute("INSERT INTO #pytest_xml_empty_null (xml_col) VALUES (?);", EMPTY_XML) -cursor.execute("INSERT INTO #pytest_xml_empty_null (xml_col) VALUES (?);", None) -conn.commit() +cursor = conn.cursor() +cursor.execute("SELECT database_id, name from sys.databases;") +rows = cursor.fetchall() -rows = [r[0] for r in cursor.execute("SELECT xml_col FROM #pytest_xml_empty_null ORDER BY id;").fetchall()] -print(rows) -assert rows[0] == EMPTY_XML -assert rows[1] is None +for row in rows: + print(f"Database ID: {row[0]}, Name: {row[1]}") -cursor.execute("DROP TABLE IF EXISTS #pytest_xml_empty_null;") -conn.commit() +cursor.close() +conn.close() \ No newline at end of file diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index ba8bfab9..7795d6d3 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2989,7 +2989,6 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column buffers.indicators[col - 1].data()); break; } - case SQL_SS_XML: case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { @@ -3192,10 +3191,6 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum } break; } - case SQL_SS_XML: { - row.append(FetchLobColumnData(hStmt, col, SQL_C_WCHAR, true, false)); - break; - } case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { @@ -3516,11 +3511,10 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML) && (columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) { - std::cout<<"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"<