Skip to content

FEAT: adding perf improvements #311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
+476 −254
Closed

FEAT: adding perf improvements #311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
459b4c1
adding perf improvements
gargsaumya Oct 30, 2025
eccfe5b
fixed converter test failures, decimal separator tests failure and ro…
gargsaumya Oct 31, 2025
285f7ca
fixed cursor's scroll/skip logic with fast-fwd cursor and tests
gargsaumya Oct 31, 2025
c9197d1
optimized cached column map
gargsaumya Oct 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 159 additions & 36 deletions mssql_python/cursor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,12 @@ def _reset_cursor(self) -> None:
log("debug", "SQLFreeHandle succeeded")

self._clear_rownumber()

# Clear pre-computed metadata
self._column_name_map = None
self._settings_snapshot = None
self._uuid_indices = None
self._converter_map = None

# Reinitialize the statement handle
self._initialize_cursor()
Expand Down Expand Up @@ -822,6 +828,65 @@ def _initialize_description(self, column_metadata: Optional[Any] = None) -> None
)
)
self.description = description

# Pre-compute shared metadata for Row optimization
self._precompute_row_metadata()

def _precompute_row_metadata(self) -> None:
"""
Pre-compute metadata shared across all Row instances for performance.
This avoids expensive per-row computations in Row.__init__.
"""
if not self.description:
self._column_name_map = None
self._settings_snapshot = None
self._uuid_indices = None
self._converter_map = None
return

# Pre-compute settings snapshot
settings = get_settings()
self._settings_snapshot = {
"lowercase": settings.lowercase,
"native_uuid": settings.native_uuid,
}

# Pre-compute column name to index mapping
self._column_name_map = {}
self._uuid_indices = []
self._converter_map = {} # Column index -> converter function

for i, col_desc in enumerate(self.description):
if col_desc: # Ensure column description exists
col_name = col_desc[0] # Name is first item in description tuple
if self._settings_snapshot.get("lowercase"):
col_name = col_name.lower()
self._column_name_map[col_name] = i

# Pre-identify UUID columns (SQL_GUID = -11)
if len(col_desc) > 1 and col_desc[1] == -11:
self._uuid_indices.append(i)

# Pre-compute output converters for each column
if len(col_desc) > 1:
sql_type = col_desc[1] # type_code is at index 1

# Check if we have output converters configured
if (hasattr(self.connection, "_output_converters")
and self.connection._output_converters):

converter = self.connection.get_output_converter(sql_type)

# If no converter found but it might be string/bytes, try WVARCHAR
if converter is None:
converter = self.connection.get_output_converter(-9) # SQL_WVARCHAR

# Store converter if found
if converter is not None:
self._converter_map[i] = {
'converter': converter,
'sql_type': sql_type
}

def _map_data_type(self, sql_type: int) -> type:
"""
Expand Down Expand Up @@ -1972,7 +2037,8 @@ def fetchone(self) -> Union[None, Row]:
# Create and return a Row object, passing column name map if available
column_map = getattr(self, "_column_name_map", None)
settings_snapshot = getattr(self, "_settings_snapshot", None)
return Row(self, self.description, row_data, column_map, settings_snapshot)
converter_map = getattr(self, "_converter_map", None)
return Row(self, self.description, row_data, column_map, settings_snapshot, converter_map)
except Exception as e: # pylint: disable=broad-exception-caught
# On error, don't increment rownumber - rethrow the error
raise e
Expand Down Expand Up @@ -2017,13 +2083,19 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]:
else:
self.rowcount = self._next_row_index

# Convert raw data to Row objects
column_map = getattr(self, "_column_name_map", None)
settings_snapshot = getattr(self, "_settings_snapshot", None)
return [
Row(self, self.description, row_data, column_map, settings_snapshot)
for row_data in rows_data
]
# Convert raw data to Row objects using pre-computed metadata
if rows_data:
column_map = getattr(self, "_column_name_map", None)
settings_snapshot = getattr(self, "_settings_snapshot", None)
converter_map = getattr(self, "_converter_map", None)

# Batch create Row objects with optimized metadata
return [
Row(self, self.description, row_data, column_map, settings_snapshot, converter_map)
for row_data in rows_data
]
else:
return []
except Exception as e: # pylint: disable=broad-exception-caught
# On error, don't increment rownumber - rethrow the error
raise e
Expand Down Expand Up @@ -2058,17 +2130,52 @@ def fetchall(self) -> List[Row]:
else:
self.rowcount = self._next_row_index

# Convert raw data to Row objects
column_map = getattr(self, "_column_name_map", None)
settings_snapshot = getattr(self, "_settings_snapshot", None)
return [
Row(self, self.description, row_data, column_map, settings_snapshot)
for row_data in rows_data
]
# Convert raw data to Row objects using pre-computed metadata
if rows_data:
column_map = getattr(self, "_column_name_map", None)
settings_snapshot = getattr(self, "_settings_snapshot", None)

# Get pre-computed converter map
converter_map = getattr(self, "_converter_map", None)

# Use optimized Row creation for large datasets
if len(rows_data) > 10000:
return self._create_rows_optimized(rows_data, column_map, settings_snapshot, converter_map)
else:
# Regular path for smaller datasets
return [
Row(self, self.description, row_data, column_map, settings_snapshot, converter_map)
for row_data in rows_data
]
else:
return []
except Exception as e: # pylint: disable=broad-exception-caught
# On error, don't increment rownumber - rethrow the error
raise e

def _create_rows_optimized(self, rows_data, column_map, settings_snapshot, converter_map):
"""
Optimized Row creation for large datasets using batch processing.
"""
# For very large datasets, minimize object creation overhead
Row_class = Row
description = self.description
cursor_ref = self

# Use more efficient approach for very large datasets
if len(rows_data) > 50000:
# Pre-allocate result list to avoid multiple reallocations
result = [None] * len(rows_data)
for i, row_data in enumerate(rows_data):
result[i] = Row_class(cursor_ref, description, row_data, column_map, settings_snapshot, converter_map)
return result
else:
# Standard list comprehension for medium-large datasets
return [
Row_class(cursor_ref, description, row_data, column_map, settings_snapshot, converter_map)
for row_data in rows_data
]

def nextset(self) -> Union[bool, None]:
"""
Skip to the next available result set.
Expand Down Expand Up @@ -2290,30 +2397,46 @@ def scroll(self, value: int, mode: str = "relative") -> None: # pylint: disable
if mode == "relative":
if value == 0:
return
ret = ddbc_bindings.DDBCSQLFetchScroll(
self.hstmt, ddbc_sql_const.SQL_FETCH_RELATIVE.value, value, row_data
)
if ret == ddbc_sql_const.SQL_NO_DATA.value:
raise IndexError(
"Cannot scroll to specified position: end of result set reached"
)
# Consume N rows; last-returned index advances by N
self._rownumber = self._rownumber + value
self._next_row_index = self._rownumber + 1

# For forward-only cursor, use SQLFetchOne repeatedly instead of SQLFetchScroll
for i in range(value):
ret = ddbc_bindings.DDBCSQLFetchOne(self.hstmt, row_data)
if ret == ddbc_sql_const.SQL_NO_DATA.value:
raise IndexError(
f"Cannot scroll to specified position: end of result set reached at position {i+1}/{value}"
)
# Clear row_data for next iteration to avoid accumulating data
row_data.clear()

# Consume N rows; advance next_row_index by N
self._next_row_index += value
self._rownumber = self._next_row_index - 1
return

# absolute(k>0): map Python k (0-based next row) to ODBC ABSOLUTE k (1-based),
# intentionally passing k so ODBC fetches row #k (1-based), i.e., 0-based (k-1),
# leaving the NEXT fetch to return 0-based index k.
ret = ddbc_bindings.DDBCSQLFetchScroll(
self.hstmt, ddbc_sql_const.SQL_FETCH_ABSOLUTE.value, value, row_data
)
if ret == ddbc_sql_const.SQL_NO_DATA.value:
raise IndexError(
f"Cannot scroll to position {value}: end of result set reached"
# For forward-only cursor, implement absolute positioning using relative scrolling
# absolute(k): position so next fetch returns row at 0-based index k
current_next_index = self._next_row_index # Where we would fetch next

if value < current_next_index:
# Can't go backward with forward-only cursor
raise NotSupportedError(
"Backward absolute positioning not supported",
f"Cannot move from next position {current_next_index} back to {value} on a forward-only cursor"
)
elif value > current_next_index:
# Move forward: skip rows from current_next_index to value
rows_to_skip = value - current_next_index
for i in range(rows_to_skip):
ret = ddbc_bindings.DDBCSQLFetchOne(self.hstmt, row_data)
if ret == ddbc_sql_const.SQL_NO_DATA.value:
raise IndexError(
f"Cannot scroll to position {value}: end of result set reached at position {current_next_index + i}"
)
# Clear row_data for next iteration
row_data.clear()
# else value == current_next_index: no movement needed

# Tests expect rownumber == value after absolute(value)
# Tests expect rownumber == value after absolute(value)
# Next fetch should return row index 'value'
self._rownumber = value
self._next_row_index = value
Expand Down Expand Up @@ -2471,4 +2594,4 @@ def setoutputsize(self, size: int, column: Optional[int] = None) -> None:
This method is a no-op in this implementation as buffer sizes
are managed automatically by the underlying driver.
"""
# This is a no-op - buffer sizes are managed automatically
# This is a no-op - buffer sizes are managed automatically
Loading
Loading