From e21c9b6e04b5600c359408165a45c2676840a5d4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 27 Sep 2025 10:14:29 +0000 Subject: [PATCH 1/4] Initial plan From 1c24dc8be5cd7d9a84e59f9e852bdbffa6d25a97 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 27 Sep 2025 10:22:47 +0000 Subject: [PATCH 2/4] Implement precreate functionality in session management Co-authored-by: faizanazim11 <20454506+faizanazim11@users.noreply.github.com> --- README.md | 110 ++++++++++++++++++++- sql_db_utils/asyncio/session_management.py | 43 +++++++- sql_db_utils/session_management.py | 43 +++++++- 3 files changed, 193 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bea0ccb..29a528b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A powerful SQL database utilities package for Python developers that provides de ## Features - Automatic declarative model generation for SQL databases -- Schema-aware session management +- Schema-aware session management with precreate and postcreate hooks - Support for both synchronous and asynchronous operations - Project-based database isolation - Security-enabled database connections @@ -32,6 +32,114 @@ pip install sql-db-utils - SQLAlchemy >= 2.0.38 - Additional dependencies based on optional features +## Session Management Hooks + +The `SQLSessionManager` provides two types of hooks for customizing database setup: + +### Precreate Hooks + +Precreate hooks execute **before** database tables and meta information are created. These are useful for: +- Setting up database extensions +- Creating custom schemas +- Preparing database environment +- Initial database configuration + +#### Automatic Precreate +Returns SQL statements to be executed automatically: + +```python +from sql_db_utils import SQLSessionManager + +session_manager = SQLSessionManager() + +@session_manager.register_precreate("my_database") +def setup_database_extensions(tenant_id): + return "CREATE EXTENSION IF NOT EXISTS uuid-ossp;" + +# Or for multiple databases +@session_manager.register_precreate(["db1", "db2"]) +def setup_multiple_databases(tenant_id): + return [ + "CREATE EXTENSION IF NOT EXISTS uuid-ossp;", + "CREATE EXTENSION IF NOT EXISTS pgcrypto;" + ] +``` + +#### Manual Precreate +Receives a session object for custom operations: + +```python +@session_manager.register_precreate_manual("my_database") +def custom_precreate_setup(session, tenant_id): + # Custom logic with session + session.execute("CREATE SCHEMA IF NOT EXISTS custom_schema;") + # Additional setup logic here +``` + +### Postcreate Hooks + +Postcreate hooks execute **after** database tables and meta information are created. These are useful for: +- Seeding initial data +- Creating triggers and procedures +- Setting up initial user permissions +- Post-creation optimizations + +#### Automatic Postcreate +Returns SQL statements to be executed automatically: + +```python +@session_manager.register_postcreate("my_database") +def seed_initial_data(tenant_id): + return "INSERT INTO users (name) VALUES ('admin') ON CONFLICT DO NOTHING;" + +# Or for multiple databases +@session_manager.register_postcreate(["db1", "db2"]) +def setup_multiple_databases(tenant_id): + return [ + "INSERT INTO settings (key, value) VALUES ('version', '1.0') ON CONFLICT DO NOTHING;", + "INSERT INTO roles (name) VALUES ('admin') ON CONFLICT DO NOTHING;" + ] +``` + +#### Manual Postcreate +Receives a session object for custom operations: + +```python +@session_manager.register_postcreate_manual("my_database") +def custom_postcreate_setup(session, tenant_id): + # Custom logic with session + session.execute("INSERT INTO initial_data (tenant_id) VALUES (:tenant_id);", {"tenant_id": tenant_id}) + # Additional setup logic here +``` + +### Execution Order + +1. **Precreate hooks** (before table creation) + - `register_precreate` functions are executed + - `register_precreate_manual` functions are executed +2. **Database table creation** (`create_default_psql_dependencies`) +3. **Postcreate hooks** (after table creation) + - `register_postcreate` functions are executed + - `register_postcreate_manual` functions are executed + +### Async Support + +All hooks work identically with the async session manager: + +```python +from sql_db_utils.asyncio import SQLSessionManager + +async_session_manager = SQLSessionManager() + +@async_session_manager.register_precreate("my_database") +def setup_extensions(tenant_id): + return "CREATE EXTENSION IF NOT EXISTS uuid-ossp;" + +@async_session_manager.register_postcreate_manual("my_database") +async def seed_data(session, tenant_id): + await session.execute("INSERT INTO users (tenant_id) VALUES (:tenant_id);", {"tenant_id": tenant_id}) +``` + ## Authors - Faizan (faizanazim11@gmail.com) diff --git a/sql_db_utils/asyncio/session_management.py b/sql_db_utils/asyncio/session_management.py index f10137f..adfc3d6 100644 --- a/sql_db_utils/asyncio/session_management.py +++ b/sql_db_utils/asyncio/session_management.py @@ -13,13 +13,15 @@ class SQLSessionManager: - __slots__ = ("_db_engines", "database_uri", "_postcreate_auto", "_postcreate_manual") + __slots__ = ("_db_engines", "database_uri", "_postcreate_auto", "_postcreate_manual", "_precreate_auto", "_precreate_manual") def __init__(self, database_uri: Union[str, None] = None) -> None: self._db_engines = {} self.database_uri = database_uri or PostgresConfig.POSTGRES_URI self._postcreate_auto: dict = {} self._postcreate_manual: dict = {} + self._precreate_auto: dict = {} + self._precreate_manual: dict = {} def __del__(self) -> None: for engine in self._db_engines.values(): @@ -81,6 +83,7 @@ async def _get_engine( await self._ensure_engine_connection(engine) if not PostgresConfig.PG_ANTI_PERSISTENT: self._db_engines[qualified_db_name] = engine + await self.run_precreate(engine, database, tenant_id) await create_default_psql_dependencies( metadata=metadata or DeclarativeBaseClassFactory(database).metadata, engine_obj=engine ) @@ -135,12 +138,50 @@ def decorator(func: Callable) -> None: return decorator + def precreate_decorator(self, raw_db: str | List[str], precreate_store: str) -> Callable: + precreate_store = getattr(self, precreate_store) + + def decorator(func: Callable) -> None: + if isinstance(raw_db, list): + for db in raw_db: + precreate_auto = precreate_store.get(db, []) + precreate_auto.append(func) + precreate_store[db] = precreate_auto + else: + precreate_auto = precreate_store.get(raw_db, []) + precreate_auto.append(func) + precreate_store[raw_db] = precreate_auto + + return decorator + def register_postcreate(self, raw_db: str | List[str]) -> Callable: return self.postcreate_decorator(raw_db, "_postcreate_auto") def register_postcreate_manual(self, raw_db: str | List[str]) -> Callable: return self.postcreate_decorator(raw_db, "_postcreate_manual") + def register_precreate(self, raw_db: str | List[str]) -> Callable: + return self.precreate_decorator(raw_db, "_precreate_auto") + + def register_precreate_manual(self, raw_db: str | List[str]) -> Callable: + return self.precreate_decorator(raw_db, "_precreate_manual") + + async def run_precreate(self, engine: AsyncEngine, raw_db: str, tenant_id: Union[str, None] = None) -> None: + session = AsyncSession(bind=engine, future=True, expire_on_commit=False) + async with session.begin(): + for precreate_func in self._precreate_auto.get(raw_db, []): + result = precreate_func(tenant_id) + if isinstance(result, list): + for statement in result: + await session.execute(statement) + else: + await session.execute(result) + for precreate_func in self._precreate_manual.get(raw_db, []): + await precreate_func(session, tenant_id) + await session.commit() + await session.close() + logging.info(f"Precreate for {raw_db} completed") + async def run_postcreate(self, engine: AsyncEngine, raw_db: str, tenant_id: Union[str, None] = None) -> None: session = AsyncSession(bind=engine, future=True, expire_on_commit=False) async with session.begin(): diff --git a/sql_db_utils/session_management.py b/sql_db_utils/session_management.py index 4db93d3..ec243a4 100644 --- a/sql_db_utils/session_management.py +++ b/sql_db_utils/session_management.py @@ -13,13 +13,15 @@ class SQLSessionManager: - __slots__ = ("_db_engines", "database_uri", "_postcreate_auto", "_postcreate_manual") + __slots__ = ("_db_engines", "database_uri", "_postcreate_auto", "_postcreate_manual", "_precreate_auto", "_precreate_manual") def __init__(self, database_uri: Union[str, None] = None) -> None: self._db_engines = {} self.database_uri = database_uri or PostgresConfig.POSTGRES_URI self._postcreate_auto: dict = {} self._postcreate_manual: dict = {} + self._precreate_auto: dict = {} + self._precreate_manual: dict = {} def __del__(self) -> None: for engine in self._db_engines.values(): @@ -81,6 +83,7 @@ def _get_engine( self._ensure_engine_connection(engine) if not PostgresConfig.PG_ANTI_PERSISTENT: self._db_engines[qualified_db_name] = engine + self.run_precreate(engine, database, tenant_id) create_default_psql_dependencies( metadata=metadata or DeclarativeBaseClassFactory(database).metadata, engine_obj=engine ) @@ -135,12 +138,50 @@ def decorator(func: Callable) -> None: return decorator + def precreate_decorator(self, raw_db: str | List[str], precreate_store: str) -> Callable: + precreate_store = getattr(self, precreate_store) + + def decorator(func: Callable) -> None: + if isinstance(raw_db, list): + for db in raw_db: + precreate_auto = precreate_store.get(db, []) + precreate_auto.append(func) + precreate_store[db] = precreate_auto + else: + precreate_auto = precreate_store.get(raw_db, []) + precreate_auto.append(func) + precreate_store[raw_db] = precreate_auto + + return decorator + def register_postcreate(self, raw_db: str | List[str]) -> Callable: return self.postcreate_decorator(raw_db, "_postcreate_auto") def register_postcreate_manual(self, raw_db: str | List[str]) -> Callable: return self.postcreate_decorator(raw_db, "_postcreate_manual") + def register_precreate(self, raw_db: str | List[str]) -> Callable: + return self.precreate_decorator(raw_db, "_precreate_auto") + + def register_precreate_manual(self, raw_db: str | List[str]) -> Callable: + return self.precreate_decorator(raw_db, "_precreate_manual") + + def run_precreate(self, engine: Engine, raw_db: str, tenant_id: Union[str, None] = None) -> None: + session = Session(bind=engine, future=True, expire_on_commit=False) + with session.begin(): + for precreate_func in self._precreate_auto.get(raw_db, []): + result = precreate_func(tenant_id) + if isinstance(result, list): + for statement in result: + session.execute(statement) + else: + session.execute(result) + for precreate_func in self._precreate_manual.get(raw_db, []): + precreate_func(session, tenant_id) + session.commit() + session.close() + logging.info(f"Precreate for {raw_db} completed") + def run_postcreate(self, engine: Engine, raw_db: str, tenant_id: Union[str, None] = None) -> None: session = Session(bind=engine, future=True, expire_on_commit=False) with session.begin(): From 56cf0477f0b9a587c426a0dde8601a5688a0abc4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 27 Sep 2025 10:32:33 +0000 Subject: [PATCH 3/4] Fix linting issues and add copilot instructions Co-authored-by: faizanazim11 <20454506+faizanazim11@users.noreply.github.com> --- .github/copilot-instructions.md | 224 +++++++++++++++++++++ sql_db_utils/asyncio/session_management.py | 9 +- sql_db_utils/session_management.py | 9 +- 3 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..1136236 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,224 @@ +# sql-db-utils - GitHub Copilot Instructions + +**SQL database utilities package for Python developers that provides declarative model generation and session management.** + +Always reference these instructions first and fallback to search or bash commands only when you encounter unexpected information that does not match the info here. + +## Working Effectively + +### Bootstrap, Build, and Test Repository: +- `pip install -e .` -- installs the package in development mode. NEVER CANCEL: Takes 10-60 seconds, may timeout due to network issues. Set timeout to 120+ seconds (extra margin for slow mirrors or network issues). +- `pip install coverage pre-commit pytest pytest-cov ruff` -- installs development dependencies. NEVER CANCEL: Takes 30-120 seconds. Set timeout to 180+ seconds (extra margin for slow mirrors or network issues). +- `python -m pytest tests/ -v` -- runs unit tests (when tests are available) +- `ruff check .` -- runs linting (takes ~0.01 seconds) +- `ruff format --check .` -- checks code formatting (takes ~0.01 seconds) + +### Environment Configuration: +- Package requires Python >= 3.13 +- Core dependencies: SQLAlchemy >= 2.0.38, sqlalchemy-utils, psycopg, python-dateutil, whenever +- Optional dependencies available: polars, pandas, async, binary, codegen +- Database support: PostgreSQL (primary), with pooling and connection management +- Configuration via environment variables and PostgreSQL connection strings + +### Run Integration Tests with Real Database: +- Start PostgreSQL: `docker run -d --name test-postgres -p 5432:5432 -e POSTGRES_PASSWORD=test postgres:15-alpine` (NEVER CANCEL: Takes 30-60 seconds for first download) +- Wait for startup: `sleep 10` +- Run integration tests: `POSTGRES_URI=postgresql://postgres:test@localhost:5432/test python -c "from sql_db_utils import SQLSessionManager; print('SQL integration test passed')"` (basic session management test) +- Clean up: `docker stop test-postgres && docker rm test-postgres` + +## Validation Scenarios + +### Always Test After Making Changes: +1. **Import Test**: `python -c "from sql_db_utils import SQLSessionManager; from sql_db_utils.asyncio import SQLSessionManager as AsyncSQLSessionManager; print('Import successful')"` +2. **Basic Session Management Test** (requires PostgreSQL running): + ```bash + POSTGRES_URI=postgresql://postgres:test@localhost:5432/test python -c " + from sql_db_utils import SQLSessionManager + manager = SQLSessionManager() + print('Session Manager created successfully') + " + ``` +3. **Precreate/Postcreate Functionality Test**: + ```bash + python -c " + from sql_db_utils import SQLSessionManager + manager = SQLSessionManager() + + @manager.register_precreate('test_db') + def test_precreate(tenant_id): + return 'SELECT 1;' + + @manager.register_postcreate('test_db') + def test_postcreate(tenant_id): + return 'SELECT 2;' + + print('Precreate/Postcreate registration successful') + " + ``` +4. **Run Full Test Suite**: `python -m pytest tests/ -v --cov=sql_db_utils --cov-report=term-missing` (when tests are available) +5. **Linting**: `ruff check . && ruff format --check .` + +### Manual Testing Requirements: +- ALWAYS test session management functionality after code changes +- Test both synchronous and asynchronous implementations +- Verify precreate and postcreate hooks work correctly +- Test with different PostgreSQL configurations and connection parameters +- Test database creation, connection pooling, and engine management + +## Common Tasks + +### Repository Structure: +``` +sql-db-utils/ +├── .github/workflows/ # CI/CD pipelines +├── sql_db_utils/ # Main package source +│ ├── __init__.py # Main exports +│ ├── config.py # Configuration settings +│ ├── constants.py # Package constants +│ ├── datetime_utils.py # Date/time utilities +│ ├── session_management.py # Core session management (sync) +│ ├── sql_creations.py # SQL table creation utilities +│ ├── sql_extras.py # Additional SQL utilities +│ ├── sql_retry_handler.py # Query retry mechanisms +│ ├── sql_utils.py # General SQL utilities +│ ├── declarative_utils.py # Declarative model utilities +│ ├── declaratives.py # Base declarative classes +│ ├── codegen.py # Code generation utilities +│ ├── aggrid/ # AG Grid integration utilities +│ │ ├── date_filters.py +│ │ ├── number_filters.py +│ │ └── text_filters.py +│ └── asyncio/ # Asynchronous implementations +│ ├── __init__.py +│ ├── session_management.py # Async session management +│ ├── sql_creations.py # Async SQL creation utilities +│ ├── sql_creation_helper.py # Async creation helpers +│ ├── sql_retry_handler.py # Async retry mechanisms +│ ├── sql_utils.py # Async SQL utilities +│ ├── declarative_utils.py # Async declarative utilities +│ ├── declaratives.py # Async base classes +│ ├── codegen.py # Async code generation +│ └── inspector_utils.py # Database inspection utilities +├── tests/ # Test files (when available) +├── pyproject.toml # Project configuration +└── README.md # Documentation +``` + +### Key Files to Check After Changes: +- Always verify `sql_db_utils/__init__.py` after changing main exports +- Check `sql_db_utils/config.py` after modifying configuration handling +- Verify sync/async parity between `sql_db_utils/session_management.py` and `sql_db_utils/asyncio/session_management.py` +- Update `sql_db_utils/sql_creations.py` and `sql_db_utils/asyncio/sql_creations.py` for SQL creation changes +- Test declarative utilities in both sync and async versions +- Verify codegen functionality if making changes to code generation features +- Update tests when adding new functionality +- Run integration tests with real PostgreSQL database + +### Development Dependencies: +- **Testing**: pytest, pytest-cov, coverage +- **Linting**: ruff (replaces black, flake8, isort) +- **Git hooks**: pre-commit +- **Type checking**: Built into package development +- **Core Dependencies**: SQLAlchemy, sqlalchemy-utils, psycopg, python-dateutil, whenever + +### Build and Package: +- `python -m build` -- builds distribution packages. NEVER CANCEL: May fail due to network timeouts depending on the configured build backend and network environment. Package requires Python >= 3.13. +- Package metadata in `pyproject.toml` +- Uses hatchling as build backend +- **Note**: Package requires specific Python version (>=3.13) which may not be available in all environments + +### Session Management Features: +- **Precreate Hooks**: Execute before database/table creation for setup tasks +- **Postcreate Hooks**: Execute after database/table creation for initialization +- **Auto Hooks**: Return SQL statements to be executed automatically +- **Manual Hooks**: Receive session objects for custom operations +- **Multi-database Support**: Register hooks for single or multiple databases +- **Tenant Support**: All hooks receive tenant_id parameter for multi-tenant applications + +## Database Features and Testing + +### Supported Database Features: +- **PostgreSQL**: Primary database with full feature support +- **Connection Pooling**: Configurable via PostgresConfig.PG_ENABLE_POOLING +- **Connection Retry**: Built-in retry mechanisms with PostgresConfig.PG_MAX_RETRY +- **Database Creation**: Automatic database creation if not exists +- **Schema Support**: Multi-schema support with declarative utilities +- **Transaction Management**: Automatic transaction handling in hooks + +### Session Management Testing: +- **Engine Creation**: Test `_get_engine()` method with various configurations +- **Hook Execution**: Verify precreate/postcreate hooks execute in correct order +- **Connection Pooling**: Test with/without pooling enabled +- **Multi-tenant**: Test with different tenant_id values +- **Error Handling**: Test connection failures and retry mechanisms + +### Setting up Test Database with Docker: +- PostgreSQL: `docker run -d --name test-postgres -p 5432:5432 -e POSTGRES_PASSWORD=test postgres:15-alpine` +- Create test database: `docker exec -it test-postgres psql -U postgres -c "CREATE DATABASE test;"` + +## CI/CD Pipeline (.github/workflows) + +### Linter Pipeline: +- Runs on pull requests and pushes +- Uses ruff for linting and formatting +- ALWAYS run `ruff check .` and `ruff format --check .` before committing +- Pre-commit hooks should handle formatting automatically + +### Package Publishing: +- Triggers on git tags +- Builds with hatchling backend +- Publishes to PyPI +- Requires Python >= 3.13 environment + +## Critical Notes + +### Session Management Execution Order: +1. **Engine Creation**: Database connection and engine setup +2. **Precreate Hooks**: Execute custom setup before table creation +3. **Table Creation**: `create_default_psql_dependencies()` creates tables/metadata +4. **Postcreate Hooks**: Execute initialization after table creation + +### Hook Implementation Patterns: +- **Auto Hooks**: Return SQL strings or lists of SQL strings +- **Manual Hooks**: Receive (session, tenant_id) parameters for custom logic +- **Registration**: Use `@manager.register_precreate()` or `@manager.register_precreate_manual()` +- **Multi-Database**: Pass list of database names to register for multiple databases + +### Async/Sync Parity: +- Both implementations must have identical API and functionality +- Async version uses `async`/`await` patterns appropriately +- Session types differ: `Session` vs `AsyncSession` +- Engine types differ: `Engine` vs `AsyncEngine` + +### Configuration and Environment: +- PostgresConfig class manages all database configuration +- Environment variables control connection parameters +- ModuleConfig handles application-level settings +- Connection pooling and retry settings are configurable + +## Troubleshooting + +### Common Issues: +1. **Import Error**: Check Python version (>=3.13 required) +2. **Connection Failures**: Verify PostgreSQL is running and accessible +3. **Linting Failures**: Run `ruff format .` to auto-fix formatting issues +4. **Missing Dependencies**: Run `pip install -e .` to reinstall package +5. **Hook Execution**: Verify hooks are registered before engine creation + +### Session Management Issues: +- **Engine Not Created**: Check database URI and connection parameters +- **Hooks Not Executing**: Ensure registration occurs before `get_session()` calls +- **Transaction Errors**: Verify database permissions and connection state +- **Pooling Issues**: Check PostgresConfig.PG_ENABLE_POOLING setting + +### Development Environment: +- Python 3.13+ required (check with `python --version`) +- PostgreSQL server required for integration testing +- Docker recommended for consistent database testing +- Pre-commit hooks help maintain code quality + +### Async/Sync Coordination: +- Changes to sync version should be mirrored in async version +- Test both implementations when making session management changes +- Verify async patterns use proper `await` keywords +- Check that both versions handle errors consistently \ No newline at end of file diff --git a/sql_db_utils/asyncio/session_management.py b/sql_db_utils/asyncio/session_management.py index adfc3d6..7bcc9b0 100644 --- a/sql_db_utils/asyncio/session_management.py +++ b/sql_db_utils/asyncio/session_management.py @@ -13,7 +13,14 @@ class SQLSessionManager: - __slots__ = ("_db_engines", "database_uri", "_postcreate_auto", "_postcreate_manual", "_precreate_auto", "_precreate_manual") + __slots__ = ( + "_db_engines", + "database_uri", + "_postcreate_auto", + "_postcreate_manual", + "_precreate_auto", + "_precreate_manual", + ) def __init__(self, database_uri: Union[str, None] = None) -> None: self._db_engines = {} diff --git a/sql_db_utils/session_management.py b/sql_db_utils/session_management.py index ec243a4..be37210 100644 --- a/sql_db_utils/session_management.py +++ b/sql_db_utils/session_management.py @@ -13,7 +13,14 @@ class SQLSessionManager: - __slots__ = ("_db_engines", "database_uri", "_postcreate_auto", "_postcreate_manual", "_precreate_auto", "_precreate_manual") + __slots__ = ( + "_db_engines", + "database_uri", + "_postcreate_auto", + "_postcreate_manual", + "_precreate_auto", + "_precreate_manual", + ) def __init__(self, database_uri: Union[str, None] = None) -> None: self._db_engines = {} From 1ace79b6eca1958d6be2423d0ccfc0e48c3f1bfe Mon Sep 17 00:00:00 2001 From: Faizan Azim Date: Sat, 27 Sep 2025 10:36:20 +0000 Subject: [PATCH 4/4] chore(version): update version to 1.2.0 --- sql_db_utils/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql_db_utils/__version__.py b/sql_db_utils/__version__.py index a82b376..c68196d 100644 --- a/sql_db_utils/__version__.py +++ b/sql_db_utils/__version__.py @@ -1 +1 @@ -__version__ = "1.1.1" +__version__ = "1.2.0"