diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..f54ccea --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,18 @@ +name: Deploy Sphinx documentation to Github Pages + +on: + push: + branches: [main, basic_dialect_docs] # branch to trigger deployment + +jobs: + pages: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + pages: write + id-token: write + steps: + - id: deployment + uses: sphinx-notes/pages@v3 \ No newline at end of file diff --git a/.gitignore b/.gitignore index adcbed1..b1132c3 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,6 @@ dmypy.json # VSCode .vscode + +docs/.build/ +.DS_Store \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..e837d68 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = .build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..83b1db5 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,72 @@ +# YDB SQLAlchemy Documentation + +This directory contains the documentation for YDB SQLAlchemy dialect. + +## Building Documentation + +### Prerequisites + +1. Install Sphinx and required extensions: +```bash +pip install sphinx sphinx-rtd-theme sphinx-copybutton +``` + +### Building HTML Documentation + +1. Navigate to the docs directory: +```bash +cd docs +``` + +2. Build the documentation: +```bash +make html +``` + +3. Open the documentation in your browser: +```bash +open .build/html/index.html +``` + +### Building Other Formats + +- **PDF**: `make latexpdf` (requires LaTeX) +- **EPUB**: `make epub` +- **Man pages**: `make man` + +### Development + +When adding new documentation: + +1. Create `.rst` files in the appropriate directory +2. Add them to the `toctree` in `index.rst` +3. Rebuild with `make html` +4. Check for warnings and fix them + +### Structure + +- `index.rst` - Main documentation page +- `installation.rst` - Installation guide +- `quickstart.rst` - Quick start guide +- `connection.rst` - Connection configuration +- `types.rst` - Data types documentation +- `migrations.rst` - Alembic migrations guide +- `api/` - API reference documentation +- `conf.py` - Sphinx configuration +- `_static/` - Static files (images, CSS, etc.) + +### Configuration + +The documentation is configured in `conf.py`. Key settings: + +- **Theme**: `sphinx_rtd_theme` (Read the Docs theme) +- **Extensions**: autodoc, napoleon, intersphinx, copybutton +- **Intersphinx**: Links to Python, SQLAlchemy, and Alembic docs + +### Troubleshooting + +**Sphinx not found**: Make sure Sphinx is installed in your virtual environment + +**Import errors**: Ensure the YDB SQLAlchemy package is installed in the same environment + +**Theme issues**: Install `sphinx-rtd-theme` if you get theme-related errors diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 0000000..0a81321 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1,4 @@ + + + + diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 0000000..cfb2493 --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,52 @@ +API Reference +============= + +This section contains the complete API reference for YDB SQLAlchemy. + +Core Module +----------- + +.. automodule:: ydb_sqlalchemy.sqlalchemy + :members: + :undoc-members: + :show-inheritance: + +Types Module +------------ + +.. automodule:: ydb_sqlalchemy.sqlalchemy.types + :members: + :undoc-members: + :show-inheritance: + +DateTime Types +-------------- + +.. automodule:: ydb_sqlalchemy.sqlalchemy.datetime_types + :members: + :undoc-members: + :show-inheritance: + +JSON Types +---------- + +.. automodule:: ydb_sqlalchemy.sqlalchemy.json + :members: + :undoc-members: + :show-inheritance: + +Compiler Module +--------------- + +.. automodule:: ydb_sqlalchemy.sqlalchemy.compiler + :members: + :undoc-members: + :show-inheritance: + +DML Operations +-------------- + +.. automodule:: ydb_sqlalchemy.sqlalchemy.dml + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..f1a3093 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + + +# -- Project information ----------------------------------------------------- + +project = 'YDB SQLAlchemy' +copyright = '2025, Yandex' +author = 'Yandex' + +# The short X.Y version +version = '0.1' +# The full version, including alpha/beta/rc tags +release = '0.1.9' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'sphinx.ext.napoleon', + 'sphinx.ext.coverage', + 'sphinx.ext.intersphinx', + 'sphinx.ext.githubpages', + 'sphinx_copybutton', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['.build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +html_theme_options = { + 'fixed_sidebar': True, + 'page_width': '1140px', + 'show_related': True, + 'show_powered_by': False +} + +html_logo = '_static/logo.svg' +html_favicon = '_static/logo.svg' + +html_show_sourcelink = False + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'ydb-sqlalchemy-doc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'ydb-sqlalchemy.tex', 'YDB SQLAlchemy Documentation', + 'Yandex', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'ydb-sqlalchemy', 'YDB SQLAlchemy Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'ydb-sqlalchemy', 'YDB SQLAlchemy Documentation', + author, 'ydb-sqlalchemy', 'SQLAlchemy dialect for YDB (Yandex Database).', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- + +autoclass_content = "both" +autodoc_typehints = "both" +autodoc_default_options = { + 'undoc-members': True, + 'member-order': 'bysource' +} + +# -- Intersphinx configuration ----------------------------------------------- + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'sqlalchemy': ('https://docs.sqlalchemy.org/en/20/', None), + 'alembic': ('https://alembic.sqlalchemy.org/en/latest/', None), +} + +# -- Copy button configuration -------------------------------------------- + +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True diff --git a/docs/connection.rst b/docs/connection.rst new file mode 100644 index 0000000..986036d --- /dev/null +++ b/docs/connection.rst @@ -0,0 +1,171 @@ +Connection Configuration +======================== + +This guide covers various ways to configure connections to YDB using SQLAlchemy. + +Connection URL Format +--------------------- + +YDB SQLAlchemy uses the following URL format: + +.. code-block:: text + + yql+ydb://host:port/database + +Basic Examples: + +.. code-block:: python + + # Synchronous connection + engine = sa.create_engine("yql+ydb://localhost:2136/local") + + # Asynchronous connection + from sqlalchemy.ext.asyncio import create_async_engine + async_engine = create_async_engine("yql+ydb_async://localhost:2136/local") + + # Remote YDB instance + engine = sa.create_engine("yql+ydb://ydb.example.com:2135/prod") + async_engine = create_async_engine("yql+ydb_async://ydb.example.com:2135/prod") + + # With database path + engine = sa.create_engine("yql+ydb://localhost:2136/local/my_database") + async_engine = create_async_engine("yql+ydb_async://localhost:2136/local/my_database") + +Authentication Methods +---------------------- + +YDB SQLAlchemy supports multiple authentication methods through the ``connect_args`` parameter. + +Anonymous Access +~~~~~~~~~~~~~~~~ + +For local development or testing: + +.. code-block:: python + + import sqlalchemy as sa + + engine = sa.create_engine("yql+ydb://localhost:2136/local") + +Static Credentials +~~~~~~~~~~~~~~~~~~ + +Use username and password authentication: + +.. code-block:: python + + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": { + "username": "your_username", + "password": "your_password" + } + } + ) + +Token Authentication +~~~~~~~~~~~~~~~~~~~~ + +Use access token for authentication: + +.. code-block:: python + + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": { + "token": "your_access_token" + } + } + ) + +Service Account Authentication +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use service account JSON key: + +.. code-block:: python + + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": { + "service_account_json": { + "id": "your_key_id", + "service_account_id": "your_service_account_id", + "created_at": "2023-01-01T00:00:00Z", + "key_algorithm": "RSA_2048", + "public_key": "-----BEGIN PUBLIC KEY-----\\n...\\n-----END PUBLIC KEY-----", + "private_key": "-----BEGIN PRIVATE KEY-----\\n...\\n-----END PRIVATE KEY-----" + } + } + } + ) + +Or load from file: + +.. code-block:: python + + import json + + with open('service_account_key.json', 'r') as f: + service_account_json = json.load(f) + + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": { + "service_account_json": service_account_json + } + } + ) + +YDB SDK Credentials +~~~~~~~~~~~~~~~~~~~ + +Use any credentials from the YDB Python SDK: + +.. code-block:: python + + import ydb.iam + + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": ydb.iam.MetadataUrlCredentials() + } + ) + + # OAuth token credentials + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": ydb.iam.OAuthCredentials("your_oauth_token") + } + ) + + # Static credentials + engine = sa.create_engine( + "yql+ydb://localhost:2136/local", + connect_args={ + "credentials": ydb.iam.StaticCredentials("username", "password") + } + ) + +TLS Configuration +--------------------- + +For secure connections to YDB: + +.. code-block:: python + + engine = sa.create_engine( + "yql+ydb://ydb.example.com:2135/prod", + connect_args={ + "credentials": {"token": "your_token"}, + "protocol": "grpc", + "root_certificates_path": "/path/to/ca-certificates.crt", + # "root_certificates": crt_string, + } + ) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..2b3c289 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,93 @@ +YDB SQLAlchemy Documentation +============================ + +Welcome to the YDB SQLAlchemy dialect documentation. This package provides a SQLAlchemy dialect for YDB (Yandex Database), allowing you to use SQLAlchemy ORM and Core with YDB databases. + +.. image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg + :target: https://github.com/ydb-platform/ydb-sqlalchemy/blob/main/LICENSE + :alt: License + +.. image:: https://badge.fury.io/py/ydb-sqlalchemy.svg + :target: https://badge.fury.io/py/ydb-sqlalchemy + :alt: PyPI version + +.. image:: https://github.com/ydb-platform/ydb-sqlalchemy/actions/workflows/tests.yml/badge.svg + :target: https://github.com/ydb-platform/ydb-sqlalchemy/actions/workflows/tests.yml + :alt: Functional tests + +Overview +-------- + +YDB SQLAlchemy is a dialect that enables SQLAlchemy to work with YDB databases. It supports both SQLAlchemy 2.0 (fully tested) and SQLAlchemy 1.4 (partially tested). + +Key Features: +~~~~~~~~~~~~~ + +* **SQLAlchemy 2.0 Support**: Full compatibility with the latest SQLAlchemy version +* **Async/Await Support**: Full async support with ``yql+ydb_async`` dialect +* **Core and ORM**: Support for both SQLAlchemy Core and ORM patterns +* **Authentication**: Multiple authentication methods including static credentials, tokens, and service accounts +* **Type System**: Comprehensive YDB type mapping to SQLAlchemy types +* **Migrations**: Alembic integration for database schema migrations +* **Pandas Integration**: Compatible with pandas DataFrame operations + +Quick Examples +~~~~~~~~~~~~~~ + +**Synchronous:** + +.. code-block:: python + + import sqlalchemy as sa + + # Create engine + engine = sa.create_engine("yql+ydb://localhost:2136/local") + + # Execute query + with engine.connect() as conn: + result = conn.execute(sa.text("SELECT 1 AS value")) + print(result.fetchone()) + +**Asynchronous:** + +.. code-block:: python + + import asyncio + from sqlalchemy.ext.asyncio import create_async_engine + + async def main(): + # Create async engine + engine = create_async_engine("yql+ydb_async://localhost:2136/local") + + # Execute query + async with engine.connect() as conn: + result = await conn.execute(sa.text("SELECT 1 AS value")) + print(await result.fetchone()) + + asyncio.run(main()) + +Table of Contents +----------------- + +.. toctree:: + :maxdepth: 2 + :caption: User Guide + + installation + quickstart + connection + types + migrations + +.. toctree:: + :maxdepth: 2 + :caption: API Reference + + api/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..9759764 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,89 @@ +Installation +============ + +This guide covers the installation of YDB SQLAlchemy dialect and its dependencies. + +Requirements +------------ + +* Python 3.7 or higher +* SQLAlchemy 1.4+ or 2.0+ (recommended) +* YDB Python SDK + +Installing from PyPI +--------------------- + +The easiest way to install YDB SQLAlchemy is using pip: + +.. code-block:: bash + + pip install ydb-sqlalchemy + +This will install the YDB SQLAlchemy dialect along with all required dependencies. + +Installing from Source +---------------------- + +If you want to install the latest development version or contribute to the project: + +1. Clone the repository: + +.. code-block:: bash + + git clone https://github.com/ydb-platform/ydb-sqlalchemy.git + cd ydb-sqlalchemy + +2. Install in development mode: + +.. code-block:: bash + + pip install -e . + +Or install directly: + +.. code-block:: bash + + pip install . + + +Verifying Installation +---------------------- + +To verify that YDB SQLAlchemy is installed correctly: + +.. code-block:: python + + import ydb_sqlalchemy + import sqlalchemy as sa + + # Check if the dialect is available + engine = sa.create_engine("yql+ydb://localhost:2136/local") + print("YDB SQLAlchemy installed successfully!") + +Docker Setup for Development +----------------------------- + +For development and testing, you can use Docker to run a local YDB instance: + +1. Clone the repository and navigate to the project directory +2. Start YDB using ``docker compose``: + +.. code-block:: bash + + docker compose up -d + +This will start a YDB instance accessible at ``localhost:2136``. + +Getting Help +~~~~~~~~~~~~ + +If you encounter issues during installation: + +1. Check the `GitHub Issues `_ +2. Review the `YDB documentation `_ +3. Create a new issue with detailed error information + +Next Steps +---------- + +After successful installation, proceed to the :doc:`quickstart` guide to learn how to use YDB SQLAlchemy in your projects. diff --git a/docs/migrations.rst b/docs/migrations.rst new file mode 100644 index 0000000..ebbdbdf --- /dev/null +++ b/docs/migrations.rst @@ -0,0 +1,484 @@ +Database Migrations with Alembic +================================ + +This guide covers how to use Alembic for database schema migrations with YDB SQLAlchemy. + +Overview +-------- + +Alembic is SQLAlchemy's database migration tool that allows you to: + +- Track database schema changes over time +- Apply incremental schema updates +- Rollback to previous schema versions +- Generate migration scripts automatically + +YDB SQLAlchemy provides full Alembic integration with some YDB-specific considerations. + +Installation +------------ + +Install Alembic alongside YDB SQLAlchemy: + +.. code-block:: bash + + pip install alembic ydb-sqlalchemy + +Initial Setup +------------- + +1. Initialize Alembic in your project: + +.. code-block:: bash + + alembic init migrations + +This creates an ``alembic.ini`` configuration file and a ``migrations/`` directory. + +2. Configure ``alembic.ini``: + +.. code-block:: ini + + # alembic.ini + [alembic] + script_location = migrations + prepend_sys_path = . + version_path_separator = os + + # YDB connection string + sqlalchemy.url = yql+ydb://localhost:2136/local + + [post_write_hooks] + + [loggers] + keys = root,sqlalchemy,alembic + + [handlers] + keys = console + + [formatters] + keys = generic + + [logger_root] + level = WARN + handlers = console + qualname = + + [logger_sqlalchemy] + level = WARN + handlers = + qualname = sqlalchemy.engine + + [logger_alembic] + level = INFO + handlers = + qualname = alembic + + [handler_console] + class = StreamHandler + args = (sys.stderr,) + level = NOTSET + formatter = generic + + [formatter_generic] + format = %(levelname)-5.5s [%(name)s] %(message)s + datefmt = %H:%M:%S + +YDB-Specific Configuration +-------------------------- + +YDB requires special configuration in ``env.py`` due to its unique characteristics: + +.. code-block:: python + + # migrations/env.py + from logging.config import fileConfig + import sqlalchemy as sa + from sqlalchemy import engine_from_config, pool + from alembic import context + from alembic.ddl.impl import DefaultImpl + + # Import your models + from myapp.models import Base + + config = context.config + + if config.config_file_name is not None: + fileConfig(config.config_file_name) + + target_metadata = Base.metadata + + # YDB-specific implementation + class YDBImpl(DefaultImpl): + __dialect__ = "yql" + + def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + # YDB-specific: Custom version table structure + ctx = context.get_context() + ctx._version = sa.Table( + ctx.version_table, + sa.MetaData(), + sa.Column("version_num", sa.String(32), nullable=False), + sa.Column("id", sa.Integer(), nullable=True, primary_key=True), + ) + + with context.begin_transaction(): + context.run_migrations() + + if context.is_offline_mode(): + run_migrations_offline() + else: + run_migrations_online() + +Creating Your First Migration +----------------------------- + +1. Define your models: + +.. code-block:: python + + # models.py + from sqlalchemy import Column, String, Integer + from sqlalchemy.ext.declarative import declarative_base + from ydb_sqlalchemy.sqlalchemy.types import UInt64 + + Base = declarative_base() + + class User(Base): + __tablename__ = 'users' + + id = Column(UInt64, primary_key=True) + username = Column(String(50), nullable=False) + email = Column(String(100), nullable=False) + full_name = Column(String(200)) + +2. Generate the initial migration: + +.. code-block:: bash + + alembic revision --autogenerate -m "Create users table" + +This creates a migration file like ``001_create_users_table.py``: + +.. code-block:: python + + """Create users table + + Revision ID: 001 + Revises: + Create Date: 2024-01-01 12:00:00.000000 + """ + from alembic import op + import sqlalchemy as sa + from ydb_sqlalchemy.sqlalchemy.types import UInt64 + + revision = '001' + down_revision = None + branch_labels = None + depends_on = None + + def upgrade() -> None: + op.create_table('users', + sa.Column('id', UInt64(), nullable=False), + sa.Column('username', sa.String(length=50), nullable=False), + sa.Column('email', sa.String(length=100), nullable=False), + sa.Column('full_name', sa.String(length=200), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + + def downgrade() -> None: + op.drop_table('users') + +3. Apply the migration: + +.. code-block:: bash + + alembic upgrade head + +Common Migration Operations +--------------------------- + +Adding a Column +~~~~~~~~~~~~~~~ + +.. code-block:: python + + # Add a new column + def upgrade() -> None: + op.add_column('users', sa.Column('created_at', sa.DateTime(), nullable=True)) + + def downgrade() -> None: + op.drop_column('users', 'created_at') + +Modifying a Column +~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + # Change column type (be careful with YDB limitations) + def upgrade() -> None: + op.alter_column('users', 'username', + existing_type=sa.String(50), + type_=sa.String(100), + nullable=False) + + def downgrade() -> None: + op.alter_column('users', 'username', + existing_type=sa.String(100), + type_=sa.String(50), + nullable=False) + +Creating Indexes +~~~~~~~~~~~~~~~~ + +.. code-block:: python + + def upgrade() -> None: + op.create_index('ix_users_email', 'users', ['email']) + + def downgrade() -> None: + op.drop_index('ix_users_email', table_name='users') + +Adding a New Table +~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + def upgrade() -> None: + op.create_table('posts', + sa.Column('id', UInt64(), nullable=False), + sa.Column('user_id', UInt64(), nullable=False), + sa.Column('title', sa.String(200), nullable=False), + sa.Column('content', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['user_id'], ['users.id']) + ) + + def downgrade() -> None: + op.drop_table('posts') + +YDB-Specific Considerations +--------------------------- + +Primary Key Limitations +~~~~~~~~~~~~~~~~~~~~~~~~ + +YDB doesn't support modifying primary key columns. Plan your primary keys carefully: + +.. code-block:: python + + # Good: Use appropriate primary key from the start + class User(Base): + __tablename__ = 'users' + id = Column(UInt64, primary_key=True) # Can't be changed later + + # If you need to change primary key structure, you'll need to: + # 1. Create new table with correct primary key + # 2. Migrate data + # 3. Drop old table + # 4. Rename new table + +Data Type Constraints +~~~~~~~~~~~~~~~~~~~~~ + +Some type changes are not supported: + +.. code-block:: python + + # Supported: Increasing string length + op.alter_column('users', 'username', + existing_type=sa.String(50), + type_=sa.String(100)) + + # Not supported: Changing fundamental type + # op.alter_column('users', 'id', + # existing_type=UInt32(), + # type_=UInt64()) # This won't work + +Working with YDB Types +~~~~~~~~~~~~~~~~~~~~~~ + +Use YDB-specific types in migrations: + +.. code-block:: python + + from ydb_sqlalchemy.sqlalchemy.types import ( + UInt64, UInt32, Decimal, YqlJSON, YqlDateTime + ) + + def upgrade() -> None: + op.create_table('financial_records', + sa.Column('id', UInt64(), nullable=False), + sa.Column('amount', Decimal(precision=15, scale=2), nullable=False), + sa.Column('metadata', YqlJSON(), nullable=True), + sa.Column('created_at', YqlDateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + +Advanced Migration Patterns +--------------------------- + +Data Migrations +~~~~~~~~~~~~~~~ + +Sometimes you need to migrate data along with schema: + +.. code-block:: python + + from alembic import op + import sqlalchemy as sa + from sqlalchemy.sql import table, column + + def upgrade() -> None: + # Add new column + op.add_column('users', sa.Column('status', sa.String(20), nullable=True)) + + # Create a temporary table representation for data migration + users_table = table('users', + column('id', UInt64), + column('status', sa.String) + ) + + # Update existing records + op.execute( + users_table.update().values(status='active') + ) + + # Make column non-nullable + op.alter_column('users', 'status', nullable=False) + + def downgrade() -> None: + op.drop_column('users', 'status') + +Conditional Migrations +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + def upgrade() -> None: + # Check if column already exists + conn = op.get_bind() + inspector = sa.inspect(conn) + columns = [col['name'] for col in inspector.get_columns('users')] + + if 'new_column' not in columns: + op.add_column('users', sa.Column('new_column', sa.String(50))) + +Migration Best Practices +------------------------ + +1. **Test Migrations**: Always test migrations on a copy of production data +2. **Backup Data**: Backup your data before running migrations in production +3. **Review Generated Migrations**: Always review auto-generated migrations before applying +4. **Use Transactions**: Migrations run in transactions by default +5. **Plan Primary Keys**: Design primary keys carefully as they can't be easily changed + +.. code-block:: python + + # Good migration practices + def upgrade() -> None: + # Add columns as nullable first + op.add_column('users', sa.Column('new_field', sa.String(100), nullable=True)) + + # Populate data + # ... data migration code ... + + # Then make non-nullable if needed + op.alter_column('users', 'new_field', nullable=False) + +Common Commands +--------------- + +.. code-block:: bash + + # Generate new migration + alembic revision --autogenerate -m "Description of changes" + + # Apply all pending migrations + alembic upgrade head + + # Apply specific migration + alembic upgrade revision_id + + # Rollback one migration + alembic downgrade -1 + + # Rollback to specific revision + alembic downgrade revision_id + + # Show current revision + alembic current + + # Show migration history + alembic history + + # Show pending migrations + alembic show head + +Troubleshooting +--------------- + +**Migration Fails with "Table already exists"** + - Check if migration was partially applied + - Use ``alembic stamp head`` to mark current state without running migrations + +**Primary Key Constraint Errors** + - YDB requires primary keys on all tables + - Ensure all tables have appropriate primary keys + +**Type Conversion Errors** + - Some type changes aren't supported in YDB + - Create new column, migrate data, drop old column instead + +**Connection Issues** + - Verify YDB is running and accessible + - Check connection string in ``alembic.ini`` + +Example Project Structure +------------------------- + +.. code-block:: text + + myproject/ + ├── alembic.ini + ├── migrations/ + │ ├── env.py + │ ├── script.py.mako + │ └── versions/ + │ ├── 001_create_users_table.py + │ ├── 002_add_posts_table.py + │ └── 003_add_user_status.py + ├── models/ + │ ├── __init__.py + │ ├── user.py + │ └── post.py + └── main.py + +This setup provides a robust foundation for managing YDB schema changes over time using Alembic migrations. diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000..974a3c2 --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,224 @@ +Quick Start +=========== + +This guide will help you get started with YDB SQLAlchemy quickly. We'll cover basic usage patterns for both SQLAlchemy Core and ORM. + +Prerequisites +------------- + +Before starting, make sure you have: + +1. YDB SQLAlchemy installed (see :doc:`installation`) +2. A running YDB instance (local or remote) +3. Basic familiarity with SQLAlchemy + +Basic Connection +---------------- + +The simplest way to connect to YDB: + +.. code-block:: python + + import sqlalchemy as sa + + # Create engine for local YDB + engine = sa.create_engine("yql+ydb://localhost:2136/local") + + # Test connection + with engine.connect() as conn: + result = conn.execute(sa.text("SELECT 1 AS value")) + print(result.fetchone()) # (1,) + +SQLAlchemy Core Example +----------------------- + +Using SQLAlchemy Core for direct SQL operations: + +.. code-block:: python + + import sqlalchemy as sa + from sqlalchemy import MetaData, Table, Column, Integer, String + + # Create engine + engine = sa.create_engine("yql+ydb://localhost:2136/local") + + # Define table structure + metadata = MetaData() + users = Table( + 'users', + metadata, + Column('id', Integer, primary_key=True), + Column('name', String(50)), + Column('email', String(100)) + ) + + # Create table + metadata.create_all(engine) + + # Insert data + with engine.connect() as conn: + # Single insert + conn.execute( + users.insert().values(id=1, name='John Doe', email='john@example.com') + ) + + # Multiple inserts + conn.execute( + users.insert(), + [ + {'id': 2, 'name': 'Jane Smith', 'email': 'jane@example.com'}, + {'id': 3, 'name': 'Bob Johnson', 'email': 'bob@example.com'} + ] + ) + + # Commit changes + conn.commit() + + # Query data + with engine.connect() as conn: + # Select all + result = conn.execute(sa.select(users)) + for row in result: + print(f"ID: {row.id}, Name: {row.name}, Email: {row.email}") + + # Select with conditions + result = conn.execute( + sa.select(users).where(users.c.name.like('John%')) + ) + print(result.fetchall()) + +SQLAlchemy ORM Example +---------------------- + +Using SQLAlchemy ORM for object-relational mapping: + +.. code-block:: python + + import sqlalchemy as sa + from sqlalchemy import Column, Integer, String + from sqlalchemy.orm import declarative_base + from sqlalchemy.orm import sessionmaker + + # Create engine + engine = sa.create_engine("yql+ydb://localhost:2136/local") + + # Define base class + Base = declarative_base() + + # Define model + class User(Base): + __tablename__ = 'users_orm' + + id = Column(Integer, primary_key=True) + name = Column(String(50)) + email = Column(String(100)) + + def __repr__(self): + return f"" + + # Create tables + Base.metadata.create_all(engine) + + # Create session + Session = sessionmaker(bind=engine) + session = Session() + + # Create and add users + user1 = User(id=1, name='Alice Brown', email='alice@example.com') + user2 = User(id=2, name='Charlie Davis', email='charlie@example.com') + + session.add_all([user1, user2]) + session.commit() + + # Query users + users = session.query(User).all() + for user in users: + print(user) + + # Query with filters + alice = session.query(User).filter(User.name == 'Alice Brown').first() + print(f"Found user: {alice}") + + # Update user + alice.email = 'alice.brown@example.com' + session.commit() + + # Delete user + session.delete(user2) + session.commit() + + session.close() + +Working with YDB-Specific Features +----------------------------------- + +YDB has some unique features that you can leverage: + +Upsert Operations +~~~~~~~~~~~~~~~~~ + +YDB supports efficient upsert operations: + +.. code-block:: python + + from ydb_sqlalchemy.sqlalchemy import upsert + + # Using upsert instead of insert + with engine.connect() as conn: + stmt = upsert(users).values( + id=1, + name='John Updated', + email='john.updated@example.com' + ) + conn.execute(stmt) + conn.commit() + +YDB-Specific Types +~~~~~~~~~~~~~~~~~~ + +Use YDB-specific data types for better performance: + +.. code-block:: python + + from ydb_sqlalchemy.sqlalchemy.types import UInt64, YqlJSON + + # Table with YDB-specific types + ydb_table = Table( + 'ydb_example', + metadata, + Column('id', UInt64, primary_key=True), + Column('data', YqlJSON), + Column('created_at', sa.DateTime) + ) + +Next Steps +---------- + +Now that you have the basics working: + +1. Learn about :doc:`connection` configuration and authentication +2. Explore :doc:`types` for YDB-specific data types +3. Set up :doc:`migrations` with Alembic +4. Check out the examples in the repository + +Common Patterns +--------------- + +Here are some common patterns you'll use frequently: + +.. code-block:: python + + # Counting records + count = conn.execute(sa.func.count(users.c.id)).scalar() + + # Aggregations + result = conn.execute( + sa.select(sa.func.max(users.c.id), sa.func.count()) + .select_from(users) + ) + + # Joins (when you have related tables) + # result = conn.execute( + # sa.select(users, orders) + # .select_from(users.join(orders, users.c.id == orders.c.user_id)) + # ) diff --git a/docs/types.rst b/docs/types.rst new file mode 100644 index 0000000..25bbe0c --- /dev/null +++ b/docs/types.rst @@ -0,0 +1,217 @@ +Data Types +========== + +YDB SQLAlchemy provides comprehensive support for YDB data types through custom SQLAlchemy types. This guide covers the available types and their usage. + +Overview +-------- + +YDB has a rich type system that includes primitive types, optional types, containers, and special types. The YDB SQLAlchemy dialect maps these types to appropriate SQLAlchemy types and provides YDB-specific types for optimal performance. +For more information about YDB data types, see the `YDB Type System Documentation `_. + +Type Mapping Summary +-------------------- + +The following table shows the complete mapping between YDB native types, SQLAlchemy types, and Python types: + +.. list-table:: YDB Type System Reference + :header-rows: 1 + :widths: 20 25 20 35 + + * - YDB Native Type + - SQLAlchemy Type + - Python Type + - Notes + * - ``Bool`` + - ``BOOLEAN`` + - ``bool`` + - + * - ``Int8`` + - + - ``int`` + - -2^7 to 2^7-1 + * - ``Int16`` + - + - ``int`` + - -2^15 to 2^15-1 + * - ``Int32`` + - + - ``int`` + - -2^31 to 2^31-1 + * - ``Int64`` + - ``INTEGER`` + - ``int`` + - -2^63 to 2^63-1 + * - ``Uint8`` + - + - ``int`` + - 0 to 2^8-1 + * - ``Uint16`` + - + - ``int`` + - 0 to 2^16-1 + * - ``Uint32`` + - + - ``int`` + - 0 to 2^32-1 + * - ``Uint64`` + - + - ``int`` + - 0 to 2^64-1 + * - ``Float`` + - ``FLOAT`` + - ``float`` + - + * - ``Double`` + - ``Double`` + - ``float`` + - Available in SQLAlchemy 2.0+ + * - ``Decimal(p,s)`` + - ``DECIMAL`` / ``NUMERIC`` + - ``decimal.Decimal`` + - + * - ``String`` + - ``BINARY`` / ``BLOB`` + - ``str`` / ``bytes`` + - + * - ``Utf8`` + - ``CHAR`` / ``VARCHAR`` / ``TEXT`` / ``NVARCHAR`` + - ``str`` + - + * - ``Date`` + - ``Date`` + - ``datetime.date`` + - + * - ``Datetime`` + - ``DATETIME`` + - ``datetime.datetime`` + - + * - ``Timestamp`` + - ``TIMESTAMP`` + - ``datetime.datetime`` + - + * - ``Json`` + - ``JSON`` + - ``dict`` / ``list`` + - + * - ``List`` + - ``ARRAY`` + - ``list`` + - + * - ``Struct<...>`` + - + - ``dict`` + - + * - ``Optional`` + - ``nullable=True`` + - ``None`` + base type + - + +Standard SQLAlchemy Types +------------------------- + +Most standard SQLAlchemy types work with YDB: + +.. code-block:: python + + from sqlalchemy import Column, Integer, String, Boolean, Float, Text + + class MyTable(Base): + __tablename__ = 'my_table' + + id = Column(Integer, primary_key=True) + name = Column(String(100)) + description = Column(Text) + is_active = Column(Boolean) + price = Column(Float) + +YDB-Specific Integer Types +-------------------------- + +YDB provides specific integer types with defined bit widths: + +.. code-block:: python + + from ydb_sqlalchemy.sqlalchemy.types import ( + Int8, Int16, Int32, Int64, + UInt8, UInt16, UInt32, UInt64 + ) + + class IntegerTypesExample(Base): + __tablename__ = 'integer_types' + + id = Column(UInt64, primary_key=True) # Unsigned 64-bit integer + small_int = Column(Int16) # Signed 16-bit integer + byte_value = Column(UInt8) # Unsigned 8-bit integer (0-255) + counter = Column(UInt32) # Unsigned 32-bit integer + +Decimal Type +------------ + +YDB supports high-precision decimal numbers: + +.. code-block:: python + + from ydb_sqlalchemy.sqlalchemy.types import Decimal + import decimal + + class FinancialData(Base): + __tablename__ = 'financial_data' + + id = Column(UInt64, primary_key=True) + # Default: Decimal(22, 9) - 22 digits total, 9 after decimal point + amount = Column(Decimal()) + + # Custom precision and scale + precise_amount = Column(Decimal(precision=15, scale=4)) + + # Return as float instead of Decimal object + percentage = Column(Decimal(precision=5, scale=2, asdecimal=False)) + + # Usage + session.add(FinancialData( + id=1, + amount=decimal.Decimal('1234567890123.123456789'), + precise_amount=decimal.Decimal('12345678901.1234'), + percentage=99.99 + )) + +Date and Time Types +------------------- + +YDB provides several date and time types: + +.. code-block:: python + + from ydb_sqlalchemy.sqlalchemy.types import YqlDate, YqlDateTime, YqlTimestamp + from sqlalchemy import DateTime + import datetime + + class EventLog(Base): + __tablename__ = 'event_log' + + id = Column(UInt64, primary_key=True) + + # Date only (YYYY-MM-DD) + event_date = Column(YqlDate) + + # DateTime with timezone support + created_at = Column(YqlDateTime(timezone=True)) + + # Timestamp (high precision) + precise_time = Column(YqlTimestamp) + + # Standard SQLAlchemy DateTime also works + updated_at = Column(DateTime) + + # Usage + now = datetime.datetime.now(datetime.timezone.utc) + today = datetime.date.today() + + session.add(EventLog( + id=1, + event_date=today, + created_at=now, + precise_time=now, + updated_at=now + ))