Merge pull request #29 from 0xdomyz/dev0.0.8

Dev0.0.8

.gitignore

@@ -1,3 +1,6 @@
+# development environment
+.vscode/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.rst

@@ -1,14 +1,24 @@
-DWOPT - Datawarehouse Operator
-==============================
+DWOPT - SQL query abstraction library
+========================================
 
 Getting summary statistics out of database tables is often an unstreamlined process.
 Does one read in millions of rows before doing any work on Python,
-or run sql elsewhere and use intermediate CSVs,
+or run SQL elsewhere and use intermediate CSVs,
 or write sql strings in python scripts?
 
-The Python package **dwopt**
-provides Excel-pivot-table-like and dataframe-summary-methods-like API,
-driven by sql templates, under a flexible summary query building framework.
+The Python package **dwopt** (Datawarehouse Operator)
+uses classes with a collection of sql templates to dynamically build and run queries,
+under a flexible summary query building framework.
+
+Specifically, it features Excel-pivot-table-like API,
+a collection of dataframe-summary-methods-like API,
+and a collection of DDL/DML statements, metadata query wrappers.
+
+Supports:
+
+* Python 3.10, 3.11.
+* Windows 10: Sqlite, Postgres, Oracle.
+* Linux: Sqlite, Postgres.
 
 See the Features and the Walk Through section for examples.
 
@@ -22,6 +32,14 @@ Installation
 
     pip install dwopt
 
+Install the database drivers for the database engines you want to use.
+
+.. code-block:: console
+
+    pip install psycopg2 # postgres
+    pip install psycopg2-binary # in case can't build psycopg2
+    
+    pip install oracledb # oracle
 
 Features
 --------
@@ -83,11 +101,18 @@ and the database engine url to access database.
         n
     0  32
 
-Supports:
+Allows for thick mode connection to Oracle database:
 
-* Python 3.9, 3.10, 3.11.
-* Windows 10: Sqlite, Postgre, Oracle.
-* Linux: Sqlite, Postgre.
+.. code-block:: python
+
+    from dwopt import db
+    url = """oracle+oracledb://dwopt_test:1234@localhost:1521/?service_name=XEPDB1 
+    &encoding=UTF-8&nencoding=UTF-8"""
+    lib_dir = "C:/app/{user_name}/product/21c/dbhomeXE/bin"
+    o = db(url, thick_mode={"lib_dir": lib_dir})
+    o.run("select * from dual")
+      dummy
+    0     X
 
 See `Testing`_ section for package version tested.
 
@@ -101,7 +126,8 @@ Use the database operator object's
 |run|_ method to run sql script file.
 One could then replace ``:`` marked parameters via mappings supplied to the method.
 
-Colon syntax is to be depreciated. A future version will use jinja2 syntax across the board.
+Colon syntax is to be deprecated.
+A future version will use jinja2 syntax across the board.
 
 .. code-block:: python
 
@@ -456,6 +482,14 @@ Example configuration to print on console and store on file with timestamps:
     )
     logging.getLogger('dwopt.dbo').setLevel(logging.INFO)
 
+Debug logging:
+
+.. code-block:: python
+
+    import logging
+    logging.basicConfig()
+    logging.getLogger('dwopt').setLevel(logging.DEBUG)
+
 Sqlalchemy logger can also be used to obtain even more details:
 
 .. code-block:: python
@@ -475,14 +509,27 @@ Testing, documentation building package:
 
 .. code-block:: console
 
+    #venv on linux
+    sudo apt-get install python3-venv
+    python3.11 -m venv dwopt_dev
+    source dwopt_dev/bin/activate
+    deactivate
+
     #testing
     python -m pip install pytest black flake8 tox
     
     #doco and packaging
-    python -m pip install sphinx sphinx_rtd_theme build twine
+    python -m pip install sphinx sphinx_rtd_theme build twine wheel
     
     #depend
-    python -m pip install -U sqlalchemy psycopg2 cx_Oracle pandas keyring
+    python -m pip install -U sqlalchemy pandas keyring
+    python -m pip install -U keyrings.alt
+    python -m pip install -U psycopg2
+    python -m pip install -U oracledb
+    
+    #consider
+    python -m pip install -U psycopg2-binary
+    python -m pip install -U cx_Oracle
     
     #package
     python -m pip install -e .
@@ -514,22 +561,31 @@ Test code styles:
 
     flake8 src/dwopt
 
+Databases used for testings are::
+
+    Postgres 15
+    Oracle express 21c
+
 Package versions tested are::
 
-    Name: SQLAlchemy
-    Version: 2.0.7
+    Name: keyring
+    Version: 24.2.0
     ---
-    Name: psycopg2
-    Version: 2.9.5
+    Name: keyrings.alt
+    Version: 5.0.0
     ---
-    Name: cx-Oracle
-    Version: 8.3.0
+    Name: oracledb
+    Version: 1.3.2
     ---
     Name: pandas
-    Version: 1.5.3
+    Version: 2.0.3
     ---
-    Name: keyring
-    Version: 23.13.1
+    Name: psycopg2-binary
+    Version: 2.9.6
+    ---
+    Name: SQLAlchemy
+    Version: 2.0.19
+
 
 Documentation
 ^^^^^^^^^^^^^^^^^

docs/source/conf.py

@@ -15,7 +15,7 @@
 project = "dwopt"
 copyright = "2023, 0xdomyz"
 author = "0xdomyz"
-release = "0.0.7"
+release = "0.0.8"
 
 
 # -- General configuration ---------------------------------------------------

setup.cfg

@@ -1,16 +1,15 @@
 [metadata]
 name = dwopt
-version = 0.0.7
+version = 0.0.8
 author = 0xdomyz
 author_email = septemberwhyms@gmail.com
-description = Datawarehouse operator
+description = SQL query abstraction library
 long_description = file: README.rst
 long_description_content_type = text/x-rst
 url = https://github.com/0xdomyz/dwopt
 project_urls =
     Bug Tracker = https://github.com/0xdomyz/dwopt/issues
 classifiers =
-    Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
     License :: OSI Approved :: MIT License
@@ -23,10 +22,9 @@ package_dir =
 packages = find:
 install_requires =
     sqlalchemy
-    psycopg2
-    cx_Oracle
     pandas
     keyring
+    keyrings.alt
 
 [options.packages.find]
 where = src

src/dwopt/__init__.py

@@ -1,14 +1,31 @@
-from .dbo import db, Db, Pg, Lt, Oc
-from .set_up import save_url, make_eng, _get_url
+import logging
+
+from .dbo import Db, Lt, Oc, Pg, db
+from .set_up import _get_url, make_eng, save_url
 from .testing import make_test_tbl
 
-pg = db(_get_url("pg"))
-lt = db(_get_url("lt"))
-oc = db(_get_url("oc"))
+_logger = logging.getLogger(__name__)
+
+try:
+    url, kwargs = _get_url("lt")
+    lt = db(url, **kwargs)
+except Exception as e:
+    msg = f"Failed to initialize default lt operator: {e}"
+    _logger.debug(msg)
+    lt = msg
+
+try:
+    url, kwargs = _get_url("pg")
+    pg = db(url, **kwargs)
+except Exception as e:
+    msg = f"Failed to initialize default lt operator: {e}"
+    _logger.debug(msg)
+    pg = msg
 
-d = db
-l = lt
-m = make_test_tbl
-o = oc
-p = pg
-s = save_url
+try:
+    url, kwargs = _get_url("oc")
+    oc = db(url, **kwargs)
+except Exception as e:
+    msg = f"Failed to initialize default lt operator: {e}"
+    _logger.debug(msg)
+    oc = msg

src/dwopt/_qry.py

@@ -1152,9 +1152,7 @@ def top(self, out=None):
             if res.empty:
                 return pd.Series(index=res.columns)
             else:
-                return res.iloc[
-                    0,
-                ]
+                return res.iloc[0,]
         elif out == 1:
             self.print(q)
         elif out == 2:
@@ -1228,7 +1226,7 @@ def valc(self, group_by, agg=None, order_by=None, n=True, out=None):
 
             from dwopt import lt, make_test_tbl
             import logging
-            logging.basicConfig(level = logging.INFO)
+            # logging.basicConfig(level = logging.INFO)
             _ = make_test_tbl(lt)
             (
                 lt.qry('test').where('score>0.5', 'dte is not null', 'cat is not null')

src/dwopt/dbo.py

@@ -11,7 +11,7 @@
 _logger = logging.getLogger(__name__)
 
 
-def db(eng):
+def db(eng, **kwargs):
     """The :class:`database operator object <dwopt.dbo._Db>` factory.
 
     Args
@@ -24,6 +24,8 @@ def db(eng):
         Alternatively a Database connection engine to be used.
         Use the :func:`dwopt.make_eng` function to make engine.
 
+    kwargs: Additional engine creation arguments.
+
     Returns
     -------
     dwopt.dbo._Db
@@ -57,12 +59,23 @@ def db(eng):
     Produce an oracle database operator object:
 
     >>> from dwopt import db, Oc
-    >>> url = "oracle://scott2:tiger@tnsname"
+    >>> url = "oracle+oracledb://scott2:tiger@tnsname"
     >>> isinstance(db(url), Oc)
     True
+
+    Use additional engine creation arguments::
+
+        from dwopt import db
+        url = (
+            "oracle+oracledb://dwopt_test:1234@localhost:1521/?service_name=XEPDB1 "
+            "&encoding=UTF-8&nencoding=UTF-8"
+        )
+        lib_dir = "C:/app/{user_name}/product/21c/dbhomeXE/bin"
+        o = db(url, thick_mode={"lib_dir": lib_dir})
+        o.run("select * from dual")
     """
     if isinstance(eng, str):
-        eng = alc.create_engine(eng)
+        eng = alc.create_engine(eng, **kwargs)
     else:
         if not isinstance(eng, alc.engine.Engine):
             raise ValueError("Invalid eng, either engine url or engine")
@@ -74,12 +87,12 @@ def db(eng):
     elif nme == "oracle":
         return Oc(eng)
     else:
-        raise ValueError("Invalid engine, either postgre, sqlite, or oracle")
+        raise ValueError("Invalid engine, either postgres, sqlite, or oracle")
 
 
-def Db(eng):
+def Db(eng, **kwargs):
     """Alias for :func:`dwopt.db`"""
-    return db(eng)
+    return db(eng, **kwargs)
 
 
 class _Db:
@@ -117,6 +130,7 @@ class _Db:
 
         Alternatively a Database connection engine to be used.
         Use the :func:`dwopt.make_eng` function to make engine.
+    kwargs: Additional engine creation arguments.
 
     Attributes
     ----------
@@ -153,11 +167,17 @@ class _Db:
     >>> p = Pg("postgresql://dwopt_tester:1234@localhost/dwopt_test")
     >>> p.mtcars(q=1).len()
     32
+
+    Additonal engine creation arguments::
+
+        from dwopt import Pg
+        p = Pg("postgresql://dwopt_tester:1234@localhost/dwopt_test", echo=1)
+        p.mtcars()
     """
 
-    def __init__(self, eng):
+    def __init__(self, eng, **kwargs):
         if isinstance(eng, str):
-            self.eng = alc.create_engine(eng)
+            self.eng = alc.create_engine(eng, **kwargs)
         else:
             self.eng = eng
         self.meta = alc.MetaData()