Merge pull request #18 from marchinho11/maintenance

* Update packages * Migrate psycopg to v3 * Update logo * Remove `ruff`
marchinho11 · Mar 1, 2024 · 5888ad8 · 5888ad8
2 parents dc97308 + ad28323
commit 5888ad8
Show file tree

Hide file tree

Showing 20 changed files with 616 additions and 599 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -9,12 +9,9 @@ jobs:
 
       - uses: psf/black@stable
         with:
-          version: "23.9.1"
+          version: "24.2.0"
 
       - uses: isort/isort-action@v1
         with:
-          isort-version: "5.12.0"
+          isort-version: "5.13.2"
 
-      - uses: chartboost/ruff-action@v1
-        with:
-          version: "0.0.292"
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -30,7 +30,7 @@ jobs:
       - name: Install poetry
         run: pipx install poetry==1.6.1
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "poetry"

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -15,7 +15,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:15
+        image: postgres:16.2
         env:
           POSTGRES_DB: ${{ env.PG_DB }}
           POSTGRES_USER: ${{ env.PG_USER }}
@@ -32,9 +32,9 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install poetry
-        run: pipx install poetry==1.6.1
+        run: pipx install poetry==1.8.1
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "poetry"
@@ -48,7 +48,7 @@ jobs:
           PG_DB: ${{ env.PG_DB }}
           PG_USER: ${{ env.PG_USER }}
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.codecov_token }}
           files: ./coverage.xml

diff --git a/.gitignore b/.gitignore
@@ -1,18 +1,14 @@
 .idea/
 .vscode/
-
 dist/
 .pytest_cache/
 .mypy_cache/
 .ruff_cache/
-**/__pycache__/*
-
+__pycache__/
 .coverage
 htmlcov/
-
 .env
-
+.DS_Store
 state.json
-coinmarket.json
+docs/notes.md
 
-docs/build
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,22 +1,17 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.9.1
+    rev: 24.2.0
     hooks:
       - id: black
         language_version: python3.10
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.292
-    hooks:
-      - id: ruff
-
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.6.0
+    rev: v1.8.0
     hooks:
       - id: mypy
         exclude: ^tests/
diff --git a/README.md b/README.md
@@ -1,9 +1,6 @@
 # <img src="https://raw.githubusercontent.com/marchinho11/hnhm/main/docs/assets/logo.png" height="40px">
 [![codecov](https://codecov.io/gh/marchinho11/hnhm/branch/main/graph/badge.svg?token=PFB1111T2D)](https://codecov.io/gh/marchinho11/hnhm)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Downloads](https://static.pepy.tech/personalized-badge/hnhm?period=week&units=international_system&left_color=black&right_color=brightgreen&left_text=PyPi%20/%20week)](https://pepy.tech/project/hnhm)
-[![Downloads](https://static.pepy.tech/personalized-badge/hnhm?period=total&units=international_system&left_color=black&right_color=brightgreen&left_text=PyPi%20/%20total)](https://pepy.tech/project/hnhm)
-![Visitors](https://api.visitorbadge.io/api/combined?path=marchinho11%2Fhnhm&label=Visitors&labelColor=%23000000&countColor=%2347c21a&style=flat&labelStyle=none)
 
 **hNhM**(highly Normalized hybrid Model) – data modeling methodology that enables iterative and Agile-driven
  modifications to your Data Warehouse (DWH). The methodology is based on Anchor Modeling and Data Vault.
@@ -16,7 +13,7 @@ The idea is to represent each Entity using 3 types of tables:
   **The best practice is to define a Group by the same data source.**
 
 
-So, the purpose of `hnhm` is to:
+The purpose of `hnhm` is to:
 * **Define** Entities, Links and Flows *declaratively* using Python
   * Describe them "logically"
   * The "physical" layer is managed by `hnhm`
@@ -42,9 +39,6 @@ Implementation of this package is based on report
   + [Physical level (Tables)](#physical-level-tables)
   + [Change types of Attributes and Groups](#change-types-of-attributes-and-groups)
 * [Supported Databases](#supported-databases)
-* [Future plans](#future-plans)
-* [Contribute!](#contribute)
-* [Created with](#created-with)
 
 
 ## Tutorial
@@ -292,46 +286,4 @@ view: entity__user
 
 ## Supported Databases
 - [x] Postgres
-- [ ] GreenPlum
-- [ ] Snowflake
-- [ ] BigQuery
-- [ ] ClickHouse
-
-
-## Future plans
-* New database connectors
-  * Snowflake
-  * BigQuery
-  * GreenPlum
-  * (?) ClickHouse
-* Tests
-  * More elegant DSL
-* State management
-  * Keep history in the database
-  * Refresh: drift detection
-  * Rollback
-  * Generate static documentation for DWH
-* Robustness
-  * Reliability
-  * Benchmarks
-  * SQL optimization, refinement of Jinja code
-* Documentation
-  * Documentation for Python sources
-  * Guides, best practices, and recipes
-* Code Style & CI
-  * Testing matrix
-
-## Contribute!
-What contributions are welcome? (**spoiler: any!**)
-- New database connectors
-- New features
-- Typos correction
-- Dependencies update
-- Documentation improvement
-- Code refinement
-- Bug fixes
-
-## Created with
-| <img height="20" src="https://upload.wikimedia.org/wikipedia/commons/1/1d/PyCharm_Icon.svg"> PyCharm Professional |
-|-------------------------------------------------------------------------------------------------------------------|
-| <img height="20" src="https://upload.wikimedia.org/wikipedia/commons/3/3a/Neovim-mark.svg"> Neovim                |
+
diff --git a/docs/assets/logo.png b/docs/assets/logo.png
diff --git a/hnhm/__init__.py b/hnhm/__init__.py
@@ -28,4 +28,4 @@
     "Boolean",
 ]
 
-__version__ = "0.0.10"
+__version__ = "0.0.11"
diff --git a/hnhm/core/migration.py b/hnhm/core/migration.py
@@ -15,6 +15,7 @@
     2. Remove Attribute, Group
     3. Remove Entity
 """
+
 import pydantic
 
 from .link import Link

diff --git a/hnhm/postgres/sql.py b/hnhm/postgres/sql.py
@@ -1,8 +1,8 @@
 import jinja2
-import psycopg2
+import psycopg
 
-from ..core.attribute import Type
-from ..core import Sql, HnhmError, ChangeType, LayoutType, task, migration
+from hnhm.core.attribute import Type
+from hnhm.core import Sql, HnhmError, ChangeType, LayoutType, task, migration
 
 PG_TYPES = {
     Type.STRING: "TEXT",
@@ -338,37 +338,26 @@ def generate_sql(self, migration_or_task: migration.Migration | task.Task) -> st
         return generate_sql(migration_or_task, self.jinja)
 
     def execute(self, sql: str):
-        connection = psycopg2.connect(
-            database=self.database,
+        with psycopg.connect(
+            dbname=self.database,
             user=self.user,
             password=self.password,
             port=self.port,
             host=self.host,
-        )
-        cursor = connection.cursor()
-
-        try:
-            cursor.execute(sql)
-            connection.commit()
-        except Exception as e:
-            raise e
-        finally:
-            cursor.close()
-            connection.close()
+        ) as conn:
+            conn.execute(sql)
+            conn.commit()
 
     def execute_many(self, sql: str, values: list):
-        connection = psycopg2.connect(
-            database=self.database,
-            user=self.user,
-            password=self.password,
-            port=self.port,
-            host=self.host,
-        )
-        cursor = connection.cursor()
-
-        try:
-            cursor.executemany(sql, values)
-            connection.commit()
-        finally:
-            cursor.close()
-            connection.close()
+        with (
+            psycopg.connect(
+                dbname=self.database,
+                user=self.user,
+                password=self.password,
+                port=self.port,
+                host=self.host,
+            ) as conn,
+            conn.cursor() as cur,
+        ):
+            cur.executemany(sql, values)
+            conn.commit()
diff --git a/hnhm/postgres/sql_templates/load_link.sql b/hnhm/postgres/sql_templates/load_link.sql
@@ -57,7 +57,7 @@ SELECT
     {% endfor -%}
     d1.valid_from,
     CASE
-        WHEN d2.valid_from IS NULL THEN 'infinity'
+        WHEN d2.valid_from IS NULL THEN '9999-12-31 00:00:00+00'
         ELSE d2.valid_from
     END AS valid_to,
     '{{ source_table }}' AS _source,

diff --git a/hnhm/postgres/sql_templates/load_new.sql b/hnhm/postgres/sql_templates/load_new.sql
@@ -56,7 +56,7 @@ SELECT
     {% endfor -%}
     d1.valid_from,
     CASE
-        WHEN d2.valid_from IS NULL THEN 'infinity'
+        WHEN d2.valid_from IS NULL THEN '9999-12-31 00:00:00+00'
         ELSE d2.valid_from
     END AS valid_to,
     d1._hash AS _hash,

diff --git a/hnhm/postgres/sql_templates/update_entity_view.sql b/hnhm/postgres/sql_templates/update_entity_view.sql
@@ -18,14 +18,14 @@ LEFT JOIN
     {{ attribute.table }}
     ON {{ hub }}.{{ sk }} = {{ attribute.table }}.{{ sk }}
     {% if attribute.change_type == 'NEW' -%}
-        AND {{ attribute.table }}.valid_to = 'infinity'
+        AND {{ attribute.table }}.valid_to = '9999-12-31 00:00:00+00'
     {%- endif %}
 {%- endfor %}
 {% for group in groups -%}
 LEFT JOIN
     {{ group.table }}
     ON {{ hub }}.{{ sk }} = {{ group.table }}.{{ sk }}
     {% if group.change_type == 'NEW' -%}
-        AND {{ group.table }}.valid_to = 'infinity'
+        AND {{ group.table }}.valid_to = '9999-12-31 00:00:00+00'
     {%- endif %}
 {%- endfor %}