Adiciona shed como ferramenta de lint (#83)

* Adiciona shed

* Adiciona pre-commit

* Formata código

.github/workflows/pull-request.yaml

@@ -16,10 +16,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ['3.9']
-        experimental: [ false ]
+        python: ["3.9"]
+        experimental: [false]
         include:
-          - python: '3.10'
+          - python: "3.10"
             experimental: true
     steps:
       - name: 🚧 Install OS dependencies
@@ -50,6 +50,24 @@ jobs:
         env:
           POETRY_VIRTUALENVS_CREATE: false
 
+      - uses: dorny/paths-filter@v2
+        name: Get modified files
+        id: file_changes
+        with:
+          list-files: shell
+          filters: |
+            python:
+              - added|modified: '**/*.py'
+              - added|modified: '**/*.md'
+              - added|modified: '**/*.rst'
+
+      - name: 🧪 pre-commit
+        if: ${{ steps.file_changes.outputs.python == 'true' }}
+        run: pre-commit run --files ${{ steps.file_changes.outputs.python_files }}
+
+      - name: 🧪 Mypy - Static type check
+        run: mypy --xml-report .mypy_coverage --html-report .mypy_coverage --ignore-missing-imports --install-types --non-interactive --pretty --no-color-output amora
+
       - name: 🚧 GCP Auth
         uses: "google-github-actions/auth@v0"
         with:
@@ -58,9 +76,6 @@ jobs:
       - name: 🚧 Set up GCP SDK
         uses: "google-github-actions/setup-gcloud@v0"
 
-      - name: 🎨 Black - Python format check
-        run: black -v --check amora tests
-
       - name: 🧪 pytest
         run: py.test --cov=amora --cov-report=term-missing --cov-report=xml ./tests
 
@@ -71,9 +86,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
-      - name: 🧪 Mypy - Static type check
-        run: mypy --xml-report .mypy_coverage --html-report .mypy_coverage --ignore-missing-imports --install-types --non-interactive --pretty --no-color-output amora
-
       - name: 📤 Uploading coverage reports as an artifact
         uses: actions/upload-artifact@v2
         with:

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+minimum_pre_commit_version: "2.9.0"
+repos:
+  - repo: https://github.com/Zac-HD/shed
+    rev: 0.9.5
+    hooks:
+      - id: shed
+        types_or: [python, markdown, rst]

amora/cli.py

@@ -1,33 +1,29 @@
 import json
+from dataclasses import dataclass
 from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
 
 import pytest
 import typer
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Optional, List
 from jinja2 import Environment, PackageLoader, select_autoescape
 from rich.console import Console
 from rich.table import Table
 from rich.text import Text
 
-from amora.config import settings
-from amora.models import (
-    AmoraModel,
-    list_models,
-    Model,
-)
-from amora.utils import list_target_files
-from amora.compilation import compile_statement
 from amora import materialization
+from amora.compilation import compile_statement
+from amora.config import settings
+from amora.models import Model, list_models
 from amora.providers.bigquery import (
-    dry_run,
-    get_schema,
     BIGQUERY_TYPES_TO_PYTHON_TYPES,
     DryRunResult,
+    dry_run,
     estimated_query_cost_in_usd,
     estimated_storage_cost_in_usd,
+    get_schema,
 )
+from amora.utils import list_target_files
 
 app = typer.Typer(
     help="Amora Data Build Tool enables engineers to transform data in their warehouses "
@@ -187,7 +183,7 @@ def has_source(self):
         @property
         def depends_on(self) -> List[str]:
             return sorted(
-                [dependency.__name__ for dependency in self.model.dependencies()]
+                dependency.__name__ for dependency in self.model.dependencies()
             )
 
         @property
@@ -305,7 +301,7 @@ def models_import(
     template = env.get_template("new-model.py.jinja2")
 
     project, dataset, table = table_reference.split(".")
-    model_name = "".join((part.title() for part in table.split("_")))
+    model_name = "".join(part.title() for part in table.split("_"))
 
     destination_file_path = Path(settings.MODELS_PATH).joinpath(
         (model_file_path or model_name.replace(".", "/")) + ".py"
@@ -403,9 +399,10 @@ def feature_store_apply():
     on the provider configuration. For example, setting local as
     your provider will result in a sqlite online store being created.
     """
-    from amora.feature_store.registry import get_repo_contents
     from feast.repo_operations import apply_total_with_repo_instance
+
     from amora.feature_store import fs
+    from amora.feature_store.registry import get_repo_contents
 
     apply_total_with_repo_instance(
         store=fs,
@@ -461,10 +458,11 @@ def feature_store_serve():
         POST /get-online-features
         GET /list-feature-views
     """
+    import uvicorn
     from feast.feature_server import get_app
+
     from amora.feature_store import fs
     from amora.feature_store.config import settings
-    import uvicorn
 
     app = get_app(store=fs)
 

amora/config.py

@@ -1,9 +1,9 @@
 import logging
 import os
-from uuid import uuid4
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Tuple
+from uuid import uuid4
 
 from pydantic import BaseSettings
 

amora/contracts.py

@@ -1,6 +1,6 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from datetime import datetime
-from typing import Optional, List
+from typing import List, Optional
 
 
 @dataclass

amora/feature_store/__init__.py

@@ -1,4 +1,4 @@
-from feast import RepoConfig, FeatureStore
+from feast import FeatureStore, RepoConfig
 
 from amora.feature_store.config import settings
 

amora/feature_store/decorators.py

@@ -1,5 +1,6 @@
 from feast import FeatureService
-from amora.feature_store.feature_view import feature_view_for_model, name_for_model
+
+from amora.feature_store.feature_view import feature_view_for_model
 from amora.feature_store.registry import FEATURE_REGISTRY
 from amora.models import Model
 

amora/feature_store/feature_view.py

@@ -1,14 +1,11 @@
-from typing import Union, NewType
-
-from feast import FeatureView, Feature, BigQuerySource
-from sqlalchemy.orm import DeclarativeMeta
+from feast import BigQuerySource, Feature, FeatureView
+from google.protobuf.duration_pb2 import Duration
 
 from amora.feature_store import settings
 from amora.feature_store.protocols import FeatureViewSourceProtocol
 from amora.feature_store.type_mapping import SQLALCHEMY_TYPES_TO_FS_TYPES
 from amora.models import Model
 from amora.providers.bigquery import get_fully_qualified_id
-from google.protobuf.duration_pb2 import Duration
 
 
 def name_for_model(model: Model) -> str:

amora/feature_store/protocols.py

@@ -1,4 +1,4 @@
-from typing import Protocol, runtime_checkable, List
+from typing import List, Protocol, runtime_checkable
 
 from amora.models import Column
 
@@ -61,10 +61,7 @@ def feature_view_features(cls) -> List[Column]:
         ```python
         @classmethod
         def feature_view_features(cls) -> List[Column]:
-            return [
-                cls.count_transactions_last_30d,
-                cls.sum_transactions_last_30d
-            ]
+            return [cls.count_transactions_last_30d, cls.sum_transactions_last_30d]
         ```
         """
         ...
@@ -83,7 +80,8 @@ def feature_view_event_timestamp(cls) -> Column:
         At your Amora Model, the column should be defined as such:
 
         ```python
-        from sqlalchemy import Column, TIMESTAMP
+        from sqlalchemy import TIMESTAMP, Column
+
         from amora.models import AmoraModel, Field
 
 

amora/feature_store/registry.py

@@ -1,5 +1,6 @@
-from typing import Dict, Iterable, Tuple, List
-from feast import FeatureView, Entity, FeatureService
+from typing import Dict, Iterable, List, Tuple
+
+from feast import Entity, FeatureService, FeatureView
 from feast.repo_contents import RepoContents
 from sqlalchemy.orm import InstrumentedAttribute
 

amora/feature_store/type_mapping.py

@@ -2,7 +2,6 @@
 from feast import ValueType
 from sqlalchemy.sql import sqltypes
 
-
 SQLALCHEMY_TYPES_TO_FS_TYPES = {
     sqltypes.Float: ValueType.FLOAT,
     sqltypes.String: ValueType.STRING,

amora/materialization.py

@@ -1,14 +1,13 @@
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import Iterable, Optional
 
-from google.cloud.bigquery import Table, Client, QueryJobConfig
+import matplotlib.pyplot as plt
+import networkx as nx
+from google.cloud.bigquery import Client, QueryJobConfig, Table
 
-from amora.models import (
-    MaterializationTypes,
-    amora_model_for_target_path,
-    Model,
-)
+from amora.config import settings
+from amora.models import MaterializationTypes, Model, amora_model_for_target_path
 
 
 @dataclass
@@ -29,6 +28,37 @@ def __repr__(self):
         return f"{self.model.__name__} -> {self.sql_stmt}"
 
 
+class DependencyDAG(nx.DiGraph):
+    def __iter__(self):
+        # todo: validar se podemos substituir por graphlib
+        return nx.topological_sort(self)
+
+    @classmethod
+    def from_tasks(cls, tasks: Iterable[Task]) -> "DependencyDAG":
+        dag = cls()
+
+        for task in tasks:
+            dag.add_node(task.model.unique_name)
+            for dependency in getattr(task.model, "__depends_on__", []):
+                dag.add_edge(dependency.unique_name, task.model.unique_name)
+
+        return dag
+
+    def draw(self) -> None:
+        plt.figure(1, figsize=settings.CLI_MATERIALIZATION_DAG_FIGURE_SIZE)
+        nx.draw(
+            self,
+            with_labels=True,
+            font_weight="bold",
+            font_size="12",
+            linewidths=4,
+            node_size=150,
+            node_color="white",
+            font_color="green",
+        )
+        plt.show()
+
+
 def materialize(sql: str, model: Model) -> Optional[Table]:
     config = model.__model_config__
     materialization = config.materialized

amora/models.py

@@ -2,20 +2,21 @@
 import re
 from dataclasses import dataclass, field
 from enum import Enum, auto
-from importlib.util import spec_from_file_location, module_from_spec
+from importlib.util import module_from_spec, spec_from_file_location
 from inspect import getfile
 from pathlib import Path
-from typing import Iterable, List, Optional, Union, Dict, Any, Tuple, Type
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
 
+from sqlalchemy import Column, MetaData, Table, select
+from sqlalchemy.orm import declared_attr
+from sqlalchemy.sql import ColumnElement
+from sqlmodel import Field, Session, SQLModel, create_engine
+
+from amora.config import settings
 from amora.logger import logger
 from amora.protocols import CompilableProtocol
-from amora.config import settings
-from sqlalchemy import MetaData, Table, select, Column
-from sqlmodel import SQLModel, Field, create_engine, Session
-from sqlalchemy.sql import ColumnElement
-from sqlalchemy.orm import declared_attr
 from amora.types import Compilable
-from amora.utils import model_path_for_target_path, list_files
+from amora.utils import list_files, model_path_for_target_path
 
 select = select
 Column = Column
@@ -92,9 +93,11 @@ def __tablename__(cls) -> str:  # type: ignore
         By default, `__tablename__` is the `snake_case` class name.
 
         ```python
-        class MyModel(AmoraModel): ...
+        class MyModel(AmoraModel):
+            ...
+
 
-        assert MyModel.__tablename__ == 'my_model
+        assert MyModel.__tablename__ == "my_model"
         ```
         """
         return re.sub(r"(?<!^)(?=[A-Z])", "_", cls.__name__).lower()

amora/protocols.py

@@ -1,4 +1,4 @@
-from typing import runtime_checkable, Protocol
+from typing import Protocol, runtime_checkable
 
 from amora.types import Compilable
 

amora/providers/bigquery.py

@@ -1,7 +1,7 @@
-from dataclasses import dataclass, field
-from datetime import datetime, date, time
+from dataclasses import dataclass
+from datetime import date, datetime, time
 from enum import Enum
-from typing import Optional, List, Union, Iterable, Any, Dict
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 import sqlalchemy
 from google.api_core.client_info import ClientInfo
@@ -14,17 +14,17 @@
 )
 from google.cloud.bigquery.table import RowIterator, _EmptyRowIterator
 from sqlalchemy import (
-    literal,
-    literal_column,
+    JSON,
+    TIMESTAMP,
+    Boolean,
+    Date,
+    DateTime,
+    Float,
     Integer,
     String,
-    DateTime,
-    Date,
     Time,
-    Float,
-    Boolean,
-    JSON,
-    TIMESTAMP,
+    literal,
+    literal_column,
 )
 from sqlalchemy.sql.selectable import CTE
 from sqlalchemy_bigquery.base import unnest