Release 2.7.0

.github/workflows/pythonci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: 3.9
-      - uses: Gr1N/setup-poetry@v7
+      - uses: Gr1N/setup-poetry@v8
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pypoetry/virtualenvs

.pre-commit-config.yaml

@@ -56,7 +56,6 @@ repos:
     hooks:
       - id: isort
         name: isort
-        args: [ "--profile", "black", "--skip", "__init__.py", "--filter-files" ]
   - repo: local
     hooks:
       - id: mypy

Makefile

@@ -61,3 +61,7 @@ launch:
 push:
 	docker push $(REGISTRY)/$(IMAGE):$(TAG)_$(DEVICE)$(TAG_EXT)
 	docker push $(REGISTRY)/$(IMAGE)-app:$(TAG)$(TAG_EXT)
+
+.PHONY: docs_serve
+docs_serve:
+	cd docs && mkdocs serve

azimuth/app.py

@@ -9,17 +9,14 @@
 from distributed import SpecCluster
 from fastapi import APIRouter, Depends, FastAPI, HTTPException
 from fastapi.exceptions import RequestValidationError
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
 from starlette.middleware.cors import CORSMiddleware
-from starlette.requests import Request
-from starlette.responses import JSONResponse
 from starlette.status import (
     HTTP_400_BAD_REQUEST,
     HTTP_401_UNAUTHORIZED,
     HTTP_403_FORBIDDEN,
     HTTP_404_NOT_FOUND,
     HTTP_422_UNPROCESSABLE_ENTITY,
-    HTTP_500_INTERNAL_SERVER_ERROR,
     HTTP_503_SERVICE_UNAVAILABLE,
 )
 
@@ -31,6 +28,7 @@
 from azimuth.types import DatasetSplitName, ModuleOptions, SupportedModule
 from azimuth.utils.cluster import default_cluster
 from azimuth.utils.conversion import JSONResponseIgnoreNan
+from azimuth.utils.exception_handlers import handle_validation_error
 from azimuth.utils.logs import set_logger_config
 from azimuth.utils.validation import assert_not_none
 
@@ -51,7 +49,6 @@
     # conventional HTTP codes.
     # This overwrites the default ValidationError response for 422 in the OpenAPI spec.
     HTTP_422_UNPROCESSABLE_ENTITY,
-    HTTP_500_INTERNAL_SERVER_ERROR,
     HTTP_503_SERVICE_UNAVAILABLE,
 )
 
@@ -60,23 +57,6 @@ class HTTPExceptionModel(BaseModel):
     detail: str
 
 
-async def handle_validation_error(request: Request, exception: RequestValidationError):
-    return JSONResponse(
-        status_code=HTTP_404_NOT_FOUND  # for errors in paths, e.g., /dataset_splits/potato
-        if "path" in (error["loc"][0] for error in exception.errors())
-        else HTTP_400_BAD_REQUEST,  # for other errors like in query params, e.g., pipeline_index=-1
-        content={"detail": str(exception)},
-    )
-
-
-async def handle_internal_error(request: Request, exception: Exception):
-    # Don't expose this unexpected internal error as that could expose a security vulnerability.
-    return JSONResponse(
-        status_code=HTTP_500_INTERNAL_SERVER_ERROR,
-        content={"detail": "Internal server error"},
-    )
-
-
 def get_dataset_split_manager_mapping() -> Dict[DatasetSplitName, Optional[DatasetSplitManager]]:
     return _dataset_split_managers
 
@@ -144,8 +124,6 @@ def start_app(config_path: Optional[str], load_config_history: bool, debug: bool
     log.info("🔭 Azimuth starting 🔭")
 
     azimuth_config = load_azimuth_config(config_path, load_config_history)
-    if azimuth_config.dataset is None:
-        raise ValueError("No dataset has been specified in the config.")
 
     local_cluster = default_cluster(large=azimuth_config.large_dask_cluster)
 
@@ -184,8 +162,9 @@ def create_app() -> FastAPI:
         default_response_class=JSONResponseIgnoreNan,
         responses={code: {"model": HTTPExceptionModel} for code in COMMON_HTTP_ERROR_CODES},
         exception_handlers={
+            ValidationError: handle_validation_error,  # for PATCH "/config",
+            # where we call old_config.copy(update=partial_config, deep=True) ourselves.
             RequestValidationError: handle_validation_error,
-            HTTP_500_INTERNAL_SERVER_ERROR: handle_internal_error,
         },
         root_path=".",  # Tells Swagger UI and ReDoc to fetch the OpenAPI spec from ./openapi.json
         # (relative) so it works through the front-end proxy.
@@ -201,16 +180,10 @@ def create_app() -> FastAPI:
     from azimuth.routers.model_performance.confidence_histogram import (
         router as confidence_histogram_router,
     )
-    from azimuth.routers.model_performance.confusion_matrix import (
-        router as confusion_matrix_router,
-    )
+    from azimuth.routers.model_performance.confusion_matrix import router as confusion_matrix_router
     from azimuth.routers.model_performance.metrics import router as metrics_router
-    from azimuth.routers.model_performance.outcome_count import (
-        router as outcome_count_router,
-    )
-    from azimuth.routers.model_performance.utterance_count import (
-        router as utterance_count_router,
-    )
+    from azimuth.routers.model_performance.outcome_count import router as outcome_count_router
+    from azimuth.routers.model_performance.utterance_count import router as utterance_count_router
     from azimuth.routers.top_words import router as top_words_router
     from azimuth.routers.utterances import router as utterances_router
     from azimuth.utils.routers import require_application_ready, require_available_model

azimuth/config.py

@@ -11,12 +11,12 @@
 
 import structlog
 from jsonlines import jsonlines
-from pydantic import BaseSettings, Extra, Field, root_validator, validator
+from pydantic import Extra, Field, ValidationError, root_validator, validator
 
 from azimuth.types import AliasModel, DatasetColumn, SupportedModelContract
 from azimuth.utils.conversion import md5_hash
 from azimuth.utils.exclude_fields_from_cache import exclude_fields_from_cache
-from azimuth.utils.openapi import fix_union_types, make_all_properties_required
+from azimuth.utils.openapi import AzimuthBaseSettings
 
 log = structlog.get_logger(__file__)
 T = TypeVar("T", bound="ProjectConfig")
@@ -87,14 +87,6 @@ class AzimuthValidationError(Exception):
     pass
 
 
-class AzimuthBaseSettings(BaseSettings):
-    class Config:
-        @staticmethod
-        def schema_extra(schema):
-            fix_union_types(schema)
-            make_all_properties_required(schema)
-
-
 class CustomObject(AzimuthBaseSettings):
     class_name: str = Field(
         ...,
@@ -128,8 +120,8 @@ class TemperatureScaling(CustomObject):
     ] = "azimuth.utils.ml.postprocessing.TemperatureScaling"
     temperature: float = Field(1, ge=0, env="TEMP")
 
-    @root_validator()
-    def check_temps(cls, values):
+    @root_validator
+    def _check_temps(cls, values):
         kwargs = values.get("kwargs", {})
         if "temperature" not in kwargs:
             kwargs["temperature"] = values.get("temperature", 1)
@@ -144,8 +136,8 @@ class ThresholdConfig(CustomObject):
     ] = "azimuth.utils.ml.postprocessing.Thresholding"
     threshold: float = Field(0.5, ge=0, le=1, env="TH")
 
-    @root_validator()
-    def check_threshold(cls, values):
+    @root_validator
+    def _check_threshold(cls, values):
         kwargs = values.get("kwargs", {})
         if "threshold" not in kwargs:
             kwargs["threshold"] = values.get("threshold", 0.5)
@@ -267,7 +259,7 @@ class ProjectConfig(AzimuthBaseSettings):
     # Name of the current project.
     name: str = Field("New project", exclude_from_cache=True)
     # Dataset object definition.
-    dataset: Optional[CustomObject] = None
+    dataset: Optional[CustomObject] = Field(None, nullable=True)
     # Column names config in dataset
     columns: ColumnConfiguration = ColumnConfiguration()
     # Name of the rejection class.
@@ -286,7 +278,6 @@ def get_project_hash(self):
             self.dict(
                 include=ProjectConfig.__fields__.keys(),
                 exclude=exclude_fields_from_cache(self),
-                by_alias=True,
             )
         )
 
@@ -341,7 +332,7 @@ class ModelContractConfig(CommonFieldsConfig):
     saliency_layer: Optional[str] = Field(None, nullable=True)
 
     @validator("pipelines", pre=True)
-    def check_pipeline_names(cls, pipeline_definitions):
+    def _check_pipeline_names(cls, pipeline_definitions):
         # We support both [] and None (null in JSON), and we standardize it to None.
         if not pipeline_definitions:
             return None
@@ -364,7 +355,6 @@ def get_model_contract_hash(self):
             self.dict(
                 include=ModelContractConfig.__fields__.keys()
                 - CommonFieldsConfig.__fields__.keys(),
-                by_alias=True,
             )
         )
 
@@ -441,8 +431,8 @@ class AzimuthConfig(
     # Reminder: If a module depends on an attribute in AzimuthConfig, the module will be forced to
     # include all other configs in its scope.
 
-    @root_validator()
-    def dynamic_language_config_values(cls, values):
+    @root_validator
+    def _dynamic_language_config_values(cls, values):
         defaults = config_defaults_per_language[values["language"]]
         if behavioral_testing := values.get("behavioral_testing"):
             neutral_token = behavioral_testing.neutral_token
@@ -482,6 +472,19 @@ def load_last_from_config_history(cls, config_history_path: str) -> Optional["Az
         else:
             return AzimuthConfigHistory.parse_obj(last_config).config
 
+    def get_config_history(self) -> List["AzimuthConfigHistoryWithHash"]:
+        config_history = []
+        try:
+            with jsonlines.open(self.get_config_history_path(), mode="r") as reader:
+                for item in reader:
+                    try:
+                        config_history.append(AzimuthConfigHistoryWithHash.parse_obj(item))
+                    except ValidationError:
+                        pass
+        except (FileNotFoundError, ValueError):
+            pass
+        return config_history
+
     def log_info(self):
         log.info(f"Config loaded for {self.name} with {self.model_contract} as a model contract.")
 
@@ -527,6 +530,14 @@ class AzimuthConfigHistory(AzimuthBaseSettings):
     created_on: str = Field(default_factory=lambda: str(datetime.now(timezone.utc)))
 
 
+class AzimuthConfigHistoryWithHash(AzimuthConfigHistory):
+    hash: str = ""
+
+    @root_validator(skip_on_failure=True)
+    def _set_hash(cls, values):
+        return {**values, "hash": md5_hash(values["config"].dict())}
+
+
 def load_azimuth_config(config_path: Optional[str], load_config_history: bool) -> AzimuthConfig:
     log.info("-------------Loading Config--------------")
     cfg = AzimuthConfig.load(config_path, load_config_history)

azimuth/modules/base_classes/caching.py

@@ -108,7 +108,7 @@ def _store_data_in_cache(self, result: List[ModuleResponse], indices: List[int])
 
         # Save all that affects caching so it can be used for identification and debugging.
         with open(os.path.join(self._cache_effective_arguments), "w") as f:
-            json.dump(self.get_effective_arguments().no_alias_dict(), f, indent=2)
+            json.dump(self.get_effective_arguments().dict(), f, indent=2)
 
     @retry(stop_max_attempt_number=5, wait_fixed=0.5)
     def _check_cache_internal(self, indices: Optional[List[int]]):

azimuth/modules/base_classes/dask_module.py

@@ -99,7 +99,7 @@ def _get_config_scope(self, config) -> ConfigScope:
             scoped_config = config.__class__
         else:
             scoped_config = base.__args__[0]
-        return cast(ConfigScope, scoped_config.parse_obj(config.dict(by_alias=True)))
+        return cast(ConfigScope, scoped_config.parse_obj(config.dict()))
 
     def start_task_on_dataset_split(
         self, client: Client, dependencies: List["DaskModule"] = None

azimuth/modules/base_classes/module.py

@@ -33,7 +33,7 @@ def __init__(
     ):
         mod_options = mod_options or ModuleOptions()
         self.mod_options = mod_options
-        defined_mod_options = set(self.mod_options.no_alias_dict(exclude_defaults=True).keys())
+        defined_mod_options = set(self.mod_options.dict(exclude_defaults=True).keys())
         if not self.required_mod_options.issubset(defined_mod_options):
             raise ValueError(f"{self.__class__.__name__} requires {self.required_mod_options}.")
         if diff := (defined_mod_options - self.required_mod_options - self.optional_mod_options):

azimuth/modules/model_performance/confidence_binning.py

@@ -11,15 +11,9 @@
 from azimuth.dataset_split_manager import DatasetSplitManager
 from azimuth.modules.base_classes import DatasetResultModule, FilterableModule
 from azimuth.types import DatasetColumn
-from azimuth.types.model_performance import (
-    ConfidenceBinDetails,
-    ConfidenceHistogramResponse,
-)
+from azimuth.types.model_performance import ConfidenceBinDetails, ConfidenceHistogramResponse
 from azimuth.types.outcomes import ALL_OUTCOMES, OutcomeName
-from azimuth.utils.dataset_operations import (
-    get_confidences_from_ds,
-    get_outcomes_from_ds,
-)
+from azimuth.utils.dataset_operations import get_confidences_from_ds, get_outcomes_from_ds
 from azimuth.utils.validation import assert_not_none
 
 CONFIDENCE_BINS_COUNT = 20

azimuth/modules/model_performance/metrics.py

@@ -14,9 +14,7 @@
 
 from azimuth.config import MetricsConfig, MetricsPerFilterConfig
 from azimuth.modules.base_classes import AggregationModule, FilterableModule
-from azimuth.modules.model_performance.confidence_binning import (
-    ConfidenceHistogramModule,
-)
+from azimuth.modules.model_performance.confidence_binning import ConfidenceHistogramModule
 from azimuth.plots.ece import make_ece_figure
 from azimuth.types import DatasetColumn, DatasetFilters
 from azimuth.types.model_performance import (

azimuth/modules/model_performance/outcome_count.py

@@ -21,11 +21,7 @@
     OutcomeCountPerThresholdValue,
 )
 from azimuth.types.outcomes import ALL_OUTCOMES, OutcomeName
-from azimuth.types.tag import (
-    ALL_DATA_ACTION_FILTERS,
-    SMART_TAGS_FAMILY_MAPPING,
-    SmartTag,
-)
+from azimuth.types.tag import ALL_DATA_ACTION_FILTERS, SMART_TAGS_FAMILY_MAPPING, SmartTag
 from azimuth.utils.dataset_operations import get_outcomes_from_ds
 from azimuth.utils.ml.model_performance import (
     sorted_by_utterance_count,

azimuth/modules/perturbation_testing/perturbation_testing.py

@@ -12,12 +12,7 @@
 from azimuth.modules.base_classes import DatasetResultModule
 from azimuth.modules.base_classes.dask_module import Worker
 from azimuth.modules.model_contract_task_mapping import model_contract_task_mapping
-from azimuth.types import (
-    DatasetColumn,
-    DatasetSplitName,
-    ModuleOptions,
-    SupportedMethod,
-)
+from azimuth.types import DatasetColumn, DatasetSplitName, ModuleOptions, SupportedMethod
 from azimuth.types.perturbation_testing import (
     PRETTY_PERTURBATION_TYPES,
     PerturbationTestClass,

azimuth/routers/app.py

@@ -24,6 +24,7 @@
     DatasetInfoResponse,
     PerturbationTestingSummary,
     StatusResponse,
+    UtteranceCountPerDatasetSplit,
 )
 from azimuth.types.perturbation_testing import (
     PerturbationTestingMergedResponse,
@@ -44,7 +45,6 @@
     require_available_model,
     require_pipeline_index,
 )
-from azimuth.utils.validation import assert_not_none
 
 router = APIRouter()
 
@@ -88,26 +88,21 @@ def get_dataset_info(
 ):
     eval_dm = dataset_split_managers.get(DatasetSplitName.eval)
     training_dm = dataset_split_managers.get(DatasetSplitName.train)
-    dm = assert_not_none(eval_dm or training_dm)
 
     return DatasetInfoResponse(
         project_name=config.name,
-        class_names=dm.get_class_names(),
         data_actions=ALL_DATA_ACTION_FILTERS,
         smart_tags=ALL_SMART_TAG_FILTERS,
-        eval_class_distribution=eval_dm.class_distribution().tolist()
-        if eval_dm is not None
-        else [],
-        train_class_distribution=training_dm.class_distribution().tolist()
-        if training_dm is not None
-        else [],
         startup_tasks={k: v.status() for k, v in startup_tasks.items()},
         model_contract=config.model_contract,
         prediction_available=predictions_available(config),
         perturbation_testing_available=perturbation_testing_available(config),
         available_dataset_splits=AvailableDatasetSplits(
             eval=eval_dm is not None, train=training_dm is not None
         ),
+        utterance_count_per_dataset_split=UtteranceCountPerDatasetSplit(
+            eval=eval_dm and eval_dm.num_rows, train=training_dm and training_dm.num_rows
+        ),
         similarity_available=similarity_available(config),
         postprocessing_editable=None
         if config.pipelines is None