Merge pull request #157 from crowdcent/feature/full-diagnostics

Options to get all Signals diagnostics

docs/evaluation.md

@@ -62,9 +62,9 @@ metrics = evaluator.full_evaluation(val_df,
 
 ## Numerai Signals specific metrics
 
-`NumeraiSignalsEvaluator` offers neutralized correlation scores. This is a special operation as it calls on Numerai servers and needs additional authentication, so it is not included in `full_evaluation`. It can still be beneficial to calculate as this metric is close to the one used for payouts.
+`NumeraiSignalsEvaluator` offers [Numerai Signals diagnostics](https://forum.numer.ai/t/signals-diagnostics-guide/5950) scores. This is a special operation as it calls on Numerai servers and needs additional authentication, so it is not included in `full_evaluation`.
 
-Example of how to get neutralized correlation scores for Numerai Signals:
+Example of how to get diagnostic scores for Numerai Signals:
 ```py
 from numerblox.misc import Key
 from numerblox.evaluation import NumeraiSignalsEvaluator
@@ -78,9 +78,8 @@ key = Key(pub_id="Hello", secret_key="World")
 # DataFrame with validation data containing prediction, friday_date, ticker and data_type columns
 val_df = pd.DataFrame()
 
-evaluator.get_neutralized_corr(val, model_name=model_name, key=key)
-# Returns a Pandas Series like this.
-# pd.Series([0.01, ..., 0.02])
+evaluator.get_neutralized_corr(val, model_name=model_name, key=key, corr_col="validationRic")
+# Returns a Pandas DataFrame with validationRic.
 ```
 
 ## Custom functions

numerblox/evaluation.py

@@ -4,7 +4,7 @@
 from scipy import stats
 from tqdm.auto import tqdm
 import matplotlib.pyplot as plt
-from typing import Tuple, List, Callable, Dict, Any
+from typing import Tuple, List, Callable, Dict, Any, Union
 from numerapi import SignalsAPI
 from joblib import Parallel, delayed
 from numerai_tools.scoring import correlation_contribution
@@ -1011,6 +1011,9 @@ def full_evaluation(
 
 class NumeraiSignalsEvaluator(BaseEvaluator):
     """Evaluator for all metrics that are relevant in Numerai Signals."""
+    # Columns retrievable from Numerai Signals diagnostics.
+    # More info: https://forum.numer.ai/t/signals-diagnostics-guide/5950
+    VALID_DIAGNOSTICS_COLS = ["validationCorrV4", "validationFncV4", "validationIcV2", "validationRic"]
 
     def __init__(
         self,
@@ -1028,10 +1031,10 @@ def __init__(
             show_detailed_progress_bar=show_detailed_progress_bar
         )
 
-    def get_neutralized_corr(
+    def get_diagnostics(
         self, val_dataf: pd.DataFrame, model_name: str, key: Key, timeout_min: int = 2,
-        corr_col: str = "validationFncV4"
-    ) -> pd.Series:
+        col: Union[str, None] = "validationFncV4"
+    ) -> pd.DataFrame:
         """
         Retrieved neutralized validation correlation by era. \n
         Calculated on Numerai servers. \n
@@ -1040,11 +1043,11 @@ def get_neutralized_corr(
         :param model_name: Any model name for which you have authentication credentials. \n
         :param key: Key object to authenticate upload of diagnostics. \n
         :param timeout_min: How many minutes to wait on diagnostics Computing on Numerai servers before timing out. \n
+        :param col: Which column to return. Should be one of ['validationCorrV4', 'validationFncV4', 'validationIcV2', 'validationRic']. If None, all columns will be returned. \n
         2 minutes by default. \n
         :return: Pandas Series with era as index and neutralized validation correlations (validationCorr).
         """
-        VALID_CORR_COLS = ["validationCorrV4", "validationFncV4", "validationIcV2", "validationRic"]
-        assert corr_col in VALID_CORR_COLS, f"corr_col should be one of {VALID_CORR_COLS}. Got: '{corr_col}'"
+        assert col in self.VALID_DIAGNOSTICS_COLS or col is None, f"corr_col should be one of {self.VALID_DIAGNOSTICS_COLS} or None. Got: '{col}'"
         api = SignalsAPI(public_id=key.pub_id, secret_key=key.secret_key)
         model_id = api.get_models()[model_name]
         diagnostics_id = api.upload_diagnostics(df=val_dataf, model_id=model_id)
@@ -1054,9 +1057,10 @@ def get_neutralized_corr(
             diagnostics_id=diagnostics_id,
             timeout_min=timeout_min,
         )
-        dataf = pd.DataFrame(data["perEraDiagnostics"]).set_index("era")[corr_col]
-        dataf.index = pd.to_datetime(dataf.index)
-        return dataf
+        diagnostics_df = pd.DataFrame(data["perEraDiagnostics"]).set_index("era")
+        diagnostics_df.index = pd.to_datetime(diagnostics_df.index)
+        return_cols = [col] if col is not None else self.VALID_DIAGNOSTICS_COLS
+        return diagnostics_df[return_cols]
 
     @staticmethod
     def __await_diagnostics(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "numerblox"
-version = "1.1.12"
+version = "1.1.13"
 description = "Solid Numerai Pipelines"
 authors = ["CrowdCent <support@crowdcent.com>"]
 license = "MIT License"

tests/test_evaluation.py

@@ -257,30 +257,45 @@ def mock_api():
         mock_instance.upload_diagnostics.return_value = "test_diag_id"
 
         # Mock diagnostics method
-        mock_instance.diagnostics.return_value = [{"status": "done", "perEraDiagnostics": [{"era": "2023-09-01", "validationFncV4": 0.6, "validationCorrV4": 0.6}]}]
+        mock_instance.diagnostics.return_value = [{"status": "done", "perEraDiagnostics": [{"era": "2023-09-01", "validationFncV4": 0.6, "validationCorrV4": 0.6, "validationIcV2": 0.3, "validationRic": 0.5}]}]
         yield mock_instance
 
 
-def test_get_neutralized_corr(create_signals_sample_data, mock_api):
+def test_get_diagnostics(create_signals_sample_data, mock_api):
     df = create_signals_sample_data
     obj = NumeraiSignalsEvaluator(era_col="date")  
-    result = obj.get_neutralized_corr(df, "test_model", Key("Hello", "World"))
+    result = obj.get_diagnostics(df, "test_model", Key("Hello", "World"), col="validationIcV2")
 
     # Asserting if the output is correct
-    assert isinstance(result, pd.Series)
-    assert result["2023-09-01"] == 0.6
+    assert isinstance(result, pd.DataFrame)
+    assert result.loc["2023-09-01"]["validationIcV2"] == 0.3
 
     # Asserting if the right methods were called
     mock_api.get_models.assert_called_once()
     mock_api.upload_diagnostics.assert_called_once_with(df=df, model_id='test_model_id')
     mock_api.diagnostics.assert_called()
 
 
-def test_get_neutralized_corr_invalid_corr_col(create_signals_sample_data):
+def test_get_diagnostics_all_cols(create_signals_sample_data, mock_api):
+    df = create_signals_sample_data
+    obj = NumeraiSignalsEvaluator(era_col="date")  
+    result = obj.get_diagnostics(df, "test_model", Key("Hello", "World"), col=None)
+
+    # Asserting if the output is correct
+    assert isinstance(result, pd.DataFrame)
+    for col in obj.VALID_DIAGNOSTICS_COLS:
+        assert col in result.columns
+    assert result.loc["2023-09-01"]["validationCorrV4"] == 0.6
+    assert result.loc["2023-09-01"]["validationIcV2"] == 0.3
+    assert result.loc["2023-09-01"]["validationRic"] == 0.5
+    assert result.loc["2023-09-01"]["validationFncV4"] == 0.6
+
+
+def test_get_diagnostics_invalid_col(create_signals_sample_data):
     df = create_signals_sample_data
     obj = NumeraiSignalsEvaluator(era_col="date")  
     with pytest.raises(AssertionError):
-        obj.get_neutralized_corr(df, "test_model", Key("Hello", "World"), corr_col="invalid_corr_col")
+        obj.get_diagnostics(df, "test_model", Key("Hello", "World"), col="invalid_corr_col")
 
 
 def test_await_diagnostics_timeout(mock_api):