aeon 1.0.0 and results loaders

.pre-commit-config.yaml

@@ -29,13 +29,13 @@ repos:
         args: [ "--create", "--python-folders", "tsml_eval" ]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.9
+    rev: v0.8.1
     hooks:
       - id: ruff
         args: [ "--fix"]
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.18.0
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
         args: [ "--py39-plus" ]
@@ -53,7 +53,7 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies: [ flake8-bugbear, flake8-print, Flake8-pyproject ]
-        args: [ "--max-line-length=88" ]
+        args: [ "--max-line-length=88", "--extend-ignore=E203" ]
 
   - repo: https://github.com/psf/black
     rev: 24.10.0
@@ -63,7 +63,7 @@ repos:
         exclude: "tsml_eval/_wip/"
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.7
+    rev: 1.9.1
     hooks:
       - id: nbqa-isort
         additional_dependencies: [ isort ]

_tsml_research_resources/soton/iridis/iridis_python.md

@@ -1,6 +1,6 @@
-# Iridis Python
+# Iridis 5 Python
 
-Installation guide for Python packages on Iridis and useful slurm commands.
+Installation guide for Python packages on Iridis 5 and useful slurm commands.
 
 The [Iridis wiki](https://sotonac.sharepoint.com/teams/HPCCommunityWiki) provides a lot of useful information and getting started guides for using Iridis.
 
@@ -158,6 +158,10 @@ For CPU experiments start with one of the following scripts:
 
 >clustering_experiments.sh
 
+These scrips can be run from the command line with the following command:
+
+>sh classification_experiments.sh
+
 You may need to use `dos2unix` to convert the line endings to unix format.
 
 The default queue for CPU jobs is _batch_. Be sure to swap the _queue_alias_ to _serial_ in the script if you want to use this, as the number of jobs submitted won't be tracked properly otherwise.

docs/api.md

@@ -42,16 +42,11 @@ Functions for evaluating the performance of a model.
     evaluation.evaluate_regressors
     evaluation.evaluate_regressors_from_file
     evaluation.evaluate_regressors_by_problem
-    evaluation.evaluate_forecasters
-    evaluation.evaluate_forecasters_from_file
-    evaluation.evaluate_forecasters_by_problem
     evaluation.storage.ClassifierResults
     evaluation.storage.ClustererResults
-    evaluation.storage.ForecasterResults
     evaluation.storage.RegressorResults
     evaluation.storage.load_classifier_results
     evaluation.storage.load_clusterer_results
-    evaluation.storage.load_forecaster_results
     evaluation.storage.load_regressor_results
 ```
 
@@ -69,13 +64,10 @@ Functions for running experiments.
     experiments.load_and_run_classification_experiment
     experiments.run_clustering_experiment
     experiments.load_and_run_clustering_experiment
-    experiments.run_forecasting_experiment
-    experiments.load_and_run_forecasting_experiment
     experiments.run_regression_experiment
     experiments.load_and_run_regression_experiment
     experiments.get_classifier_by_name
     experiments.get_clusterer_by_name
-    experiments.get_forecaster_by_name
     experiments.get_regressor_by_name
     experiments.classification_cross_validation
     experiments.classification_cross_validation_folds
@@ -115,12 +107,16 @@ Public utility functions used elsewhere in the package.
     utils.resampling.resample_data_indices
     utils.resampling.stratified_resample_data
     utils.resampling.stratified_resample_data_indices
+    utils.results_loading.load_estimator_results
+    utils.results_loading.estimator_results_to_dict
+    utils.results_loading.load_estimator_results_to_dict
+    utils.results_loading.estimator_results_to_array
+    utils.results_loading.load_estimator_results_to_array
     utils.results_repair.fix_broken_second_line
     utils.results_validation.validate_results_file
     utils.results_validation.compare_result_file_resample
     utils.results_writing.write_classification_results
     utils.results_writing.write_regression_results
     utils.results_writing.write_clustering_results
-    utils.results_writing.write_forecasting_results
     utils.results_writing.write_results_to_tsml_format
 ```

docs/user_guide.md

@@ -144,29 +144,6 @@ TSCL
 
 :::
 
-:::{grid-item-card}
-:img-top: examples/images/forecasting.png
-:class-img-top: tsml-card-image
-:text-align: center
-
-Time Series Forecasting
-
-^^^
-
-Run basic forecasting experiments using `tsml-eval`.
-
-+++
-
-```{button-ref} /examples/forecasting_experiments.ipynb
-:color: primary
-:click-parent:
-:expand:
-
-Forecasting
-```
-
-:::
-
 ::::
 
 ```{toctree}
@@ -178,5 +155,4 @@ Forecasting
 ./examples/classification_experiments.ipynb
 ./examples/regression_experiments.ipynb
 ./examples/clustering_experiments.ipynb
-./examples/forecasting_experiments.ipynb
 ```

examples/classification_experiments.ipynb

@@ -18,7 +18,7 @@
    "source": [
     "import numpy as np\n",
     "import pandas as pd\n",
-    "from aeon.benchmarking import get_estimator_results\n",
+    "from aeon.benchmarking.results_loaders import get_estimator_results\n",
     "from aeon.classification import DummyClassifier\n",
     "from aeon.datasets import load_classification\n",
     "from aeon.visualisation import plot_critical_difference\n",

examples/clustering_experiments.ipynb

@@ -25,9 +25,9 @@
    "source": [
     "import numpy as np\n",
     "import pandas as pd\n",
-    "from aeon.benchmarking import get_estimator_results\n",
+    "from aeon.benchmarking.metrics.clustering import clustering_accuracy_score\n",
+    "from aeon.benchmarking.results_loaders import get_estimator_results\n",
     "from aeon.datasets import load_classification\n",
-    "from aeon.performance_metrics.clustering import clustering_accuracy_score\n",
     "from aeon.visualisation import plot_critical_difference\n",
     "from sklearn import preprocessing\n",
     "from tsml.datasets import load_minimal_chinatown\n",

examples/forecasting_experiments.ipynb

@@ -1,124 +1,89 @@
 {
  "cells": [
   {
+   "metadata": {},
    "cell_type": "markdown",
    "source": [
     "# Time Series Forecasting Experiments\n",
     "\n",
     "In this notebook we will show how to run a simple time series forecasting experiment using the `tsml-eval` package. Time series forecasting is the task of predicting future values of a time series.\n",
     "\n",
     "<img src=\"./images/forecasting.png\" width=\"600\" alt=\"time series classification\" style=\"margin:20px 20px\">"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+   ]
   },
   {
+   "metadata": {},
    "cell_type": "code",
+   "outputs": [],
    "execution_count": null,
    "source": [
-    "import numpy as np\n",
-    "from aeon.datasets import load_airline\n",
-    "from aeon.forecasting.model_selection import temporal_train_test_split\n",
-    "\n",
-    "from tsml_eval.evaluation.storage import load_forecaster_results\n",
-    "from tsml_eval.experiments import (\n",
-    "    experiments,\n",
-    "    get_forecaster_by_name,\n",
-    "    run_forecasting_experiment,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false
-   },
-   "outputs": []
+    "# import numpy as np\n",
+    "# from aeon.datasets import load_airline\n",
+    "# from aeon.forecasting.model_selection import temporal_train_test_split\n",
+    "#\n",
+    "# from tsml_eval.evaluation.storage import load_forecaster_results\n",
+    "# from tsml_eval.experiments import (\n",
+    "#     experiments,\n",
+    "#     get_forecaster_by_name,\n",
+    "#     run_forecasting_experiment,\n",
+    "# )"
+   ]
   },
   {
+   "metadata": {},
    "cell_type": "code",
-   "execution_count": 2,
    "outputs": [],
+   "execution_count": null,
    "source": [
-    "y = load_airline()\n",
-    "train, test = temporal_train_test_split(y)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2023-12-21T15:56:01.639286900Z",
-     "start_time": "2023-12-21T15:56:01.631308700Z"
-    }
-   }
+    "# y = load_airline()\n",
+    "# train, test = temporal_train_test_split(y)"
+   ]
   },
   {
+   "metadata": {},
    "cell_type": "code",
-   "execution_count": 3,
    "outputs": [],
+   "execution_count": null,
    "source": [
-    "# set_regressor can be used to find various regressors by string, but\n",
-    "# any aeon, tsml or sklearn regressor can be used in the experiments function\n",
-    "forecaster = get_forecaster_by_name(\"NaiveForecaster\")\n",
-    "\n",
-    "# record memory usage every 0.1 seconds, just here for notebook speed\n",
-    "# does not need to be changed for usage\n",
-    "experiments.MEMRECORD_INTERVAL = 0.1\n",
-    "\n",
-    "run_forecasting_experiment(\n",
-    "    np.array(train),\n",
-    "    np.array(test),\n",
-    "    forecaster,\n",
-    "    \"./generated_results/\",\n",
-    "    dataset_name=\"Airline\",\n",
-    "    random_seed=0,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2023-12-21T15:56:02.616604100Z",
-     "start_time": "2023-12-21T15:56:01.642279500Z"
-    }
-   }
+    "# # set_regressor can be used to find various regressors by string, but\n",
+    "# # any aeon, tsml or sklearn regressor can be used in the experiments function\n",
+    "# forecaster = get_forecaster_by_name(\"NaiveForecaster\")\n",
+    "#\n",
+    "# # record memory usage every 0.1 seconds, just here for notebook speed\n",
+    "# # does not need to be changed for usage\n",
+    "# experiments.MEMRECORD_INTERVAL = 0.1\n",
+    "#\n",
+    "# run_forecasting_experiment(\n",
+    "#     np.array(train),\n",
+    "#     np.array(test),\n",
+    "#     forecaster,\n",
+    "#     \"./generated_results/\",\n",
+    "#     dataset_name=\"Airline\",\n",
+    "#     random_seed=0,\n",
+    "# )"
+   ]
   },
   {
+   "metadata": {},
    "cell_type": "markdown",
    "source": [
     "A function is also available to load the dataset as well as run an experiment, see `load_and_run_forecasting_experiment` in `tsml_eval.experiments`.\n",
     "\n",
     "Both experiment functions will output a results file in the `{results_dir}/{forecaster_name}/Predictions/{dataset_name}/` directory. These files can be loaded individually, or used as a collection in the `evaluation` module. See the [evaluation](evaluation.ipynb) notebook for more details."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+   ]
   },
   {
+   "metadata": {},
    "cell_type": "code",
-   "execution_count": 4,
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336.\n",
-      " 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336. 336.\n",
-      " 336. 336. 336. 336. 336. 336. 336. 336.]\n",
-      "0.19886711926999853\n"
-     ]
-    }
-   ],
+   "outputs": [],
+   "execution_count": null,
    "source": [
-    "fr = load_forecaster_results(\n",
-    "    \"./generated_results/NaiveForecaster/Predictions/Airline/testResample0.csv\"\n",
-    ")\n",
-    "print(fr.predictions)\n",
-    "print(fr.mean_absolute_percentage_error)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2023-12-21T15:56:02.624274500Z",
-     "start_time": "2023-12-21T15:56:02.617293100Z"
-    }
-   }
+    "# fr = load_forecaster_results(\n",
+    "#     \"./generated_results/NaiveForecaster/Predictions/Airline/testResample0.csv\"\n",
+    "# )\n",
+    "# print(fr.predictions)\n",
+    "# print(fr.mean_absolute_percentage_error)"
+   ]
   }
  ],
  "metadata": {

examples/regression_experiments.ipynb

@@ -21,7 +21,7 @@
    "source": [
     "import numpy as np\n",
     "import pandas as pd\n",
-    "from aeon.benchmarking import get_estimator_results\n",
+    "from aeon.benchmarking.results_loaders import get_estimator_results\n",
     "from aeon.datasets import load_regression\n",
     "from aeon.regression import DummyRegressor\n",
     "from aeon.visualisation import plot_critical_difference\n",

pyproject.toml

@@ -41,8 +41,8 @@ classifiers = [
 ]
 requires-python = ">=3.9,<3.13"
 dependencies = [
-    "aeon>=0.11.0,<1.0.0",
-    "tsml>=0.4.0,<0.5.0",
+    "aeon>=1.0.0,<1.1.0",
+    "tsml>=0.5.0,<0.6.0",
     "scikit-learn>=1.0.0,<1.5.0",
     "matplotlib",
     "gpustat",
@@ -55,16 +55,14 @@ all_extras = [
     "tsml[all_extras]",
     "xgboost",
 ]
-all-extras = ["tsml-eval[all_extras]"]
 unstable_extras = [
     "aeon[unstable_extras]",
     "tsml[unstable_extras]",
 ]
-unstable-extras = ["tsml-eval[unstable_extras]"]
 deep_learning = [
     "aeon[dl]",
 ]
-deep-learning = ["tsml-eval[deep_learning]"]
+
 dev = [
     "pre-commit",
     "pytest",

tsml_eval/estimators/classification/hybrid/tests/test_hivecote.py

@@ -5,7 +5,7 @@
 
 import numpy as np
 from aeon.datasets import load_arrow_head, load_italy_power_demand
-from aeon.testing.utils.estimator_checks import _assert_array_almost_equal
+from numpy.testing import assert_array_almost_equal
 
 from tsml_eval.estimators.classification.hybrid.hivecote_from_file import (
     FromFileHIVECOTE,
@@ -54,7 +54,7 @@ def test_hivecote_from_file():
     # 0.07846402768, 0.92153597231
 
     assert probas.shape == (X_test.shape[0], 2)
-    _assert_array_almost_equal(probas[0], np.array([0.0785, 0.9215]), decimal=4)
+    assert_array_almost_equal(probas[0], np.array([0.0785, 0.9215]), decimal=4)
 
 
 def test_tuned_hivecote_from_file():
@@ -74,4 +74,4 @@ def test_tuned_hivecote_from_file():
     probas = hc2.predict_proba(X_test)
 
     assert probas.shape == (X_test.shape[0], 3)
-    _assert_array_almost_equal(probas[0], np.array([0.6092, 0.2308, 0.16]), decimal=4)
+    assert_array_almost_equal(probas[0], np.array([0.6092, 0.2308, 0.16]), decimal=4)