From 44b78550447a23204b9d7fa6f02d3cf6a1d65da2 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Tue, 12 Aug 2025 12:32:45 +0100 Subject: [PATCH 01/17] forecasting PR draft --- tsml_eval/experiments/_get_forecaster.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tsml_eval/experiments/_get_forecaster.py b/tsml_eval/experiments/_get_forecaster.py index a76967cc..da797e06 100644 --- a/tsml_eval/experiments/_get_forecaster.py +++ b/tsml_eval/experiments/_get_forecaster.py @@ -55,6 +55,8 @@ def _set_forecaster_stats(f, random_state, n_jobs, kwargs): if f == "etsforecaster" or f == "ets": return ETSForecaster(**kwargs) + # todo + def _set_forecaster_other(f, random_state, n_jobs, kwargs): if f == "dummyforecaster" or f == "dummy": From be6b8a68e477bc5b6a7e59305ad0f107d3bd2571 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Tue, 12 Aug 2025 12:40:46 +0100 Subject: [PATCH 02/17] fix --- .github/workflows/issue_comment_edited.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/issue_comment_edited.yml b/.github/workflows/issue_comment_edited.yml index 8ca7a019..5f9517e0 100644 --- a/.github/workflows/issue_comment_edited.yml +++ b/.github/workflows/issue_comment_edited.yml @@ -53,9 +53,3 @@ jobs: commit_options: --allow-empty create_branch: false skip_dirty_check: true - - - name: Uncheck relevant boxes - run: python .github/utilities/uncheck_welcome_boxes.py - env: - CONTEXT_GITHUB: ${{ toJson(github) }} - GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} From fa79edf707e3224449ca81f7b443551e8a2eb451 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 17:31:41 +0100 Subject: [PATCH 03/17] 1.3.0 and script changes --- .github/utilities/pr_welcome_edited.py | 7 - .github/utilities/uncheck_welcome_boxes.py | 36 +++ .github/workflows/issue_comment_edited.yml | 6 + _tsml_research_resources/README.md | 6 +- .../soton/iridis/README.md | 2 +- .../taskfarm_classification_experiments.sh | 61 +++-- .../taskfarm_clustering_experiments.sh | 231 +++++++++++++++++ .../taskfarm_regression_experiments.sh | 219 ++++++++++++++++ .../gpu_classification_experiments.sh | 6 +- .../gpu_scipts/gpu_clustering_experiments.sh | 4 +- .../gpu_scipts/gpu_regression_experiments.sh | 4 +- .../soton/iridis/iridis_python.md | 58 +++-- .../soton/iridis/range_scancel.sh | 12 + .../classification_experiments.sh | 6 +- .../serial_scripts/clustering_experiments.sh | 4 +- .../serial_scripts/regression_experiments.sh | 4 +- .../threaded_classification_experiments.sh | 18 +- .../threaded_clustering_experiments.sh | 165 ++++++++++++ .../threaded_regression_experiments.sh | 153 ++++++++++++ ...ded_taskfarm_classification_experiments.sh | 223 +++++++++++++++++ ...hreaded_taskfarm_clustering_experiments.sh | 235 ++++++++++++++++++ ...hreaded_taskfarm_regression_experiments.sh | 223 +++++++++++++++++ .../uea/ada/classification_experiments.sh | 2 +- .../uea/ada/clustering_experiments.sh | 2 +- .../uea/ada/gpu_classification_experiments.sh | 2 +- .../uea/ada/gpu_clustering_experiments.sh | 2 +- .../uea/ada/gpu_regression_experiments.sh | 2 +- .../uea/ada/regression_experiments.sh | 2 +- docs/conf.py | 1 + pyproject.toml | 3 +- .../multiple_estimator_evaluation.py | 8 +- tsml_eval/experiments/_get_classifier.py | 20 +- tsml_eval/experiments/_get_data_transform.py | 49 +++- tsml_eval/experiments/_get_forecaster.py | 92 ++++++- .../experiments/tests/test_data_transform.py | 3 +- .../experiments/tests/test_regression.py | 1 + .../run_distance_experiments.py | 12 +- 37 files changed, 1768 insertions(+), 116 deletions(-) create mode 100644 .github/utilities/uncheck_welcome_boxes.py create mode 100644 _tsml_research_resources/soton/iridis/batch_scripts/taskfarm_clustering_experiments.sh create mode 100644 _tsml_research_resources/soton/iridis/batch_scripts/taskfarm_regression_experiments.sh create mode 100644 _tsml_research_resources/soton/iridis/range_scancel.sh rename _tsml_research_resources/soton/iridis/{thread_scripts => threaded_scripts}/threaded_classification_experiments.sh (88%) create mode 100644 _tsml_research_resources/soton/iridis/threaded_scripts/threaded_clustering_experiments.sh create mode 100644 _tsml_research_resources/soton/iridis/threaded_scripts/threaded_regression_experiments.sh create mode 100644 _tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_classification_experiments.sh create mode 100644 _tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_clustering_experiments.sh create mode 100644 _tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_regression_experiments.sh diff --git a/.github/utilities/pr_welcome_edited.py b/.github/utilities/pr_welcome_edited.py index 13c14213..885905c0 100644 --- a/.github/utilities/pr_welcome_edited.py +++ b/.github/utilities/pr_welcome_edited.py @@ -44,13 +44,6 @@ print(f"branch={branch_name}", file=fh) # noqa: T201 if "- [x] Push an empty commit to re-run CI checks" in comment_body: - comment.edit( - comment_body.replace( - "- [x] Push an empty commit to re-run CI checks", - "- [ ] Push an empty commit to re-run CI checks", - ) - ) - with open(os.environ["GITHUB_OUTPUT"], "a") as fh: print("empty_commit=true", file=fh) # noqa: T201 else: diff --git a/.github/utilities/uncheck_welcome_boxes.py b/.github/utilities/uncheck_welcome_boxes.py new file mode 100644 index 00000000..1ad49a12 --- /dev/null +++ b/.github/utilities/uncheck_welcome_boxes.py @@ -0,0 +1,36 @@ +"""Uncheck relevant boxes after running the comment edit workflow.""" + +import json +import os +import sys + +from github import Github + +context_dict = json.loads(os.getenv("CONTEXT_GITHUB")) + +repo = context_dict["repository"] +g = Github(os.getenv("GITHUB_TOKEN")) +repo = g.get_repo(repo) +issue_number = context_dict["event"]["issue"]["number"] +issue = repo.get_issue(number=issue_number) +comment_body = context_dict["event"]["comment"]["body"] +comment_user = context_dict["event"]["comment"]["user"]["login"] + +if ( + "[bot]" in context_dict["event"]["sender"]["login"] + or issue.pull_request is None + or comment_user != "tsml-actions-bot[bot]" + or "## Thank you for contributing to `tsml-eval`" not in comment_body +): + sys.exit(0) + +pr = issue.as_pull_request() +comment = pr.get_issue_comment(context_dict["event"]["comment"]["id"]) + +if "- [x] Push an empty commit to re-run CI checks" in comment_body: + comment.edit( + comment_body.replace( + "- [x] Push an empty commit to re-run CI checks", + "- [ ] Push an empty commit to re-run CI checks", + ) + ) diff --git a/.github/workflows/issue_comment_edited.yml b/.github/workflows/issue_comment_edited.yml index 5f9517e0..8ca7a019 100644 --- a/.github/workflows/issue_comment_edited.yml +++ b/.github/workflows/issue_comment_edited.yml @@ -53,3 +53,9 @@ jobs: commit_options: --allow-empty create_branch: false skip_dirty_check: true + + - name: Uncheck relevant boxes + run: python .github/utilities/uncheck_welcome_boxes.py + env: + CONTEXT_GITHUB: ${{ toJson(github) }} + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/_tsml_research_resources/README.md b/_tsml_research_resources/README.md index 665e1aac..ba55c502 100644 --- a/_tsml_research_resources/README.md +++ b/_tsml_research_resources/README.md @@ -1,8 +1,8 @@ This directory contains instructions and utilities for running experiments using -tsml-eval on University of East Anglia (UEA) and University of Southampton (Soton) -hardware. +tsml-eval on University of East Anglia (UEA), University of Southampton (Soton) and +University of Bradford hardware. -For non-UEA/Soton users, the directory contents are likely irrelevant. +For non-UEA/Soton/Bradford users, the directory contents are likely irrelevant. Some of the contents could be adapted for other purposes or could be generalised for other Slurm/Linux devices, but running the scripts or following the hardware instructions will likely achieve nothing without alterations. diff --git a/_tsml_research_resources/soton/iridis/README.md b/_tsml_research_resources/soton/iridis/README.md index ca5fec19..99ffeb64 100644 --- a/_tsml_research_resources/soton/iridis/README.md +++ b/_tsml_research_resources/soton/iridis/README.md @@ -9,7 +9,7 @@ batch_scripts: but requires more setup to work efficiently. thread_scripts: - Similar to serial_scripts but for multi-threaded jobs. + Similar to serial_scripts and batch_scripts but for multithreaded jobs. gpu_scripts: Scripts for running GPU jobs. These are similar to the serial scripts, but diff --git a/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_classification_experiments.sh b/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_classification_experiments.sh index 777c91fd..2f15fffe 100644 --- a/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_classification_experiments.sh +++ b/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_classification_experiments.sh @@ -12,9 +12,19 @@ max_num_submitted=900 # Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx queue="batch" -# The number of tasks/threads to use in each job. 40 is the number of cores on batch nodes +# The number of tasks to submit in each job. This can be larger than the number of cores, but tasks will be delayed until a core is free n_tasks_per_node=40 +# The number of cores to request from the node. Don't go over the number of cores for the node. 40 is the number of cores on batch nodes +# If you are not using the whole node, please make sure you are requesting memory correctly +max_cpus_to_use=40 + +# Create a separate submission list for each classifier. This will stop the mixing of +# large and small jobs in the same node, but results in some smaller scripts submitted +# to serial when moving between classifiers. +# For small workloads i.e. single resample 10 datasets, turning this off will be the only way to get on the batch queue realistically +split_classifiers="true" + # Enter your username and email here username="ajb2u23" mail="NONE" @@ -29,7 +39,7 @@ start_point=1 # Put your home directory here local_path="/mainfs/home/$username/" -# Datasets to use and directory of data files. This can either be a text file or directory of text files +# Datasets to use and directory of data files. Dataset list can either be a text file or directory of text files # Separate text files will not run jobs of the same dataset in the same node. This is good to keep large and small datasets separate data_dir="$local_path/Data/" dataset_list="$local_path/DataSetLists/ClassificationBatch/" @@ -45,7 +55,7 @@ script_file_path="$local_path/tsml-eval/tsml_eval/experiments/classification_exp # Separate environments for GPU and CPU are recommended env_name="eval-py11" -# Classifiers to loop over. Must be separated by a space. Different classifiers will not run in the same node +# Classifiers to loop over. Must be separated by a space. Different classifiers will not run in the same node by default # See list of potential classifiers in set_classifier classifiers_to_run="ROCKET DrCIF" @@ -62,12 +72,10 @@ predefined_folds="false" # Normalise data before fit/predict normalise_data="false" - # ====================================================================================== # Experiment configuration end # ====================================================================================== - # Set to -tr to generate test files generate_train_files=$([ "${generate_train_files,,}" == "true" ] && echo "-tr" || echo "") @@ -80,23 +88,29 @@ normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") # This creates the submission file to run and does clean up submit_jobs () { +if ((cmdCount>=max_cpus_to_use)); then + cpuCount=$max_cpus_to_use +else + cpuCount=$cmdCount +fi + echo "#!/bin/bash #SBATCH --mail-type=${mail} #SBATCH --mail-user=${mailto} #SBATCH --job-name=batch-${dt} #SBATCH -p ${queue} #SBATCH -t ${max_time} -#SBATCH -o ${out_dir}/${classifier}/%A-${dt}.out -#SBATCH -e ${out_dir}/${classifier}/%A-${dt}.err +#SBATCH -o ${outDir}/%A-${dt}.out +#SBATCH -e ${outDir}/%A-${dt}.err #SBATCH --nodes=1 -#SBATCH --ntasks=${cmdCount} +#SBATCH --ntasks=${cpuCount} . /etc/profile module load anaconda/py3.10 source activate $env_name -staskfarm ${out_dir}/${classifier}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub +staskfarm ${outDir}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub echo "At experiment ${expCount}, ${totalCount} jobs submitted total" @@ -108,6 +122,7 @@ rm generatedSubmissionFile-${dt}.sub totalCount=0 expCount=0 +dt=$(date +%Y%m%d%H%M%S) # turn a directory of files into a list if [[ -d $dataset_list ]]; then @@ -126,11 +141,15 @@ for classifier in $classifiers_to_run; do mkdir -p "${out_dir}/${classifier}/" -# create a new command list for each classifier and dataset list -# we use time for unique names -sleep 1 -cmdCount=0 -dt=$(date +%Y%m%d%H%M%S) +if [ "${split_classifiers,,}" == "true" ]; then + # we use time for unique names + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) + outDir=${out_dir}/${classifier} +else + outDir=${out_dir} +fi while read dataset; do @@ -162,7 +181,7 @@ if ((cmdCount>=n_tasks_per_node)); then num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) while [ "${num_jobs}" -ge "${max_num_submitted}" ] do - echo Waiting 60s, "${num_jobs}" currently submitted on ${queue}, user-defined max is ${max_num_submitted} + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} sleep 60 num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) done @@ -174,7 +193,7 @@ fi # Input args to the default classification_experiments are in main method of # https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/classification_experiments.py -echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} ${resample} ${generate_train_files} ${predefined_folds} ${normalise_data}" >> ${out_dir}/${classifier}/generatedCommandList-${dt}.txt +echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} ${resample} ${generate_train_files} ${predefined_folds} ${normalise_data} > ${out_dir}/${classifier}/output-${dataset}-${resample}-${dt}.txt 2>&1" >> ${outDir}/generatedCommandList-${dt}.txt ((cmdCount++)) ((totalCount++)) @@ -183,12 +202,18 @@ done fi done < ${dataset_file} -if ((cmdCount>0)); then - # final submit for this dataset list and classifier +if [[ "${split_classifiers,,}" == "true" && $cmdCount -gt 0 ]]; then + # final submit for this classifier submit_jobs fi done + +if [[ "${split_classifiers,,}" != "true" && $cmdCount -gt 0 ]]; then + # final submit for this dataset list + submit_jobs +fi + done echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_clustering_experiments.sh b/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_clustering_experiments.sh new file mode 100644 index 00000000..b30c8754 --- /dev/null +++ b/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_clustering_experiments.sh @@ -0,0 +1,231 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=10 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=900 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of tasks to submit in each job. This can be larger than the number of cores, but tasks will be delayed until a core is free +n_tasks_per_node=40 + +# The number of cores to request from the node. Don't go over the number of cores for the node. 40 is the number of cores on batch nodes +# If you are not using the whole node, please make sure you are requesting memory correctly +max_cpus_to_use=40 + +# Create a separate submission list for each clusterer. This will stop the mixing of +# large and small jobs in the same node, but results in some smaller scripts submitted +# to serial when moving between clusterers. +# For small workloads i.e. single resample 10 datasets, turning this off will be the only way to get on the batch queue realistically +split_clusterers="true" + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto=$username"@soton.ac.uk" + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 clusterers = 9 experiments to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Dataset list can either be a text file or directory of text files +# Separate text files will not run jobs of the same dataset in the same node. This is good to keep large and small datasets separate +data_dir="$local_path/Data/" +dataset_list="$local_path/DataSetLists/ClusteringBatch/" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/ClusteringResults/results/" +out_dir="$local_path/ClusteringResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/clustering_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="eval-py11" + +# Clusterers to loop over. Must be separated by a space. Different clusterers will not run in the same node by default +# See list of potential clusterers in set_clusterer +clusterers_to_run="kmedoids-squared kmedoids-euclidean" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the test data as well as train, usually slower +generate_test_files="true" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Boolean on if to combine the test/train split +combine_test_train_split="false" + +# Normalise data before fit/predict +normalise_data="true" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -te to generate test files +generate_test_files=$([ "${generate_test_files,,}" == "true" ] && echo "-te" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Update result path to split combined test train split and test train split +results_dir="${results_dir}$([ "${combine_test_train_split,,}" == "true" ] && echo "combine-test-train-split/" || echo "test-train-split/")" + +# Update out path to split combined test train split and test train split +out_dir="${out_dir}$([ "${combine_test_train_split,,}" == "true" ] && echo "combine-test-train-split/" || echo "test-train-split/")" + +# Set to -utts to combine test train split +combine_test_train_split=$([ "${combine_test_train_split,,}" == "true" ] && echo "-ctts" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# This creates the submission file to run and does clean up +submit_jobs () { + +if ((cmdCount>=max_cpus_to_use)); then + cpuCount=$max_cpus_to_use +else + cpuCount=$cmdCount +fi + +echo "#!/bin/bash +#SBATCH --mail-type=${mail} +#SBATCH --mail-user=${mailto} +#SBATCH --job-name=batch-${dt} +#SBATCH -p ${queue} +#SBATCH -t ${max_time} +#SBATCH -o ${outDir}/%A-${dt}.out +#SBATCH -e ${outDir}/%A-${dt}.err +#SBATCH --nodes=1 +#SBATCH --ntasks=${cpuCount} + +. /etc/profile + +module load anaconda/py3.10 +source activate $env_name + +staskfarm ${outDir}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub + +echo "At experiment ${expCount}, ${totalCount} jobs submitted total" + +sbatch < generatedSubmissionFile-${dt}.sub + +rm generatedSubmissionFile-${dt}.sub + +} + +totalCount=0 +expCount=0 +dt=$(date +%Y%m%d%H%M%S) + +# turn a directory of files into a list +if [[ -d $dataset_list ]]; then + file_names="" + for file in ${dataset_list}/*; do + file_names="$file_names$dataset_list$(basename "$file") " + done + dataset_list=$file_names +fi + +for dataset_file in $dataset_list; do + +echo "Dataset list ${dataset_file}" + +for clusterer in $clusterers_to_run; do + +mkdir -p "${out_dir}/${clusterer}/" + +if [ "${split_clusterers,,}" == "true" ]; then + # we use time for unique names + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) + outDir=${out_dir}/${clusterer} +else + outDir=${out_dir} +fi + +while read dataset; do + +# Skip to the script start point +((expCount++)) +if ((expCount>=start_point)); then + +# This finds the resamples to run and skips jobs which have test/train files already written to the results directory. +# This can result in uneven sized command lists +resamples_to_run="" +for (( i=start_fold-1; i=n_tasks_per_node)); then + submit_jobs + + # This is the loop to stop you from dumping everything in the queue at once, see max_num_submitted + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + while [ "${num_jobs}" -ge "${max_num_submitted}" ] + do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + done + + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) +fi + +# Input args to the default clustering_experiments are in main method of +# https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/clustering_experiments.py +echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${clusterer} ${dataset} ${resample} ${generate_test_files} ${predefined_folds} ${combine_test_train_split} ${normalise_data} > ${out_dir}/${clusterer}/output-${dataset}-${resample}-${dt}.txt 2>&1" >> ${outDir}/generatedCommandList-${dt}.txt + +((cmdCount++)) +((totalCount++)) + +done +fi +done < ${dataset_file} + +if [[ "${split_clusterers,,}" == "true" && $cmdCount -gt 0 ]]; then + # final submit for this clusterer + submit_jobs +fi + +done + +if [[ "${split_clusterers,,}" != "true" && $cmdCount -gt 0 ]]; then + # final submit for this dataset list + submit_jobs +fi + +done + +echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_regression_experiments.sh b/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_regression_experiments.sh new file mode 100644 index 00000000..ec378afc --- /dev/null +++ b/_tsml_research_resources/soton/iridis/batch_scripts/taskfarm_regression_experiments.sh @@ -0,0 +1,219 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=10 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=900 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of tasks to submit in each job. This can be larger than the number of cores, but tasks will be delayed until a core is free +n_tasks_per_node=40 + +# The number of cores to request from the node. Don't go over the number of cores for the node. 40 is the number of cores on batch nodes +# If you are not using the whole node, please make sure you are requesting memory correctly +max_cpus_to_use=40 + +# Create a separate submission list for each regressor. This will stop the mixing of +# large and small jobs in the same node, but results in some smaller scripts submitted +# to serial when moving between regressors. +# For small workloads i.e. single resample 10 datasets, turning this off will be the only way to get on the batch queue realistically +split_regressors="true" + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto=$username"@soton.ac.uk" + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 regressors = 9 experiments to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Dataset list can either be a text file or directory of text files +# Separate text files will not run jobs of the same dataset in the same node. This is good to keep large and small datasets separate +data_dir="$local_path/Data/" +dataset_list="$local_path/DataSetLists/RegressionBatch/" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/RegressionResults/results/" +out_dir="$local_path/RegressionResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/regression_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="eval-py11" + +# Regressors to loop over. Must be separated by a space. Different regressors will not run in the same node by default +# See list of potential regressors in set_regressor +regressors_to_run="ROCKET DrCIF" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the train data as well as test, usually slower +generate_train_files="false" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Normalise data before fit/predict +normalise_data="false" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -tr to generate test files +generate_train_files=$([ "${generate_train_files,,}" == "true" ] && echo "-tr" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# This creates the submission file to run and does clean up +submit_jobs () { + +if ((cmdCount>=max_cpus_to_use)); then + cpuCount=$max_cpus_to_use +else + cpuCount=$cmdCount +fi + +echo "#!/bin/bash +#SBATCH --mail-type=${mail} +#SBATCH --mail-user=${mailto} +#SBATCH --job-name=batch-${dt} +#SBATCH -p ${queue} +#SBATCH -t ${max_time} +#SBATCH -o ${outDir}/%A-${dt}.out +#SBATCH -e ${outDir}/%A-${dt}.err +#SBATCH --nodes=1 +#SBATCH --ntasks=${cpuCount} + +. /etc/profile + +module load anaconda/py3.10 +source activate $env_name + +staskfarm ${outDir}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub + +echo "At experiment ${expCount}, ${totalCount} jobs submitted total" + +sbatch < generatedSubmissionFile-${dt}.sub + +rm generatedSubmissionFile-${dt}.sub + +} + +totalCount=0 +expCount=0 +dt=$(date +%Y%m%d%H%M%S) + +# turn a directory of files into a list +if [[ -d $dataset_list ]]; then + file_names="" + for file in ${dataset_list}/*; do + file_names="$file_names$dataset_list$(basename "$file") " + done + dataset_list=$file_names +fi + +for dataset_file in $dataset_list; do + +echo "Dataset list ${dataset_file}" + +for regressor in $regressors_to_run; do + +mkdir -p "${out_dir}/${regressor}/" + +if [ "${split_regressors,,}" == "true" ]; then + # we use time for unique names + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) + outDir=${out_dir}/${regressor} +else + outDir=${out_dir} +fi + +while read dataset; do + +# Skip to the script start point +((expCount++)) +if ((expCount>=start_point)); then + +# This finds the resamples to run and skips jobs which have test/train files already written to the results directory. +# This can result in uneven sized command lists +resamples_to_run="" +for (( i=start_fold-1; i=n_tasks_per_node)); then + submit_jobs + + # This is the loop to stop you from dumping everything in the queue at once, see max_num_submitted + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + while [ "${num_jobs}" -ge "${max_num_submitted}" ] + do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + done + + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) +fi + +# Input args to the default regression_experiments are in main method of +# https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/regression_experiments.py +echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${regressor} ${dataset} ${resample} ${generate_train_files} ${predefined_folds} ${normalise_data} > ${out_dir}/${regressor}/output-${dataset}-${resample}-${dt}.txt 2>&1" >> ${outDir}/generatedCommandList-${dt}.txt + +((cmdCount++)) +((totalCount++)) + +done +fi +done < ${dataset_file} + +if [[ "${split_regressors,,}" == "true" && $cmdCount -gt 0 ]]; then + # final submit for this regressor + submit_jobs +fi + +done + +if [[ "${split_regressors,,}" != "true" && $cmdCount -gt 0 ]]; then + # final submit for this dataset list + submit_jobs +fi + +done + +echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh index dab1a41b..55ae9056 100644 --- a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh +++ b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh @@ -86,7 +86,7 @@ if ((count>=start_point)); then num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) while [ "${num_jobs}" -ge "${max_num_submitted}" ] do - echo Waiting 60s, "${num_jobs}" currently submitted on ${queue}, user-defined max is ${max_num_submitted} + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} sleep 60 num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) done @@ -98,7 +98,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${classifier}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh index 810063d3..5da6ea06 100644 --- a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh +++ b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh @@ -110,7 +110,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +python -u ${script_file_path} ${data_dir} ${results_dir} ${clusterer} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_test_files} ${predefined_folds} ${combine_test_train_split} ${normalise_data}" > generatedFile.sub echo "${count} ${clusterer}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh index 38c3cd07..04f68666 100644 --- a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh +++ b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh @@ -98,7 +98,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +python -u ${script_file_path} ${data_dir} ${results_dir} ${regressor} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${regressor}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/iridis_python.md b/_tsml_research_resources/soton/iridis/iridis_python.md index 5c73a713..b865fd9c 100644 --- a/_tsml_research_resources/soton/iridis/iridis_python.md +++ b/_tsml_research_resources/soton/iridis/iridis_python.md @@ -1,4 +1,5 @@ # Iridis 5 Python +##### Last updated: 07/09/2025 Installation guide for Python packages on Iridis 5 and useful slurm commands. @@ -20,7 +21,7 @@ There is a Southampton Microsoft Teams group called "HPC Community" where you ca You need to be on a Soton network machine or have the VPN running to connect to Iridis. Connect to one of the addresses listed above. -The recommended way of connecting to Iridis is using Putty as a command-line interface and WinSCP for file management. +The recommended way of connecting to Iridis in our group is using Putty for a SSH command-line interface and WinSCP for FTP file management. Copies of data files used in experiments must be stored on the cluster, the best place to put these files is on your user area scratch storage. It is a good idea to create shortcuts to and from your scratch drive. Alternatively, you can read from someone else's directory (i.e. `/mainfs/scratch/mbm1g23/`). @@ -28,11 +29,11 @@ Copies of data files used in experiments must be stored on the cluster, the best Complete these steps sequentially for a fresh installation. -By default, commands will be run on the login node. Beyond simple commands or scripts, an interactive session should be started. For most usage here you will not require one. +By default, commands will be run on the login node. Beyond simple commands or scripts, an interactive session should be started. For most usage (including setting up and running submission scripts as guided here) you will not require one. >sinteractive -__DO NOT__ enter interactive mode for commands that require an internet connection (i.e. steps 1-4), as only the login nodes have one. +__DO NOT__ enter interactive mode for commands that require an internet connection (i.e. steps 1-4), as it will not work. ### 1. Clone the code from GitHub @@ -64,15 +65,15 @@ You can check the current version using: #### 3.1. Set up scratch symbolic link for conda -Installing complex conda packages on your main home drive will quickly see you hitting the limit on the number of files you can store. To avoid this, it is recommended you create a symbolic link to your scratch storage. +Installing complex conda packages on your main home drive will quickly see you hitting the limit on the number of files you can store. To avoid this, it is recommended you create a symbolic link to your scratch storage (this assumes you are in your home directory and there is a symlink to your scratch drive). >mkdir /scratch//.conda >ln -s /scratch//.conda ~/.conda -Hitting this limit is very annoying, as it will prevent you from creating new conda environments or installing new packages (or doing anything really). +Hitting this limit is very annoying, as it will prevent you from creating new conda environments, installing new packages or saving results file (or doing anything really). -For conda related storage guidance, see the [related HPC webpage](https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Conda.aspx#conda-and-inodes) +For conda related storage guidance, see the [related HPC webpage](https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Conda.aspx#conda-and-inodes). #### 3.2. Create environment @@ -89,7 +90,7 @@ Your environment should be listed now when you use the following command: >conda info --envs -#### 3.3. Removing an environment +#### 3.3. Removing an environment (not required for first setup) At some point you may want to remove an environment, either because it is no longer needed or you want to start fresh. You can do this with the following command: @@ -106,7 +107,7 @@ After installation, the installed packages can be viewed with: >conda list -Note that you can install a specific GitHub branch for packages such as `aeon` like so +Note that you can install a specific GitHub branch for packages such as `aeon` like so. It is important to uninstall any existing version first. >pip uninstall aeon @@ -118,11 +119,15 @@ or #### 4.1. tsml-eval CPU -Move to the package directory and run: +Move to the package directory i.e. + +>ls tsml-eval + +This will have a `pyproject.toml` file. Run: >pip install --editable . -For release specific dependency versions you can also run: +For release specific dependency versions you can also run (replace `requirements.txt` with the relevant file): >pip install -r requirements.txt @@ -134,7 +139,7 @@ For some extras you may need a gcc installation i.e.: >module add gcc/11.1.0 -Most extra dependencies can be installed with the all_extras dependency set: +Most extra dependencies can be installed with the `all_extras` dependency set: >pip install -e .[all_extras] @@ -142,7 +147,7 @@ Some dependencies are unstable, so the following may fail to install. >pip install -e .[all_extras,unstable_extras] -If any a dependency install is "Killed", it is likely the interactive session has run out of memory. Either give it more memory, or use a non-cached package i.e. +If any a dependency install is "Killed", it is likely the session has run out of memory. Either give it more memory, or use a non-cached package i.e. >pip install PACKAGE_NAME --no-cache-dir @@ -154,10 +159,14 @@ It is recommended to use a different environment for GPU jobs. Move to the packa # Running experiments -For running jobs on Iridis, we recommend using copies of the submission scripts provided in this folder. +For running jobs on Iridis, we recommend using *copies* of the submission scripts provided in this folder. **NOTE: Scripts will not run properly if done whilst the conda environment is active.** +Disable the conda environment before running scripts if you have installed packages: + +>conda deactivate + ## Running `tsml-eval` CPU experiments For CPU experiments start with one of the following scripts: @@ -176,11 +185,24 @@ You may need to use `dos2unix` to convert the line endings to unix format. The default queue for CPU jobs is _batch_. Be sure to swap the _queue_alias_ to _serial_ in the script if you want to use this, as the number of jobs submitted won't be tracked properly otherwise. -Do not run threaded code on the cluster without reserving whole nodes, as there is nothing to stop the job from using -the CPU resources allocated to others. The default python file in the scripts attempts to avoid threading as much as possible. You should ensure processes are not intentionally using multiple threads if you change it. +Do not run threaded code on the cluster without requesting the correct amount of CPUs or reserving a whole node, as there is nothing to stop the job from using the CPU resources allocated to others. The default python file in the scripts attempts to avoid threading as much as possible. You should ensure processes are not intentionally using multiple threads if you change it. Requesting memory for a job will allocate it all on the jobs assigned node. New jobs will not be submitted to a node if the total allocated memory exceeds the amount available for the node. As such, requesting too much memory can block new jobs from using the node. This is ok if the memory is actually being used, but large amounts of memory should not be requested unless you know it will be required for the jobs you are submitting. Iridis is a shared resource, and instantly requesting hundreds of GB will hurt the overall efficiency of the cluster. +## Running `tsml-eval` CPU experiments on the Iridis 5 batch queue + +If you submit less than 20 tasks when requesting the _batch_ queue, your job will be redirected to the _serial_ queue. This has a much smaller job limit which you will reach quickly when submitting a lot of jobs. If you submit a single task in each submission, you will only be running ~32 jobs at once. + +To get around this, you can use the batch submission scripts provided in the `batch_scripts` folder. These scripts submit multiple tasks in a single job, allowing you to run many more experiments at once. + +>taskfarm_classification_experiments.sh + +>taskfarm_regression_experiments.sh + +>taskfarm_clustering_experiments.sh + +They are named this as they use the `staskfarm` utility to run different processes over multiple threads. Read through the configuration as it is slightly different to the serial scripts. You can split task groupings by dataset by loading from a directory of submission scripts and keep classifiers separate with a variable. + ## Running `tsml-eval` GPU experiments For GPU experiments use one of the following scripts: @@ -205,7 +227,7 @@ __Tip__: to simplify and just use 'queue' in the terminal to run the above comma >alias queue='squeue -u USERNAME --format="%12i %15P %20j %10u %10t %10M %10D %20R" -r' -To kill all user jobs +To kill all user jobs: >scancel -u USERNAME @@ -217,6 +239,10 @@ To delete one job it’s: >scancel 11133013_1 +To delete jobs in a specific job ID range use the `range_scancel` script: + +>sh range_scancel.sh + ## Helpful links conda cheat sheet: diff --git a/_tsml_research_resources/soton/iridis/range_scancel.sh b/_tsml_research_resources/soton/iridis/range_scancel.sh new file mode 100644 index 00000000..7e38e73c --- /dev/null +++ b/_tsml_research_resources/soton/iridis/range_scancel.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# start and end job IDs to cancel +start=7615996 +end=7616025 + +for ((n=start; n<=end; n++)); do + echo "Cancelling job ID $n" + if ! scancel "$n" 2>/dev/null; then + echo "Failed to cancel job ID $n" + fi +done diff --git a/_tsml_research_resources/soton/iridis/serial_scripts/classification_experiments.sh b/_tsml_research_resources/soton/iridis/serial_scripts/classification_experiments.sh index a1c273ef..0b48af08 100644 --- a/_tsml_research_resources/soton/iridis/serial_scripts/classification_experiments.sh +++ b/_tsml_research_resources/soton/iridis/serial_scripts/classification_experiments.sh @@ -90,7 +90,7 @@ if ((count>=start_point)); then num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) while [ "${num_jobs}" -ge "${max_num_submitted}" ] do - echo Waiting 60s, "${num_jobs}" currently submitted on ${queue}, user-defined max is ${max_num_submitted} + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} sleep 60 num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) done @@ -102,7 +102,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${classifier}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/serial_scripts/clustering_experiments.sh b/_tsml_research_resources/soton/iridis/serial_scripts/clustering_experiments.sh index 291f12a3..e901989d 100644 --- a/_tsml_research_resources/soton/iridis/serial_scripts/clustering_experiments.sh +++ b/_tsml_research_resources/soton/iridis/serial_scripts/clustering_experiments.sh @@ -114,7 +114,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +python -u ${script_file_path} ${data_dir} ${results_dir} ${clusterer} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_test_files} ${predefined_folds} ${combine_test_train_split} ${normalise_data}" > generatedFile.sub echo "${count} ${clusterer}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/serial_scripts/regression_experiments.sh b/_tsml_research_resources/soton/iridis/serial_scripts/regression_experiments.sh index d9442c76..c7eef2da 100644 --- a/_tsml_research_resources/soton/iridis/serial_scripts/regression_experiments.sh +++ b/_tsml_research_resources/soton/iridis/serial_scripts/regression_experiments.sh @@ -102,7 +102,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +python -u ${script_file_path} ${data_dir} ${results_dir} ${regressor} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${regressor}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/thread_scripts/threaded_classification_experiments.sh b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_classification_experiments.sh similarity index 88% rename from _tsml_research_resources/soton/iridis/thread_scripts/threaded_classification_experiments.sh rename to _tsml_research_resources/soton/iridis/threaded_scripts/threaded_classification_experiments.sh index dc6accf2..e9d6a135 100644 --- a/_tsml_research_resources/soton/iridis/thread_scripts/threaded_classification_experiments.sh +++ b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_classification_experiments.sh @@ -12,14 +12,17 @@ max_num_submitted=100 # Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx queue="batch" -# The number of threads to request. Please check the number of cores for the node you are using, some are exclusive (i.e. batch) so you should request all cores -n_threads=40 +# The number of threads to request. Please check the number of cores for the node you are using, some are exclusive (i.e. batch) so you should request all cores or a taskfarm script +n_threads=10 # Enter your username and email here username="ajb2u23" mail="NONE" mailto="$username@soton.ac.uk" +# MB for jobs, increase incrementally and try not to use more than you need. If you need hundreds of GB consider the huge memory queue +max_memory=8000 + # Max allowable is 60 hours max_time="60:00:00" @@ -90,7 +93,7 @@ if ((count>=start_point)); then num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) while [ "${num_jobs}" -ge "${max_num_submitted}" ] do - echo Waiting 60s, "${num_jobs}" currently submitted on ${queue}, user-defined max is ${max_num_submitted} + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} sleep 60 num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) done @@ -102,7 +105,7 @@ array_jobs="" for (( i=start_fold-1; i generatedFile.sub +# Input args to the default threaded_classification_experiments are in main method of +# https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/threaded_classification_experiments.py +python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) -nj ${n_threads} ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${classifier}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_clustering_experiments.sh b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_clustering_experiments.sh new file mode 100644 index 00000000..73b52028 --- /dev/null +++ b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_clustering_experiments.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=30 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=100 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of threads to request. Please check the number of cores for the node you are using, some are exclusive (i.e. batch) so you should request all cores or a taskfarm script +n_threads=10 + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto="$username@soton.ac.uk" + +# MB for jobs, increase incrementally and try not to use more than you need. If you need hundreds of GB consider the huge memory queue +max_memory=8000 + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 clusterers = 9 jobs to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Default is Tony's work space, all should be able to read these. Change if you want to use different data or lists +data_dir="$local_path/Data/" +datasets="$local_path/DataSetLists/Clustering.txt" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/ClusteringResults/results/" +out_dir="$local_path/ClusteringResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/threaded_clustering_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="tsml-eval" + +# Clusterers to loop over. Must be separated by a space +# See list of potential clusterers in set_clusterer +clusterers_to_run="kmedoids-squared kmedoids-euclidean" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the test data as well as train, usually slower +generate_test_files="true" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Boolean on if to combine the test/train split +combine_test_train_split="false" + +# Normalise data before fit/predict +normalise_data="true" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -te to generate test files +generate_test_files=$([ "${generate_test_files,,}" == "true" ] && echo "-te" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Update result path to split combined test train split and test train split +results_dir="${results_dir}$([ "${combine_test_train_split,,}" == "true" ] && echo "combine-test-train-split/" || echo "test-train-split/")" + +# Update out path to split combined test train split and test train split +out_dir="${out_dir}$([ "${combine_test_train_split,,}" == "true" ] && echo "combine-test-train-split/" || echo "test-train-split/")" + +# Set to -utts to combine test train split +combine_test_train_split=$([ "${combine_test_train_split,,}" == "true" ] && echo "-ctts" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# dont submit to serial directly +queue=$([ "$queue" == "serial" ] && echo "batch" || echo "$queue") +queue_alias=$([ "$queue" == "batch" ] && echo "serial" || echo "$queue") + +count=0 +while read dataset; do +for clusterer in $clusterers_to_run; do + +# Skip to the script start point +((count++)) +if ((count>=start_point)); then + +# This is the loop to keep from dumping everything in the queue which is maintained around max_num_submitted jobs +num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) +while [ "${num_jobs}" -ge "${max_num_submitted}" ] +do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) +done + +mkdir -p "${out_dir}${clusterer}/${dataset}/" + +# This skips jobs which have test/train files already written to the results directory. Only looks for Resamples, not Folds (old file name) +array_jobs="" +for (( i=start_fold-1; i generatedFile.sub + +echo "${count} ${clusterer}/${dataset}" + +sbatch < generatedFile.sub + +else + echo "${count} ${clusterer}/${dataset}" has finished all required resamples, skipping +fi + +fi +done +done < ${datasets} + +echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_regression_experiments.sh b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_regression_experiments.sh new file mode 100644 index 00000000..e10b768f --- /dev/null +++ b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_regression_experiments.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=30 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=100 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of threads to request. Please check the number of cores for the node you are using, some are exclusive (i.e. batch) so you should request all cores or a taskfarm script +n_threads=10 + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto="$username@soton.ac.uk" + +# MB for jobs, increase incrementally and try not to use more than you need. If you need hundreds of GB consider the huge memory queue +max_memory=8000 + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 regressors = 9 jobs to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Default is Tony's work space, all should be able to read these. Change if you want to use different data or lists +data_dir="$local_path/Data/" +datasets="$local_path/DataSetLists/Regression.txt" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/RegressionResults/results/" +out_dir="$local_path/RegressionResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/threaded_regression_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="tsml-eval" + +# Regressors to loop over. Must be separated by a space +# See list of potential regressors in set_regressor +regressors_to_run="RocketRegressor TimeSeriesForestRegressor" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the train data as well as test, usually slower +generate_train_files="false" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Normalise data before fit/predict +normalise_data="false" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -tr to generate test files +generate_train_files=$([ "${generate_train_files,,}" == "true" ] && echo "-tr" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# dont submit to serial directly +queue=$([ "$queue" == "serial" ] && echo "batch" || echo "$queue") +queue_alias=$([ "$queue" == "batch" ] && echo "serial" || echo "$queue") + +count=0 +while read dataset; do +for regressor in $regressors_to_run; do + +# Skip to the script start point +((count++)) +if ((count>=start_point)); then + +# This is the loop to keep from dumping everything in the queue which is maintained around max_num_submitted jobs +num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) +while [ "${num_jobs}" -ge "${max_num_submitted}" ] +do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue_alias}" -e "PD ${queue_alias}" | wc -l) +done + +mkdir -p "${out_dir}${regressor}/${dataset}/" + +# This skips jobs which have test/train files already written to the results directory. Only looks for Resamples, not Folds (old file name) +array_jobs="" +for (( i=start_fold-1; i generatedFile.sub + +echo "${count} ${regressor}/${dataset}" + +sbatch < generatedFile.sub + +else + echo "${count} ${regressor}/${dataset}" has finished all required resamples, skipping +fi + +fi +done +done < ${datasets} + +echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_classification_experiments.sh b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_classification_experiments.sh new file mode 100644 index 00000000..14105f12 --- /dev/null +++ b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_classification_experiments.sh @@ -0,0 +1,223 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=10 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=900 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of tasks to submit in each job. This can be larger than the number of cores, but tasks will be delayed until a core is free +n_tasks_per_node=4 + +# The number of threads to use per task. You can only run as many tasks as there are CPUs available, 4 tasks with 10 threads will take up a full batch node +n_threads_per_task=10 + +# The number of cores to request from the node. Don't go over the number of cores for the node. 40 is the number of cores on batch nodes +# If you are not using the whole node, please make sure you are requesting memory correctly +max_cpus_to_use=40 + +# Create a separate submission list for each classifier. This will stop the mixing of +# large and small jobs in the same node, but results in some smaller scripts submitted +# to serial when moving between classifiers. +# For small workloads i.e. single resample 10 datasets, turning this off will be the only way to get on the batch queue realistically +split_classifiers="true" + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto=$username"@soton.ac.uk" + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 classifiers = 9 experiments to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Dataset list can either be a text file or directory of text files +# Separate text files will not run jobs of the same dataset in the same node. This is good to keep large and small datasets separate +data_dir="$local_path/Data/" +dataset_list="$local_path/DataSetLists/ClassificationBatch/" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/ClassificationResults/results/" +out_dir="$local_path/ClassificationResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/threaded_classification_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="eval-py11" + +# Classifiers to loop over. Must be separated by a space. Different classifiers will not run in the same node by default +# See list of potential classifiers in set_classifier +classifiers_to_run="ROCKET DrCIF" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the train data as well as test, usually slower +generate_train_files="false" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Normalise data before fit/predict +normalise_data="false" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -tr to generate test files +generate_train_files=$([ "${generate_train_files,,}" == "true" ] && echo "-tr" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# This creates the submission file to run and does clean up +submit_jobs () { + +totalThreads=$((cmdCount * n_threads_per_task)) +if ((totalJobs>=max_cpus_to_use)); then + cpuCount=$max_cpus_to_use +else + cpuCount=$totalThreads +fi + +echo "#!/bin/bash +#SBATCH --mail-type=${mail} +#SBATCH --mail-user=${mailto} +#SBATCH --job-name=batch-${dt} +#SBATCH -p ${queue} +#SBATCH -t ${max_time} +#SBATCH -o ${outDir}/%A-${dt}.out +#SBATCH -e ${outDir}/%A-${dt}.err +#SBATCH --nodes=1 +#SBATCH --ntasks=${cpuCount} + +. /etc/profile + +module load anaconda/py3.10 +source activate $env_name + +staskfarm ${outDir}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub + +echo "At experiment ${expCount}, ${totalCount} jobs submitted total" + +sbatch < generatedSubmissionFile-${dt}.sub + +rm generatedSubmissionFile-${dt}.sub + +} + +totalCount=0 +expCount=0 +dt=$(date +%Y%m%d%H%M%S) + +# turn a directory of files into a list +if [[ -d $dataset_list ]]; then + file_names="" + for file in ${dataset_list}/*; do + file_names="$file_names$dataset_list$(basename "$file") " + done + dataset_list=$file_names +fi + +for dataset_file in $dataset_list; do + +echo "Dataset list ${dataset_file}" + +for classifier in $classifiers_to_run; do + +mkdir -p "${out_dir}/${classifier}/" + +if [ "${split_classifiers,,}" == "true" ]; then + # we use time for unique names + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) + outDir=${out_dir}/${classifier} +else + outDir=${out_dir} +fi + +while read dataset; do + +# Skip to the script start point +((expCount++)) +if ((expCount>=start_point)); then + +# This finds the resamples to run and skips jobs which have test/train files already written to the results directory. +# This can result in uneven sized command lists +resamples_to_run="" +for (( i=start_fold-1; i=n_tasks_per_node)); then + submit_jobs + + # This is the loop to stop you from dumping everything in the queue at once, see max_num_submitted + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + while [ "${num_jobs}" -ge "${max_num_submitted}" ] + do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + done + + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) +fi + +# Input args to the default threaded_classification_experiments are in main method of +# https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/threaded_classification_experiments.py +echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} ${resample} -nj ${n_threads_per_task} ${generate_train_files} ${predefined_folds} ${normalise_data} > ${out_dir}/${classifier}/output-${dataset}-${resample}-${dt}.txt 2>&1" >> ${outDir}/generatedCommandList-${dt}.txt + +((cmdCount++)) +((totalCount++)) + +done +fi +done < ${dataset_file} + +if [[ "${split_classifiers,,}" == "true" && $cmdCount -gt 0 ]]; then + # final submit for this classifier + submit_jobs +fi + +done + +if [[ "${split_classifiers,,}" != "true" && $cmdCount -gt 0 ]]; then + # final submit for this dataset list + submit_jobs +fi + +done + +echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_clustering_experiments.sh b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_clustering_experiments.sh new file mode 100644 index 00000000..0c89dc62 --- /dev/null +++ b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_clustering_experiments.sh @@ -0,0 +1,235 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=10 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=900 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of tasks to submit in each job. This can be larger than the number of cores, but tasks will be delayed until a core is free +n_tasks_per_node=4 + +# The number of threads to use per task. You can only run as many tasks as there are CPUs available, 4 tasks with 10 threads will take up a full batch node +n_threads_per_task=10 + +# The number of cores to request from the node. Don't go over the number of cores for the node. 40 is the number of cores on batch nodes +# If you are not using the whole node, please make sure you are requesting memory correctly +max_cpus_to_use=40 + +# Create a separate submission list for each clusterer. This will stop the mixing of +# large and small jobs in the same node, but results in some smaller scripts submitted +# to serial when moving between clusterers. +# For small workloads i.e. single resample 10 datasets, turning this off will be the only way to get on the batch queue realistically +split_clusterers="true" + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto=$username"@soton.ac.uk" + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 clusterers = 9 experiments to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Dataset list can either be a text file or directory of text files +# Separate text files will not run jobs of the same dataset in the same node. This is good to keep large and small datasets separate +data_dir="$local_path/Data/" +dataset_list="$local_path/DataSetLists/ClusteringBatch/" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/ClusteringResults/results/" +out_dir="$local_path/ClusteringResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/threaded_clustering_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="eval-py11" + +# Clusterers to loop over. Must be separated by a space. Different clusterers will not run in the same node by default +# See list of potential clusterers in set_clusterer +clusterers_to_run="kmedoids-squared kmedoids-euclidean" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the test data as well as train, usually slower +generate_test_files="true" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Boolean on if to combine the test/train split +combine_test_train_split="false" + +# Normalise data before fit/predict +normalise_data="true" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -te to generate test files +generate_test_files=$([ "${generate_test_files,,}" == "true" ] && echo "-te" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Update result path to split combined test train split and test train split +results_dir="${results_dir}$([ "${combine_test_train_split,,}" == "true" ] && echo "combine-test-train-split/" || echo "test-train-split/")" + +# Update out path to split combined test train split and test train split +out_dir="${out_dir}$([ "${combine_test_train_split,,}" == "true" ] && echo "combine-test-train-split/" || echo "test-train-split/")" + +# Set to -utts to combine test train split +combine_test_train_split=$([ "${combine_test_train_split,,}" == "true" ] && echo "-ctts" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# This creates the submission file to run and does clean up +submit_jobs () { + +totalThreads=$((cmdCount * n_threads_per_task)) +if ((totalJobs>=max_cpus_to_use)); then + cpuCount=$max_cpus_to_use +else + cpuCount=$totalThreads +fi + +echo "#!/bin/bash +#SBATCH --mail-type=${mail} +#SBATCH --mail-user=${mailto} +#SBATCH --job-name=batch-${dt} +#SBATCH -p ${queue} +#SBATCH -t ${max_time} +#SBATCH -o ${outDir}/%A-${dt}.out +#SBATCH -e ${outDir}/%A-${dt}.err +#SBATCH --nodes=1 +#SBATCH --ntasks=${cpuCount} + +. /etc/profile + +module load anaconda/py3.10 +source activate $env_name + +staskfarm ${outDir}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub + +echo "At experiment ${expCount}, ${totalCount} jobs submitted total" + +sbatch < generatedSubmissionFile-${dt}.sub + +rm generatedSubmissionFile-${dt}.sub + +} + +totalCount=0 +expCount=0 +dt=$(date +%Y%m%d%H%M%S) + +# turn a directory of files into a list +if [[ -d $dataset_list ]]; then + file_names="" + for file in ${dataset_list}/*; do + file_names="$file_names$dataset_list$(basename "$file") " + done + dataset_list=$file_names +fi + +for dataset_file in $dataset_list; do + +echo "Dataset list ${dataset_file}" + +for clusterer in $clusterers_to_run; do + +mkdir -p "${out_dir}/${clusterer}/" + +if [ "${split_clusterers,,}" == "true" ]; then + # we use time for unique names + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) + outDir=${out_dir}/${clusterer} +else + outDir=${out_dir} +fi + +while read dataset; do + +# Skip to the script start point +((expCount++)) +if ((expCount>=start_point)); then + +# This finds the resamples to run and skips jobs which have test/train files already written to the results directory. +# This can result in uneven sized command lists +resamples_to_run="" +for (( i=start_fold-1; i=n_tasks_per_node)); then + submit_jobs + + # This is the loop to stop you from dumping everything in the queue at once, see max_num_submitted + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + while [ "${num_jobs}" -ge "${max_num_submitted}" ] + do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + done + + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) +fi + +# Input args to the default threaded_clustering_experiments are in main method of +# https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/threaded_clustering_experiments.py +echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${clusterer} ${dataset} ${resample} ${generate_test_files} ${predefined_folds} ${combine_test_train_split} ${normalise_data} > ${out_dir}/${clusterer}/output-${dataset}-${resample}-${dt}.txt 2>&1" >> ${outDir}/generatedCommandList-${dt}.txt + +((cmdCount++)) +((totalCount++)) + +done +fi +done < ${dataset_file} + +if [[ "${split_clusterers,,}" == "true" && $cmdCount -gt 0 ]]; then + # final submit for this clusterer + submit_jobs +fi + +done + +if [[ "${split_clusterers,,}" != "true" && $cmdCount -gt 0 ]]; then + # final submit for this dataset list + submit_jobs +fi + +done + +echo Finished submitting jobs diff --git a/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_regression_experiments.sh b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_regression_experiments.sh new file mode 100644 index 00000000..c1511f8c --- /dev/null +++ b/_tsml_research_resources/soton/iridis/threaded_scripts/threaded_taskfarm_regression_experiments.sh @@ -0,0 +1,223 @@ +#!/bin/bash +# Check and edit all options before the first run! +# While reading is fine, please dont write anything to the default directories in this script + +# Start and end for resamples +max_folds=10 +start_fold=1 + +# To avoid hitting the cluster queue limit we have a higher level queue +max_num_submitted=900 + +# Queue options are https://sotonac.sharepoint.com/teams/HPCCommunityWiki/SitePages/Iridis%205%20Job-submission-and-Limits-Quotas.aspx +queue="batch" + +# The number of tasks to submit in each job. This can be larger than the number of cores, but tasks will be delayed until a core is free +n_tasks_per_node=4 + +# The number of threads to use per task. You can only run as many tasks as there are CPUs available, 4 tasks with 10 threads will take up a full batch node +n_threads_per_task=10 + +# The number of cores to request from the node. Don't go over the number of cores for the node. 40 is the number of cores on batch nodes +# If you are not using the whole node, please make sure you are requesting memory correctly +max_cpus_to_use=40 + +# Create a separate submission list for each regressor. This will stop the mixing of +# large and small jobs in the same node, but results in some smaller scripts submitted +# to serial when moving between regressors. +# For small workloads i.e. single resample 10 datasets, turning this off will be the only way to get on the batch queue realistically +split_regressors="true" + +# Enter your username and email here +username="ajb2u23" +mail="NONE" +mailto=$username"@soton.ac.uk" + +# Max allowable is 60 hours +max_time="60:00:00" + +# Start point for the script i.e. 3 datasets, 3 regressors = 9 experiments to submit, start_point=5 will skip to job 5 +start_point=1 + +# Put your home directory here +local_path="/mainfs/home/$username/" + +# Datasets to use and directory of data files. Dataset list can either be a text file or directory of text files +# Separate text files will not run jobs of the same dataset in the same node. This is good to keep large and small datasets separate +data_dir="$local_path/Data/" +dataset_list="$local_path/DataSetLists/RegressionBatch/" + +# Results and output file write location. Change these to reflect your own file structure +results_dir="$local_path/RegressionResults/results/" +out_dir="$local_path/RegressionResults/output/" + +# The python script we are running +script_file_path="$local_path/tsml-eval/tsml_eval/experiments/threaded_regression_experiments.py" + +# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md +# Separate environments for GPU and CPU are recommended +env_name="eval-py11" + +# Regressors to loop over. Must be separated by a space. Different regressors will not run in the same node by default +# See list of potential regressors in set_regressor +regressors_to_run="ROCKET DrCIF" + +# You can add extra arguments here. See tsml_eval/utils/arguments.py parse_args +# You will have to add any variable to the python call close to the bottom of the script +# and possibly to the options handling below + +# generate a results file for the train data as well as test, usually slower +generate_train_files="false" + +# If set for true, looks for _TRAIN.ts file. This is useful for running tsml-java resamples +predefined_folds="false" + +# Normalise data before fit/predict +normalise_data="false" + +# ====================================================================================== +# Experiment configuration end +# ====================================================================================== + +# Set to -tr to generate test files +generate_train_files=$([ "${generate_train_files,,}" == "true" ] && echo "-tr" || echo "") + +# Set to -pr to use predefined folds +predefined_folds=$([ "${predefined_folds,,}" == "true" ] && echo "-pr" || echo "") + +# Set to -rn to normalise data +normalise_data=$([ "${normalise_data,,}" == "true" ] && echo "-rn" || echo "") + +# This creates the submission file to run and does clean up +submit_jobs () { + +totalThreads=$((cmdCount * n_threads_per_task)) +if ((totalJobs>=max_cpus_to_use)); then + cpuCount=$max_cpus_to_use +else + cpuCount=$totalThreads +fi + +echo "#!/bin/bash +#SBATCH --mail-type=${mail} +#SBATCH --mail-user=${mailto} +#SBATCH --job-name=batch-${dt} +#SBATCH -p ${queue} +#SBATCH -t ${max_time} +#SBATCH -o ${outDir}/%A-${dt}.out +#SBATCH -e ${outDir}/%A-${dt}.err +#SBATCH --nodes=1 +#SBATCH --ntasks=${cpuCount} + +. /etc/profile + +module load anaconda/py3.10 +source activate $env_name + +staskfarm ${outDir}/generatedCommandList-${dt}.txt" > generatedSubmissionFile-${dt}.sub + +echo "At experiment ${expCount}, ${totalCount} jobs submitted total" + +sbatch < generatedSubmissionFile-${dt}.sub + +rm generatedSubmissionFile-${dt}.sub + +} + +totalCount=0 +expCount=0 +dt=$(date +%Y%m%d%H%M%S) + +# turn a directory of files into a list +if [[ -d $dataset_list ]]; then + file_names="" + for file in ${dataset_list}/*; do + file_names="$file_names$dataset_list$(basename "$file") " + done + dataset_list=$file_names +fi + +for dataset_file in $dataset_list; do + +echo "Dataset list ${dataset_file}" + +for regressor in $regressors_to_run; do + +mkdir -p "${out_dir}/${regressor}/" + +if [ "${split_regressors,,}" == "true" ]; then + # we use time for unique names + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) + outDir=${out_dir}/${regressor} +else + outDir=${out_dir} +fi + +while read dataset; do + +# Skip to the script start point +((expCount++)) +if ((expCount>=start_point)); then + +# This finds the resamples to run and skips jobs which have test/train files already written to the results directory. +# This can result in uneven sized command lists +resamples_to_run="" +for (( i=start_fold-1; i=n_tasks_per_node)); then + submit_jobs + + # This is the loop to stop you from dumping everything in the queue at once, see max_num_submitted + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + while [ "${num_jobs}" -ge "${max_num_submitted}" ] + do + echo Waiting 60s, ${num_jobs} currently submitted on ${queue}, user-defined max is ${max_num_submitted} + sleep 60 + num_jobs=$(squeue -u ${username} --format="%20P %5t" -r | awk '{print $2, $1}' | grep -e "R ${queue}" -e "PD ${queue}" | wc -l) + done + + sleep 1 + cmdCount=0 + dt=$(date +%Y%m%d%H%M%S) +fi + +# Input args to the default threaded_regression_experiments are in main method of +# https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/threaded_regression_experiments.py +echo "python -u ${script_file_path} ${data_dir} ${results_dir} ${regressor} ${dataset} ${resample} ${generate_train_files} ${predefined_folds} ${normalise_data} > ${out_dir}/${regressor}/output-${dataset}-${resample}-${dt}.txt 2>&1" >> ${outDir}/generatedCommandList-${dt}.txt + +((cmdCount++)) +((totalCount++)) + +done +fi +done < ${dataset_file} + +if [[ "${split_regressors,,}" == "true" && $cmdCount -gt 0 ]]; then + # final submit for this regressor + submit_jobs +fi + +done + +if [[ "${split_regressors,,}" != "true" && $cmdCount -gt 0 ]]; then + # final submit for this dataset list + submit_jobs +fi + +done + +echo Finished submitting jobs diff --git a/_tsml_research_resources/uea/ada/classification_experiments.sh b/_tsml_research_resources/uea/ada/classification_experiments.sh index 3ce96c74..00bab3a1 100644 --- a/_tsml_research_resources/uea/ada/classification_experiments.sh +++ b/_tsml_research_resources/uea/ada/classification_experiments.sh @@ -98,7 +98,7 @@ array_jobs="" for (( i=start_fold-1; i=1.0.0,<1.3.0", + "aeon>=1.0.0,<1.4.0", "tsml>=0.6.0,<0.8.0", "scikit-learn>=1.0.0,<1.7.0", "matplotlib", @@ -84,6 +84,7 @@ docs = [ "sphinx_issues", "sphinx-copybutton", "sphinx-remove-toctrees", + "sphinx-reredirects", "sphinxext-opengraph", "nbsphinx", "numpydoc", diff --git a/tsml_eval/evaluation/multiple_estimator_evaluation.py b/tsml_eval/evaluation/multiple_estimator_evaluation.py index fe5f4274..1b5945c1 100644 --- a/tsml_eval/evaluation/multiple_estimator_evaluation.py +++ b/tsml_eval/evaluation/multiple_estimator_evaluation.py @@ -1329,11 +1329,11 @@ def _figures_for_statistic( df.columns = estimators mcm = create_multi_comparison_matrix( df, - output_dir=f"{save_path}/{statistic_name}/figures/", - pdf_savename=f"{eval_name}_{statistic_name.lower()}_mcm", + save_path=f"{save_path}/{statistic_name}/figures/{eval_name}" + f"_{statistic_name.lower()}_mcm", + formats="pdf", show_symetry=True, - order_win_tie_loss="higher" if higher_better else "lower", - order_better="decreasing" if higher_better else "increasing", + higher_stat_better=higher_better, used_statistic=statistic_name, ) pickle.dump( diff --git a/tsml_eval/experiments/_get_classifier.py b/tsml_eval/experiments/_get_classifier.py index 0f656e9d..cbfad5bd 100644 --- a/tsml_eval/experiments/_get_classifier.py +++ b/tsml_eval/experiments/_get_classifier.py @@ -776,16 +776,13 @@ def _set_classifier_interval_based( elif c == "drcif-pipeline": import numpy as np - from aeon.transformations.collection.feature_based import Catch22 - from sklearn.ensemble import ExtraTreesClassifier - from tsml.interval_based import RandomIntervalClassifier - from tsml.transformations import ( + from aeon.transformations.collection import ( ARCoefficientTransformer, - FunctionTransformer, PeriodogramTransformer, ) - from tsml.utils.numba_functions.general import first_order_differences_3d - from tsml.utils.numba_functions.stats import ( + from aeon.transformations.collection.feature_based import Catch22 + from aeon.utils.numba.general import first_order_differences_3d + from aeon.utils.numba.stats import ( row_iqr, row_mean, row_median, @@ -795,6 +792,9 @@ def _set_classifier_interval_based( row_slope, row_std, ) + from sklearn.ensemble import ExtraTreesClassifier + from tsml.interval_based import RandomIntervalClassifier + from tsml.transformations import FunctionTransformer def sqrt_times_15_plus_5_mv(X): return int( @@ -815,7 +815,7 @@ def sqrt_times_15_plus_5_mv(X): series_transformers = [ None, FunctionTransformer(func=first_order_differences_3d, validate=False), - PeriodogramTransformer(use_pyfftw=True), + PeriodogramTransformer(), ARCoefficientTransformer(replace_nan=True), ] @@ -901,11 +901,11 @@ def _set_classifier_shapelet_based( elif c == "sastclassifier" or c == "sast": from aeon.classification.shapelet_based import SASTClassifier - return SASTClassifier(seed=random_state, n_jobs=n_jobs, **kwargs) + return SASTClassifier(random_state=random_state, n_jobs=n_jobs, **kwargs) elif c == "rsastclassifier" or c == "rsast": from aeon.classification.shapelet_based import RSASTClassifier - return RSASTClassifier(seed=random_state, n_jobs=n_jobs, **kwargs) + return RSASTClassifier(random_state=random_state, n_jobs=n_jobs, **kwargs) elif c == "learningshapeletclassifier" or c == "ls": from aeon.classification.shapelet_based import LearningShapeletClassifier diff --git a/tsml_eval/experiments/_get_data_transform.py b/tsml_eval/experiments/_get_data_transform.py index 389bf2f7..1aefdb57 100644 --- a/tsml_eval/experiments/_get_data_transform.py +++ b/tsml_eval/experiments/_get_data_transform.py @@ -18,14 +18,17 @@ ] unequal_transformers = [ ["padder", "zero-padder"], + "zero-padder-min", "mean-padder", - "zero-noise-padder", + "mean-padder-min", + ["noise-padder", "zero-noise-padder"], "zero-noise-padder-min", "mean-noise-padder", "mean-noise-padder-min", ["truncator", "truncate"], "truncate-max", "resizer", + "resizer-min", ] @@ -90,51 +93,73 @@ def _set_scaling_transformer(t, random_state, n_jobs): def _set_unequal_transformer(t, random_state, n_jobs): if t == "padder" or t == "zero-padder": - from aeon.transformations.collection import Padder + from aeon.transformations.collection.unequal_length import Padder return Padder() + elif t == "zero-padder-min": + from aeon.transformations.collection.unequal_length import Padder + + return Padder( + padded_length="min", + error_on_long=False, + random_state=random_state, + ) elif t == "mean-padder": - from tsml_eval._wip.unequal_length._pad import Padder + from aeon.transformations.collection.unequal_length import Padder return Padder(fill_value="mean", random_state=random_state) - elif t == "zero-noise-padder": - from tsml_eval._wip.unequal_length._pad import Padder + elif t == "mean-padder-min": + from aeon.transformations.collection.unequal_length import Padder + + return Padder( + padded_length="min", + fill_value="mean", + error_on_long=False, + random_state=random_state, + ) + elif t == "noise-padder" or t == "zero-noise-padder": + from aeon.transformations.collection.unequal_length import Padder return Padder(add_noise=0.001, random_state=random_state) elif t == "zero-noise-padder-min": - from tsml_eval._wip.unequal_length._pad import Padder + from aeon.transformations.collection.unequal_length import Padder return Padder( - pad_length="min", + padded_length="min", add_noise=0.001, error_on_long=False, random_state=random_state, ) elif t == "mean-noise-padder": - from tsml_eval._wip.unequal_length._pad import Padder + from aeon.transformations.collection.unequal_length import Padder return Padder(fill_value="mean", add_noise=0.001, random_state=random_state) elif t == "mean-noise-padder-min": - from tsml_eval._wip.unequal_length._pad import Padder + from aeon.transformations.collection.unequal_length import Padder return Padder( + padded_length="min", fill_value="mean", add_noise=0.001, error_on_long=False, random_state=random_state, ) elif t == "truncator" or t == "truncate": - from tsml_eval._wip.unequal_length._truncate import Truncator + from aeon.transformations.collection.unequal_length import Truncator return Truncator() elif t == "truncate-max": - from tsml_eval._wip.unequal_length._truncate import Truncator + from aeon.transformations.collection.unequal_length import Truncator return Truncator(truncated_length="max", error_on_short=False) elif t == "resizer": - from tsml_eval._wip.unequal_length._resize import Resizer + from aeon.transformations.collection.unequal_length import Resizer return Resizer() + elif t == "resizer-min": + from aeon.transformations.collection.unequal_length import Resizer + + return Resizer(resized_length="min") def _set_unbalanced_transformer(t, random_state, n_jobs): diff --git a/tsml_eval/experiments/_get_forecaster.py b/tsml_eval/experiments/_get_forecaster.py index da797e06..4f68ec75 100644 --- a/tsml_eval/experiments/_get_forecaster.py +++ b/tsml_eval/experiments/_get_forecaster.py @@ -2,12 +2,27 @@ __maintainer__ = ["MatthewMiddlehurst"] -from aeon.forecasting import ETSForecaster, NaiveForecaster - from tsml_eval.utils.functions import str_in_nested_list +deep_forecasters = [ + ["tcnforecaster", "tcn"], +] +ml_forecasters = [ + "setartree", + "setarforest", +] stats_forecasters = [ + ["arimaforecaster", "arima"], + "autoarima", ["etsforecaster", "ets"], + ["tarforecaster", "tar"], + "autotar", + ["setarforecaster", "setar"], + ["thetaforecaster", "theta"], + ["tvpforecaster", "tvp"], +] +regression_forecasters = [ + "randomforest", ] other_forecasters = [ ["naiveforecaster", "naive"], @@ -43,21 +58,84 @@ def get_forecaster_by_name(forecaster_name, random_state=None, n_jobs=1, **kwarg """ f = forecaster_name.lower() - if str_in_nested_list(stats_forecasters, f): + if str_in_nested_list(deep_forecasters, f): + return _set_forecaster_deep(f, random_state, n_jobs, kwargs) + elif str_in_nested_list(ml_forecasters, f): + return _set_forecaster_ml(f, random_state, n_jobs, kwargs) + elif str_in_nested_list(stats_forecasters, f): return _set_forecaster_stats(f, random_state, n_jobs, kwargs) + elif str_in_nested_list(regression_forecasters, f): + return _set_forecaster_regression(f, random_state, n_jobs, kwargs) elif str_in_nested_list(other_forecasters, f): return _set_forecaster_other(f, random_state, n_jobs, kwargs) else: raise ValueError(f"UNKNOWN FORECASTER: {f} in get_forecaster_by_name") +def _set_forecaster_deep(f, random_state, n_jobs, kwargs): + if f == "tcnforecaster" or f == "tcn": + from aeon.forecasting.deep_learning import TCNForecaster + + return TCNForecaster(random_state=random_state, **kwargs) + + +def _set_forecaster_ml(f, random_state, n_jobs, kwargs): + if f == "setartree": + from aeon.forecasting.machine_learning import SETARTree + + return SETARTree(**kwargs) + elif f == "setarforest": + from aeon.forecasting.machine_learning import SETARForest + + return SETARForest(random_state=random_state, **kwargs) + + def _set_forecaster_stats(f, random_state, n_jobs, kwargs): - if f == "etsforecaster" or f == "ets": - return ETSForecaster(**kwargs) + if f == "arimaforecaster" or f == "arima": + from aeon.forecasting.stats import ARIMA - # todo + return ARIMA(**kwargs) + elif f == "autoarima": + from aeon.forecasting.stats import AutoARIMA + + return AutoARIMA(**kwargs) + elif f == "etsforecaster" or f == "ets": + from aeon.forecasting.stats import ETS + + return ETS(**kwargs) + elif f == "tarforecaster" or f == "tar": + from aeon.forecasting.stats import TAR + + return TAR(**kwargs) + elif f == "autotar": + from aeon.forecasting.stats import AutoTAR + + return AutoTAR(**kwargs) + elif f == "setarforecaster" or f == "setar": + from aeon.forecasting.machine_learning import SETAR + + return SETAR(**kwargs) + elif f == "thetaforecaster" or f == "theta": + from aeon.forecasting.stats import Theta + + return Theta(**kwargs) + elif f == "tvpforecaster" or f == "tvp": + from aeon.forecasting.stats import TVP + + return TVP(**kwargs) + + +def _set_forecaster_regression(f, random_state, n_jobs, kwargs): + if f == "randomforest": + from aeon.forecasting import RegressionForecaster + from sklearn.ensemble import RandomForestRegressor + + reg = RandomForestRegressor(random_state=random_state, n_jobs=n_jobs, **kwargs) + return RegressionForecaster(10, regressor=reg) def _set_forecaster_other(f, random_state, n_jobs, kwargs): - if f == "dummyforecaster" or f == "dummy": + if f == "naiveforecaster" or f == "naive": + from aeon.forecasting import NaiveForecaster + return NaiveForecaster(**kwargs) diff --git a/tsml_eval/experiments/tests/test_data_transform.py b/tsml_eval/experiments/tests/test_data_transform.py index 16e2b701..56f97349 100644 --- a/tsml_eval/experiments/tests/test_data_transform.py +++ b/tsml_eval/experiments/tests/test_data_transform.py @@ -1,7 +1,8 @@ """Tests for data transforms in experiments.""" import pytest -from aeon.transformations.collection import Normalizer, Padder +from aeon.transformations.collection import Normalizer +from aeon.transformations.collection.unequal_length import Padder from tsml_eval.experiments import _get_data_transform, get_data_transform_by_name from tsml_eval.testing.testing_utils import _check_set_method, _check_set_method_results diff --git a/tsml_eval/experiments/tests/test_regression.py b/tsml_eval/experiments/tests/test_regression.py index 9ce61e5b..56135720 100644 --- a/tsml_eval/experiments/tests/test_regression.py +++ b/tsml_eval/experiments/tests/test_regression.py @@ -214,6 +214,7 @@ def test_aeon_regressors_available(): "SklearnRegressorWrapper", "IntervalForestRegressor", # just missing + "RecurrentRegressor", ] est = [e for e, _ in all_estimators(type_filter="regressor")] diff --git a/tsml_eval/publications/y2023/distance_based_clustering/run_distance_experiments.py b/tsml_eval/publications/y2023/distance_based_clustering/run_distance_experiments.py index e497df23..ca98e3d3 100644 --- a/tsml_eval/publications/y2023/distance_based_clustering/run_distance_experiments.py +++ b/tsml_eval/publications/y2023/distance_based_clustering/run_distance_experiments.py @@ -5,13 +5,12 @@ import os import sys -from aeon.transformations.collection import Normalizer - os.environ["MKL_NUM_THREADS"] = "1" # must be done before numpy import!! os.environ["NUMEXPR_NUM_THREADS"] = "1" # must be done before numpy import!! os.environ["OMP_NUM_THREADS"] = "1" # must be done before numpy import!! -from tsml.base import _clone_estimator +from aeon.base._base import _clone_estimator +from aeon.transformations.collection import Normalizer from tsml_eval.experiments import load_and_run_clustering_experiment from tsml_eval.publications.y2023.distance_based_clustering.set_distance_clusterer import ( # noqa: E501 @@ -102,12 +101,7 @@ def _run_experiment(args): cnl = clusterer.lower() if cnl.find("kmeans") or cnl.find("k-means"): kwargs["averaging_method"] = "mean" - average_params = { - **distance_params, - "averaging_distance_metric": distance, - "medoids_distance_metric": distance, - } - kwargs["average_params"] = average_params + kwargs["average_params"] = {} # Skip if not overwrite and results already present # this is also checked in load_and_run, but doing a quick check here so can From f3440fb1957165d3f761c9fbb9a18930e7f83ecc Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 17:46:39 +0100 Subject: [PATCH 04/17] dep --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b6eddb69..8d59c657 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ all_extras = [ "aeon[all_extras]", "tsml[all_extras]", "xgboost", + "wildboard<=1.2", # latest version puts a cap on scikit-learn ] unstable_extras = [ "aeon[unstable_extras]", From 821f24436cbd4750ea93fb4fecd3f11f03441976 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 17:48:48 +0100 Subject: [PATCH 05/17] correct dep --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8d59c657..6da31863 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ all_extras = [ "aeon[all_extras]", "tsml[all_extras]", "xgboost", - "wildboard<=1.2", # latest version puts a cap on scikit-learn + "wildboar<=1.2.0", # latest version puts a cap on scikit-learn ] unstable_extras = [ "aeon[unstable_extras]", From 3e83966bab676fc874eb752c26d25e2eb0f3ffd8 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 19:10:55 +0100 Subject: [PATCH 06/17] fixes --- tsml_eval/estimators/clustering/_sklearn_clusterer.py | 7 ++++--- .../publications/y2023/rist_pipeline/rist_pipeline.ipynb | 6 ++---- .../tser_archive_expansion/tser_archive_expansion.ipynb | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tsml_eval/estimators/clustering/_sklearn_clusterer.py b/tsml_eval/estimators/clustering/_sklearn_clusterer.py index c0327139..39a59e25 100644 --- a/tsml_eval/estimators/clustering/_sklearn_clusterer.py +++ b/tsml_eval/estimators/clustering/_sklearn_clusterer.py @@ -6,7 +6,7 @@ import numpy as np from aeon.base._base import _clone_estimator from sklearn.base import ClusterMixin -from sklearn.utils.validation import check_is_fitted +from sklearn.utils.validation import check_is_fitted, validate_data from tsml.base import BaseTimeSeriesEstimator @@ -34,7 +34,8 @@ def fit(self, X, y=None): if self.clusterer is None: raise ValueError("Clusterer not set") - X = self._validate_data( + X = validate_data( + self, X=X, ensure_univariate=not self.concatenate_channels, ensure_equal_length=not self.pad_unequal, @@ -60,7 +61,7 @@ def predict(self, X) -> np.ndarray: """Wrap predict.""" check_is_fitted(self) - X = self._validate_data(X=X, reset=False) + X = validate_data(self, X=X, reset=False) X = self._convert_X( X, pad_unequal=self.pad_unequal, diff --git a/tsml_eval/publications/y2023/rist_pipeline/rist_pipeline.ipynb b/tsml_eval/publications/y2023/rist_pipeline/rist_pipeline.ipynb index 122a7b3b..13b9bd9a 100644 --- a/tsml_eval/publications/y2023/rist_pipeline/rist_pipeline.ipynb +++ b/tsml_eval/publications/y2023/rist_pipeline/rist_pipeline.ipynb @@ -82,7 +82,7 @@ "\n", "from aeon.classification.hybrid import RISTClassifier\n", "from aeon.regression.hybrid import RISTRegressor\n", - "from sklearn.metrics import accuracy_score, mean_squared_error\n", + "from sklearn.metrics import accuracy_score, root_mean_squared_error\n", "from tsml.datasets import load_minimal_chinatown, load_minimal_gas_prices" ], "metadata": { @@ -254,9 +254,7 @@ "output_type": "execute_result" } ], - "source": [ - "mean_squared_error(y_test_r, y_pred_r, squared=False)" - ], + "source": "root_mean_squared_error(y_test_r, y_pred_r)", "metadata": { "collapsed": false, "ExecuteTime": { diff --git a/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb b/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb index 541576b1..4012d345 100644 --- a/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb +++ b/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb @@ -64,7 +64,7 @@ "cell_type": "code", "source": [ "from aeon.regression.convolution_based import RocketRegressor\n", - "from sklearn.metrics import mean_squared_error\n", + "from sklearn.metrics import root_mean_squared_error\n", "from tsml.datasets import load_minimal_gas_prices\n", "\n", "from tsml_eval.estimators import SklearnToTsmlRegressor\n", @@ -225,7 +225,7 @@ " # fit and predict\n", " regressor.fit(X_train, y_train)\n", " y_pred = regressor.predict(X_test)\n", - " rmse.append(mean_squared_error(y_test, y_pred, squared=False))\n", + " rmse.append(root_mean_squared_error(y_test, y_pred))\n", "\n", "rmse" ], From 5c29292cd4bfd9c300387435d58f7756f70b9930 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 19:30:36 +0100 Subject: [PATCH 07/17] revert --- docs/conf.py | 3 +++ tsml_eval/estimators/clustering/_sklearn_clusterer.py | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0e630389..52d199f3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -105,6 +105,9 @@ remove_from_toctrees = ["auto_generated/*"] +# sphinx-reredirects +redirects = {"": ""} + # nbsphinx nbsphinx_execute = "never" diff --git a/tsml_eval/estimators/clustering/_sklearn_clusterer.py b/tsml_eval/estimators/clustering/_sklearn_clusterer.py index 39a59e25..c0327139 100644 --- a/tsml_eval/estimators/clustering/_sklearn_clusterer.py +++ b/tsml_eval/estimators/clustering/_sklearn_clusterer.py @@ -6,7 +6,7 @@ import numpy as np from aeon.base._base import _clone_estimator from sklearn.base import ClusterMixin -from sklearn.utils.validation import check_is_fitted, validate_data +from sklearn.utils.validation import check_is_fitted from tsml.base import BaseTimeSeriesEstimator @@ -34,8 +34,7 @@ def fit(self, X, y=None): if self.clusterer is None: raise ValueError("Clusterer not set") - X = validate_data( - self, + X = self._validate_data( X=X, ensure_univariate=not self.concatenate_channels, ensure_equal_length=not self.pad_unequal, @@ -61,7 +60,7 @@ def predict(self, X) -> np.ndarray: """Wrap predict.""" check_is_fitted(self) - X = validate_data(self, X=X, reset=False) + X = self._validate_data(X=X, reset=False) X = self._convert_X( X, pad_unequal=self.pad_unequal, From 4f6282528cd073dc7ab0b160bd94ef4a5528d649 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 19:52:29 +0100 Subject: [PATCH 08/17] more fixes --- docs/conf.py | 4 ++-- tsml_eval/estimators/clustering/consensus/ivc.py | 6 +++--- tsml_eval/estimators/clustering/consensus/ivc_from_file.py | 5 +++-- tsml_eval/estimators/clustering/consensus/simple_vote.py | 6 +++--- .../clustering/consensus/simple_vote_from_file.py | 5 +++-- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 52d199f3..e11e8cf1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,9 +55,9 @@ "sphinx_issues", "sphinx_copybutton", "sphinx_remove_toctrees", + "sphinx_reredirects", "versionwarning.extension", "myst_parser", - "sphinx-reredirects", ] templates_path = ["_templates"] @@ -106,7 +106,7 @@ remove_from_toctrees = ["auto_generated/*"] # sphinx-reredirects -redirects = {"": ""} +redirects = {"redirect": "index"} # nbsphinx diff --git a/tsml_eval/estimators/clustering/consensus/ivc.py b/tsml_eval/estimators/clustering/consensus/ivc.py index 6ba113c8..ec83121f 100644 --- a/tsml_eval/estimators/clustering/consensus/ivc.py +++ b/tsml_eval/estimators/clustering/consensus/ivc.py @@ -7,7 +7,7 @@ from sklearn.base import BaseEstimator, ClusterMixin from sklearn.cluster import KMeans from sklearn.utils import check_random_state -from sklearn.utils.validation import check_is_fitted +from sklearn.utils.validation import check_is_fitted, validate_data class IterativeVotingClustering(ClusterMixin, BaseEstimator): @@ -82,7 +82,7 @@ def fit(self, X, y=None): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, ensure_min_samples=self.n_clusters) + X = validate_data(self, X=X, ensure_min_samples=self.n_clusters) rng = check_random_state(self.random_state) @@ -137,7 +137,7 @@ def predict(self, X): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, reset=False) + X = validate_data(self, X=X, reset=False) cluster_assignments = np.zeros( (len(self._clusterers), X.shape[0]), dtype=np.int32 diff --git a/tsml_eval/estimators/clustering/consensus/ivc_from_file.py b/tsml_eval/estimators/clustering/consensus/ivc_from_file.py index a10076e1..64259e7b 100644 --- a/tsml_eval/estimators/clustering/consensus/ivc_from_file.py +++ b/tsml_eval/estimators/clustering/consensus/ivc_from_file.py @@ -4,6 +4,7 @@ import pandas as pd from sklearn import preprocessing from sklearn.utils import check_random_state +from sklearn.utils.validation import validate_data from tsml_eval.estimators.clustering.consensus.ivc import IterativeVotingClustering @@ -79,7 +80,7 @@ def fit(self, X, y=None): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, ensure_min_samples=self.n_clusters) + X = validate_data(self, X=X, ensure_min_samples=self.n_clusters) # load train file at path (trainResample.csv if random_state is None, # trainResample{self.random_state}.csv otherwise) @@ -153,7 +154,7 @@ def predict(self, X): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, reset=False) + X = validate_data(self, X=X, reset=False) # load train file at path (trainResample.csv if random_state is None, # trainResample{self.random_state}.csv otherwise) diff --git a/tsml_eval/estimators/clustering/consensus/simple_vote.py b/tsml_eval/estimators/clustering/consensus/simple_vote.py index 7b106a25..282841cc 100644 --- a/tsml_eval/estimators/clustering/consensus/simple_vote.py +++ b/tsml_eval/estimators/clustering/consensus/simple_vote.py @@ -7,7 +7,7 @@ from sklearn.base import BaseEstimator, ClusterMixin from sklearn.cluster import KMeans from sklearn.utils import check_random_state -from sklearn.utils.validation import check_is_fitted +from sklearn.utils.validation import check_is_fitted, validate_data class SimpleVote(ClusterMixin, BaseEstimator): @@ -63,7 +63,7 @@ def fit(self, X, y=None): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, ensure_min_samples=self.n_clusters) + X = validate_data(self, X=X, ensure_min_samples=self.n_clusters) rng = check_random_state(self.random_state) @@ -134,7 +134,7 @@ def predict_proba(self, X): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, reset=False) + X = validate_data(self, X=X, reset=False) cluster_assignments = np.zeros( (len(self._clusterers), X.shape[0]), dtype=np.int32 diff --git a/tsml_eval/estimators/clustering/consensus/simple_vote_from_file.py b/tsml_eval/estimators/clustering/consensus/simple_vote_from_file.py index 9af2036c..116d18d3 100644 --- a/tsml_eval/estimators/clustering/consensus/simple_vote_from_file.py +++ b/tsml_eval/estimators/clustering/consensus/simple_vote_from_file.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd from sklearn import preprocessing +from sklearn.utils.validation import validate_data from tsml_eval.estimators.clustering.consensus.simple_vote import SimpleVote @@ -61,7 +62,7 @@ def fit(self, X, y=None): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, ensure_min_samples=self.n_clusters) + X = validate_data(self, X=X, ensure_min_samples=self.n_clusters) # load train file at path (trainResample.csv if random_state is None, # trainResample{self.random_state}.csv otherwise) @@ -141,7 +142,7 @@ def predict_proba(self, X): "A valid sklearn input such as a 2d numpy array is required." "Sparse input formats are currently not supported." ) - X = self._validate_data(X=X, reset=False) + X = validate_data(self, X=X, reset=False) # load train file at path (trainResample.csv if random_state is None, # trainResample{self.random_state}.csv otherwise) From 675f288c4da9e7abd1428270478c327e8d7e393f Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 21:32:51 +0100 Subject: [PATCH 09/17] tsml update --- pyproject.toml | 3 +- tsml_eval/estimators/__init__.py | 12 --- .../estimators/classification/__init__.py | 8 -- .../classification/_sklearn_classifier.py | 93 ------------------- tsml_eval/estimators/clustering/__init__.py | 6 -- .../clustering/_sklearn_clusterer.py | 77 --------------- tsml_eval/estimators/regression/__init__.py | 7 -- .../regression/_sklearn_regressor.py | 76 --------------- tsml_eval/estimators/tests/__init__.py | 1 - tsml_eval/estimators/tests/test_estimators.py | 23 ----- tsml_eval/experiments/experiments.py | 6 +- 11 files changed, 4 insertions(+), 308 deletions(-) delete mode 100644 tsml_eval/estimators/classification/_sklearn_classifier.py delete mode 100644 tsml_eval/estimators/clustering/_sklearn_clusterer.py delete mode 100644 tsml_eval/estimators/regression/__init__.py delete mode 100644 tsml_eval/estimators/regression/_sklearn_regressor.py delete mode 100644 tsml_eval/estimators/tests/__init__.py delete mode 100644 tsml_eval/estimators/tests/test_estimators.py diff --git a/pyproject.toml b/pyproject.toml index 6da31863..9400f7ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ classifiers = [ requires-python = ">=3.10,<3.14" dependencies = [ "aeon>=1.0.0,<1.4.0", - "tsml>=0.6.0,<0.8.0", + "tsml>=0.6.0,<0.9.0", "scikit-learn>=1.0.0,<1.7.0", "matplotlib", "seaborn", @@ -55,7 +55,6 @@ all_extras = [ "aeon[all_extras]", "tsml[all_extras]", "xgboost", - "wildboar<=1.2.0", # latest version puts a cap on scikit-learn ] unstable_extras = [ "aeon[unstable_extras]", diff --git a/tsml_eval/estimators/__init__.py b/tsml_eval/estimators/__init__.py index b170881a..9a1dc001 100644 --- a/tsml_eval/estimators/__init__.py +++ b/tsml_eval/estimators/__init__.py @@ -1,13 +1 @@ """Estimators that are not present in tsml or aeon.""" - -__all__ = [ - "SklearnToTsmlClassifier", - "SklearnToTsmlClusterer", - "SklearnToTsmlRegressor", -] - -from tsml_eval.estimators.classification._sklearn_classifier import ( - SklearnToTsmlClassifier, -) -from tsml_eval.estimators.clustering._sklearn_clusterer import SklearnToTsmlClusterer -from tsml_eval.estimators.regression._sklearn_regressor import SklearnToTsmlRegressor diff --git a/tsml_eval/estimators/classification/__init__.py b/tsml_eval/estimators/classification/__init__.py index b782f488..8ef434d9 100644 --- a/tsml_eval/estimators/classification/__init__.py +++ b/tsml_eval/estimators/classification/__init__.py @@ -1,9 +1 @@ """Classification estimators.""" - -__all__ = [ - "SklearnToTsmlClassifier", -] - -from tsml_eval.estimators.classification._sklearn_classifier import ( - SklearnToTsmlClassifier, -) diff --git a/tsml_eval/estimators/classification/_sklearn_classifier.py b/tsml_eval/estimators/classification/_sklearn_classifier.py deleted file mode 100644 index 8eeadfe1..00000000 --- a/tsml_eval/estimators/classification/_sklearn_classifier.py +++ /dev/null @@ -1,93 +0,0 @@ -"""A tsml wrapper for sklearn classifiers.""" - -__maintainer__ = ["MatthewMiddlehurst"] -__all__ = ["SklearnToTsmlClassifier"] - -import numpy as np -from aeon.base._base import _clone_estimator -from sklearn.base import ClassifierMixin -from sklearn.utils.multiclass import check_classification_targets -from sklearn.utils.validation import check_is_fitted -from tsml.base import BaseTimeSeriesEstimator - - -class SklearnToTsmlClassifier(ClassifierMixin, BaseTimeSeriesEstimator): - """Wrapper for sklearn estimators to use the tsml base class.""" - - def __init__( - self, - classifier=None, - pad_unequal=False, - concatenate_channels=False, - clone_estimator=True, - random_state=None, - ): - self.classifier = classifier - self.pad_unequal = pad_unequal - self.concatenate_channels = concatenate_channels - self.clone_estimator = clone_estimator - self.random_state = random_state - - super().__init__() - - def fit(self, X, y): - """Wrap fit.""" - if self.classifier is None: - raise ValueError("Classifier not set") - - X, y = self._validate_data( - X=X, - y=y, - ensure_univariate=not self.concatenate_channels, - ensure_equal_length=not self.pad_unequal, - ) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - check_classification_targets(y) - self.classes_ = np.unique(y) - - self._classifier = ( - _clone_estimator(self.classifier, self.random_state) - if self.clone_estimator - else self.classifier - ) - self._classifier.fit(X, y) - - return self - - def predict(self, X) -> np.ndarray: - """Wrap predict.""" - check_is_fitted(self) - - X = self._validate_data(X=X, reset=False) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - return self._classifier.predict(X) - - def predict_proba(self, X) -> np.ndarray: - """Wrap predict_proba.""" - check_is_fitted(self) - - X = self._validate_data(X=X, reset=False) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - return self._classifier.predict_proba(X) - - def _more_tags(self): - return { - "X_types": ["2darray"], - "equal_length_only": (False if self.pad_unequal else True), - "univariate_only": False if self.concatenate_channels else True, - } diff --git a/tsml_eval/estimators/clustering/__init__.py b/tsml_eval/estimators/clustering/__init__.py index b9df55b4..94cf556e 100644 --- a/tsml_eval/estimators/clustering/__init__.py +++ b/tsml_eval/estimators/clustering/__init__.py @@ -1,7 +1 @@ """Clustering estimators.""" - -__all__ = [ - "SklearnToTsmlClusterer", -] - -from tsml_eval.estimators.clustering._sklearn_clusterer import SklearnToTsmlClusterer diff --git a/tsml_eval/estimators/clustering/_sklearn_clusterer.py b/tsml_eval/estimators/clustering/_sklearn_clusterer.py deleted file mode 100644 index c0327139..00000000 --- a/tsml_eval/estimators/clustering/_sklearn_clusterer.py +++ /dev/null @@ -1,77 +0,0 @@ -"""A tsml wrapper for sklearn clusterers.""" - -__maintainer__ = ["MatthewMiddlehurst"] -__all__ = ["SklearnToTsmlClusterer"] - -import numpy as np -from aeon.base._base import _clone_estimator -from sklearn.base import ClusterMixin -from sklearn.utils.validation import check_is_fitted -from tsml.base import BaseTimeSeriesEstimator - - -class SklearnToTsmlClusterer(ClusterMixin, BaseTimeSeriesEstimator): - """Wrapper for sklearn estimators to use the tsml base class.""" - - def __init__( - self, - clusterer=None, - pad_unequal=False, - concatenate_channels=False, - clone_estimator=True, - random_state=None, - ): - self.clusterer = clusterer - self.pad_unequal = pad_unequal - self.concatenate_channels = concatenate_channels - self.clone_estimator = clone_estimator - self.random_state = random_state - - super().__init__() - - def fit(self, X, y=None): - """Wrap fit.""" - if self.clusterer is None: - raise ValueError("Clusterer not set") - - X = self._validate_data( - X=X, - ensure_univariate=not self.concatenate_channels, - ensure_equal_length=not self.pad_unequal, - ) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - self._clusterer = ( - _clone_estimator(self.clusterer, self.random_state) - if self.clone_estimator - else self.clusterer - ) - self._clusterer.fit(X, y) - - self.labels_ = self._clusterer.labels_ - - return self - - def predict(self, X) -> np.ndarray: - """Wrap predict.""" - check_is_fitted(self) - - X = self._validate_data(X=X, reset=False) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - return self._clusterer.predict(X) - - def _more_tags(self): - return { - "X_types": ["2darray"], - "equal_length_only": (False if self.pad_unequal else True), - "univariate_only": False if self.concatenate_channels else True, - } diff --git a/tsml_eval/estimators/regression/__init__.py b/tsml_eval/estimators/regression/__init__.py deleted file mode 100644 index 940ed5d4..00000000 --- a/tsml_eval/estimators/regression/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Regression estimators.""" - -__all__ = [ - "SklearnToTsmlRegressor", -] - -from tsml_eval.estimators.regression._sklearn_regressor import SklearnToTsmlRegressor diff --git a/tsml_eval/estimators/regression/_sklearn_regressor.py b/tsml_eval/estimators/regression/_sklearn_regressor.py deleted file mode 100644 index 5bcc7fc9..00000000 --- a/tsml_eval/estimators/regression/_sklearn_regressor.py +++ /dev/null @@ -1,76 +0,0 @@ -"""A tsml wrapper for sklearn regressors.""" - -__maintainer__ = ["MatthewMiddlehurst"] -__all__ = ["SklearnToTsmlRegressor"] - -import numpy as np -from aeon.base._base import _clone_estimator -from sklearn.base import RegressorMixin -from sklearn.utils.validation import check_is_fitted -from tsml.base import BaseTimeSeriesEstimator - - -class SklearnToTsmlRegressor(RegressorMixin, BaseTimeSeriesEstimator): - """Wrapper for sklearn estimators to use the tsml base class.""" - - def __init__( - self, - regressor=None, - pad_unequal=False, - concatenate_channels=False, - clone_estimator=True, - random_state=None, - ): - self.regressor = regressor - self.pad_unequal = pad_unequal - self.concatenate_channels = concatenate_channels - self.clone_estimator = clone_estimator - self.random_state = random_state - - super().__init__() - - def fit(self, X, y): - """Wrap fit.""" - if self.regressor is None: - raise ValueError("Regressor not set") - - X, y = self._validate_data( - X=X, - y=y, - ensure_univariate=not self.concatenate_channels, - ensure_equal_length=not self.pad_unequal, - ) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - self._regressor = ( - _clone_estimator(self.regressor, self.random_state) - if self.clone_estimator - else self.regressor - ) - self._regressor.fit(X, y) - - return self - - def predict(self, X) -> np.ndarray: - """Wrap predict.""" - check_is_fitted(self) - - X = self._validate_data(X=X, reset=False) - X = self._convert_X( - X, - pad_unequal=self.pad_unequal, - concatenate_channels=self.concatenate_channels, - ) - - return self._regressor.predict(X) - - def _more_tags(self): - return { - "X_types": ["2darray"], - "equal_length_only": (False if self.pad_unequal else True), - "univariate_only": False if self.concatenate_channels else True, - } diff --git a/tsml_eval/estimators/tests/__init__.py b/tsml_eval/estimators/tests/__init__.py deleted file mode 100644 index 8ec376c7..00000000 --- a/tsml_eval/estimators/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Estimator tests.""" diff --git a/tsml_eval/estimators/tests/test_estimators.py b/tsml_eval/estimators/tests/test_estimators.py deleted file mode 100644 index 2817f1a2..00000000 --- a/tsml_eval/estimators/tests/test_estimators.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Test estimators implemented in tsml-eval.""" - -from sklearn.cluster import KMeans -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from tsml.utils.testing import parametrize_with_checks - -from tsml_eval.estimators import ( - SklearnToTsmlClassifier, - SklearnToTsmlClusterer, - SklearnToTsmlRegressor, -) - - -@parametrize_with_checks( - [ - SklearnToTsmlClassifier(classifier=RandomForestClassifier(n_estimators=5)), - SklearnToTsmlRegressor(regressor=RandomForestRegressor(n_estimators=5)), - SklearnToTsmlClusterer(clusterer=KMeans(n_clusters=2, max_iter=5)), - ] -) -def test_tsml_wrapper_estimator(estimator, check): - """Test that tsml wrapper estimators adhere to tsml conventions.""" - check(estimator) diff --git a/tsml_eval/experiments/experiments.py b/tsml_eval/experiments/experiments.py index 00ef75a2..e78ca8df 100644 --- a/tsml_eval/experiments/experiments.py +++ b/tsml_eval/experiments/experiments.py @@ -37,13 +37,13 @@ ) from sklearn.model_selection import cross_val_predict from tsml.base import BaseTimeSeriesEstimator -from tsml.utils.validation import is_clusterer - -from tsml_eval.estimators import ( +from tsml.compose import ( SklearnToTsmlClassifier, SklearnToTsmlClusterer, SklearnToTsmlRegressor, ) +from tsml.utils.validation import is_clusterer + from tsml_eval.utils.datasets import load_experiment_data from tsml_eval.utils.experiments import ( _check_existing_results, From a3c163bcb20c19354f66437c513daee446eadc45 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 21:44:01 +0100 Subject: [PATCH 10/17] redirects --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d1fbb11c..10712d9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ docs = [ "sphinx_issues", "sphinx-copybutton", "sphinx-remove-toctrees", + "sphinx-reredirects", "sphinxext-opengraph", "nbsphinx", "numpydoc", From ef3903941fdad478d8227174389887a41834dace Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 10 Sep 2025 22:39:23 +0100 Subject: [PATCH 11/17] notebook --- examples/regression_experiments.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/regression_experiments.ipynb b/examples/regression_experiments.ipynb index f5f694c8..62955c88 100644 --- a/examples/regression_experiments.ipynb +++ b/examples/regression_experiments.ipynb @@ -29,7 +29,7 @@ "from aeon.datasets import load_regression\n", "from aeon.regression import DummyRegressor\n", "from aeon.visualisation import plot_critical_difference\n", - "from sklearn.metrics import mean_squared_error\n", + "from sklearn.metrics import root_mean_squared_error\n", "from tsml.datasets import load_minimal_gas_prices\n", "\n", "from tsml_eval.evaluation.storage import load_regressor_results\n", @@ -166,7 +166,7 @@ " test_X, test_y = load_regression(d, split=\"test\")\n", " reg = reg.fit(train_X, train_y)\n", " y_pred = reg.predict(test_X)\n", - " results[d] = mean_squared_error(test_y, y_pred, squared=False)\n", + " results[d] = root_mean_squared_error(test_y, y_pred)\n", "\n", "results" ], From 74d85024b05d823b51180e0c490907d57a6fe50e Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Thu, 11 Sep 2025 11:36:02 +0100 Subject: [PATCH 12/17] notebooks --- tsml_eval/publications/y2023/tsc_bakeoff/tsc_bakeoff_2023.ipynb | 2 +- .../y2023/tser_archive_expansion/tser_archive_expansion.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tsml_eval/publications/y2023/tsc_bakeoff/tsc_bakeoff_2023.ipynb b/tsml_eval/publications/y2023/tsc_bakeoff/tsc_bakeoff_2023.ipynb index c523e858..042e8924 100644 --- a/tsml_eval/publications/y2023/tsc_bakeoff/tsc_bakeoff_2023.ipynb +++ b/tsml_eval/publications/y2023/tsc_bakeoff/tsc_bakeoff_2023.ipynb @@ -117,9 +117,9 @@ "\n", "from aeon.classification.interval_based import TimeSeriesForestClassifier\n", "from sklearn.metrics import accuracy_score\n", + "from tsml.compose import SklearnToTsmlClassifier\n", "from tsml.datasets import load_minimal_chinatown\n", "\n", - "from tsml_eval.estimators import SklearnToTsmlClassifier\n", "from tsml_eval.publications.y2023.tsc_bakeoff import _set_bakeoff_classifier\n", "from tsml_eval.utils.estimator_validation import is_sklearn_classifier" ], diff --git a/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb b/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb index 4012d345..167da8a5 100644 --- a/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb +++ b/tsml_eval/publications/y2023/tser_archive_expansion/tser_archive_expansion.ipynb @@ -65,9 +65,9 @@ "source": [ "from aeon.regression.convolution_based import RocketRegressor\n", "from sklearn.metrics import root_mean_squared_error\n", + "from tsml.compose import SklearnToTsmlRegressor\n", "from tsml.datasets import load_minimal_gas_prices\n", "\n", - "from tsml_eval.estimators import SklearnToTsmlRegressor\n", "from tsml_eval.publications.y2023.tser_archive_expansion import _set_tser_exp_regressor\n", "from tsml_eval.utils.estimator_validation import is_sklearn_regressor" ], From 4402f6900ac8600e4547b922040f0ce7091ea62c Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 24 Sep 2025 22:30:27 +0100 Subject: [PATCH 13/17] eval and gpu update --- .../gpu_classification_experiments.sh | 10 +- .../gpu_scipts/gpu_clustering_experiments.sh | 10 +- .../gpu_scipts/gpu_regression_experiments.sh | 10 +- .../soton/iridis/iridis_python.md | 41 ++- .../uea/ada/ada_python.md | 8 +- .../multiple_estimator_evaluation.py | 277 ++++++++++++++---- .../evaluation/storage/classifier_results.py | 8 +- .../evaluation/storage/clusterer_results.py | 8 +- .../evaluation/storage/estimator_results.py | 6 +- .../evaluation/storage/forecaster_results.py | 8 +- .../evaluation/storage/regressor_results.py | 24 +- 11 files changed, 301 insertions(+), 109 deletions(-) diff --git a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh index 55ae9056..69ec1384 100644 --- a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh +++ b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_classification_experiments.sh @@ -40,9 +40,8 @@ out_dir="$local_path/ClassificationResults/output/" # The python script we are running script_file_path="$local_path/tsml-eval/tsml_eval/experiments/classification_experiments.py" -# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md -# Separate environments for GPU and CPU are recommended -env_name="tsml-eval-gpu" +# the path to the apptainer sandbox. The above script or most other files do not need to be in the sandbox +container_path="scratch/tensorflow_sandbox/" # Classifiers to loop over. Must be separated by a space # See list of potential classifiers in set_classifier @@ -124,12 +123,11 @@ echo "#!/bin/bash . /etc/profile -module load anaconda/py3.10 -source activate $env_name +module load apptainer/1.3.3 # Input args to the default classification_experiments are in main method of # https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/classification_experiments.py -python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub +apptainer exec --nv ${container_path} echo "Running Apptainer job."; python -u ${script_file_path} ${data_dir} ${results_dir} ${classifier} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${classifier}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh index 5da6ea06..16859aeb 100644 --- a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh +++ b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_clustering_experiments.sh @@ -40,9 +40,8 @@ out_dir="$local_path/ClusteringResults/output/" # The python script we are running script_file_path="$local_path/tsml-eval/tsml_eval/experiments/clustering_experiments.py" -# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md -# Separate environments for GPU and CPU are recommended -env_name="tsml-eval-gpu" +# the path to the apptainer sandbox. The above script or most other files do not need to be in the sandbox +container_path="scratch/tensorflow_sandbox/" # Clusterers to loop over. Must be separated by a space # See list of potential clusterers in set_clusterer @@ -136,12 +135,11 @@ echo "#!/bin/bash . /etc/profile -module load anaconda/py3.10 -source activate $env_name +module load apptainer/1.3.3 # Input args to the default clustering_experiments are in main method of # https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/clustering_experiments.py -python -u ${script_file_path} ${data_dir} ${results_dir} ${clusterer} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_test_files} ${predefined_folds} ${combine_test_train_split} ${normalise_data}" > generatedFile.sub +apptainer exec --nv ${container_path} echo "Running Apptainer job."; python -u ${script_file_path} ${data_dir} ${results_dir} ${clusterer} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_test_files} ${predefined_folds} ${combine_test_train_split} ${normalise_data}" > generatedFile.sub echo "${count} ${clusterer}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh index 04f68666..e9fd2441 100644 --- a/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh +++ b/_tsml_research_resources/soton/iridis/gpu_scipts/gpu_regression_experiments.sh @@ -40,9 +40,8 @@ out_dir="$local_path/RegressionResults/output/" # The python script we are running script_file_path="$local_path/tsml-eval/tsml_eval/experiments/regression_experiments.py" -# Environment name, change accordingly, for set up, see https://github.com/time-series-machine-learning/tsml-eval/blob/main/_tsml_research_resources/soton/iridis/iridis_python.md -# Separate environments for GPU and CPU are recommended -env_name="tsml-eval-gpu" +# the path to the apptainer sandbox. The above script or most other files do not need to be in the sandbox +container_path="scratch/tensorflow_sandbox/" # Regressors to loop over. Must be separated by a space # See list of potential regressors in set_regressor @@ -124,12 +123,11 @@ echo "#!/bin/bash . /etc/profile -module load anaconda/py3.10 -source activate $env_name +module load apptainer/1.3.3 # Input args to the default regression_experiments are in main method of # https://github.com/time-series-machine-learning/tsml-eval/blob/main/tsml_eval/experiments/regression_experiments.py -python -u ${script_file_path} ${data_dir} ${results_dir} ${regressor} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub +apptainer exec --nv ${container_path} echo "Running Apptainer job."; python -u ${script_file_path} ${data_dir} ${results_dir} ${regressor} ${dataset} \$((\$SLURM_ARRAY_TASK_ID - 1)) ${generate_train_files} ${predefined_folds} ${normalise_data}" > generatedFile.sub echo "${count} ${regressor}/${dataset}" diff --git a/_tsml_research_resources/soton/iridis/iridis_python.md b/_tsml_research_resources/soton/iridis/iridis_python.md index b865fd9c..41908255 100644 --- a/_tsml_research_resources/soton/iridis/iridis_python.md +++ b/_tsml_research_resources/soton/iridis/iridis_python.md @@ -77,8 +77,7 @@ For conda related storage guidance, see the [related HPC webpage](https://sotona #### 3.2. Create environment -Create a new environment with a name of your choice. Replace PYTHON_VERSION with 3.12 -by default. +Create a new environment with a name of your choice. Replace PYTHON_VERSION with 3.12 by default. >conda create -n ENV_NAME python=PYTHON_VERSION @@ -121,7 +120,7 @@ or Move to the package directory i.e. ->ls tsml-eval +>cd tsml-eval This will have a `pyproject.toml` file. Run: @@ -153,11 +152,33 @@ If any a dependency install is "Killed", it is likely the session has run out of #### 5.1. tsml-eval GPU -It is recommended to use a different environment for GPU jobs. Move to the package directory and install the required packages for GPU jobs: +Currently the recommended way to run GPU jobs on Iridis is using an apptainer container built from an NVIDIA tensorflow docker image. Pulling the docker image will likely require an [NVIDIA NGC account](https://catalog.ngc.nvidia.com/) and API key. ->pip install --editable . tensorflow[and-cuda] tensorrt +>module load apptainer/1.3.3 -# Running experiments +>export APPTAINER_DOCKER_USERNAME='$oauthtoken' + +>export APPTAINER_DOCKER_PASSWORD=PUT_YOUR_API_KEY_HERE + +Pull the image you want, this can be image which has the necessary dependencies but was last tested with: + +>apptainer pull docker://nvcr.io/nvidia/tensorflow:25.02-tf2-py3 + +Create a writable sandbox from the image. This is probably large with a lot of files so will be best on scratch: + +>apptainer build --sandbox scratch/tensorflow_sandbox/ tensorflow_25.02-tf2-py3.sif + +Open a shell in the container: + +>apptainer shell --writable scratch/tensorflow_sandbox + +Install `tsml-eval` like the above instructions, this does not have to be in the sandbox: + +>cd tsml-eval + +>pip install --editable . + +## Running experiments For running jobs on Iridis, we recommend using *copies* of the submission scripts provided in this folder. @@ -167,7 +188,7 @@ Disable the conda environment before running scripts if you have installed packa >conda deactivate -## Running `tsml-eval` CPU experiments +### Running `tsml-eval` CPU experiments For CPU experiments start with one of the following scripts: @@ -189,7 +210,7 @@ Do not run threaded code on the cluster without requesting the correct amount of Requesting memory for a job will allocate it all on the jobs assigned node. New jobs will not be submitted to a node if the total allocated memory exceeds the amount available for the node. As such, requesting too much memory can block new jobs from using the node. This is ok if the memory is actually being used, but large amounts of memory should not be requested unless you know it will be required for the jobs you are submitting. Iridis is a shared resource, and instantly requesting hundreds of GB will hurt the overall efficiency of the cluster. -## Running `tsml-eval` CPU experiments on the Iridis 5 batch queue +### Running `tsml-eval` CPU experiments on the Iridis 5 batch queue If you submit less than 20 tasks when requesting the _batch_ queue, your job will be redirected to the _serial_ queue. This has a much smaller job limit which you will reach quickly when submitting a lot of jobs. If you submit a single task in each submission, you will only be running ~32 jobs at once. @@ -203,7 +224,7 @@ To get around this, you can use the batch submission scripts provided in the `ba They are named this as they use the `staskfarm` utility to run different processes over multiple threads. Read through the configuration as it is slightly different to the serial scripts. You can split task groupings by dataset by loading from a directory of submission scripts and keep classifiers separate with a variable. -## Running `tsml-eval` GPU experiments +### Running `tsml-eval` GPU experiments For GPU experiments use one of the following scripts: @@ -213,7 +234,7 @@ For GPU experiments use one of the following scripts: >gpu_clustering_experiments.sh -It is recommended you use different environments for CPU and GPU jobs. +It is recommended you use different environments for CPU and GPU jobs. Using an apptainer container this will be standard, make sure to set the path to your sandbox in the script. The default queue for GPU jobs is _gpu_. diff --git a/_tsml_research_resources/uea/ada/ada_python.md b/_tsml_research_resources/uea/ada/ada_python.md index 430cb0fe..9632a9f2 100644 --- a/_tsml_research_resources/uea/ada/ada_python.md +++ b/_tsml_research_resources/uea/ada/ada_python.md @@ -118,7 +118,7 @@ If any a dependency install is "Killed", it is likely the interactive session ha >pip install PACKAGE_NAME --no-cache-dir -#### 5.1. tsml-eval GPU +#### 5.2. tsml-eval GPU For GPU jobs we require two additional ADA modules, CUDA and cuDNN: @@ -130,13 +130,13 @@ A specific Tensorflow version is required to match the available CUDA install. >pip install --editable . tensorflow==2.3.0 -# Running experiments +## Running experiments For running jobs on ADA, we recommend using copies the submission scripts provided in this folder. **NOTE: Scripts will not run properly if done whilst the conda environment is active.** -## Running `tsml-eval` CPU experiments +### Running `tsml-eval` CPU experiments For CPU experiments start with one of the following scripts: @@ -156,7 +156,7 @@ the CPU resources allocated to others. The default python file in the scripts at Requesting memory for a job will allocate it all on the jobs assigned node. New jobs will not be submitted to a node if the total allocated memory exceeds the amount available for the node. As such, requesting too much memory can block new jobs from using the node. This is ok if the memory is actually being used, but large amounts of memory should not be requested unless you know it will be required for the jobs you are submitting. ADA is a shared resource, and instantly requesting hundreds of GB will hurt the overall efficiency of the cluster. -## Running `tsml-eval` GPU experiments +### Running `tsml-eval` GPU experiments For GPU experiments use one of the following scripts: diff --git a/tsml_eval/evaluation/multiple_estimator_evaluation.py b/tsml_eval/evaluation/multiple_estimator_evaluation.py index 1b5945c1..4f2221e4 100644 --- a/tsml_eval/evaluation/multiple_estimator_evaluation.py +++ b/tsml_eval/evaluation/multiple_estimator_evaluation.py @@ -2,6 +2,7 @@ import os import pickle +import re import warnings from datetime import datetime @@ -49,6 +50,7 @@ def evaluate_classifiers( classifier_results, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, estimator_names=None, ): @@ -67,6 +69,11 @@ def evaluate_classifiers( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. estimator_names : list of str, default=None @@ -78,6 +85,7 @@ def evaluate_classifiers( ClassifierResults.statistics, save_path, error_on_missing, + continue_on_missing, eval_name, estimator_names, ) @@ -87,6 +95,7 @@ def evaluate_classifiers_from_file( load_paths, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, estimator_names=None, @@ -106,6 +115,11 @@ def evaluate_classifiers_from_file( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -130,6 +144,7 @@ def evaluate_classifiers_from_file( classifier_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=estimator_names, ) @@ -140,9 +155,10 @@ def evaluate_classifiers_by_problem( classifier_names, dataset_names, save_path, - resamples=None, + resamples=1, load_train_results=False, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, verbose=False, @@ -180,12 +196,19 @@ def evaluate_classifiers_by_problem( length as load_path. save_path : str The path to save the evaluation results to. - resamples : int or list of int, default=None - The resamples to evaluate. If int, evaluates resamples 0 to resamples-1. + resamples : int or list of int, default=1 + The resamples to evaluate. + If int, evaluates resamples 0 to resamples-1. + if None, treats resample as empty i.e. {split}Resample.csv. load_train_results : bool, default=False Whether to load train results as well as test results. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -287,6 +310,7 @@ def evaluate_classifiers_by_problem( classifier_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=names, ) @@ -296,6 +320,7 @@ def evaluate_clusterers( clusterer_results, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, estimator_names=None, ): @@ -314,6 +339,11 @@ def evaluate_clusterers( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. estimator_names : list of str, default=None @@ -325,6 +355,7 @@ def evaluate_clusterers( ClustererResults.statistics, save_path, error_on_missing, + continue_on_missing, eval_name, estimator_names, ) @@ -334,6 +365,7 @@ def evaluate_clusterers_from_file( load_paths, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, estimator_names=None, @@ -353,6 +385,11 @@ def evaluate_clusterers_from_file( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -377,6 +414,7 @@ def evaluate_clusterers_from_file( clusterer_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=estimator_names, ) @@ -387,9 +425,10 @@ def evaluate_clusterers_by_problem( clusterer_names, dataset_names, save_path, - resamples=None, + resamples=1, load_test_results=True, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, verbose=False, @@ -427,12 +466,19 @@ def evaluate_clusterers_by_problem( length as load_path. save_path : str The path to save the evaluation results to. - resamples : int or list of int, default=None - The resamples to evaluate. If int, evaluates resamples 0 to resamples-1. + resamples : int or list of int, default=1 + The resamples to evaluate. + If int, evaluates resamples 0 to resamples-1. + if None, treats resample as empty i.e. {split}Resample.csv. load_test_results : bool, default=True Whether to load test results as well as train results. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -534,6 +580,7 @@ def evaluate_clusterers_by_problem( clusterer_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=names, ) @@ -543,6 +590,7 @@ def evaluate_regressors( regressor_results, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, estimator_names=None, ): @@ -561,6 +609,11 @@ def evaluate_regressors( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. estimator_names : list of str, default=None @@ -572,6 +625,7 @@ def evaluate_regressors( RegressorResults.statistics, save_path, error_on_missing, + continue_on_missing, eval_name, estimator_names, ) @@ -581,6 +635,7 @@ def evaluate_regressors_from_file( load_paths, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, estimator_names=None, @@ -600,6 +655,11 @@ def evaluate_regressors_from_file( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -624,6 +684,7 @@ def evaluate_regressors_from_file( regressor_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=estimator_names, ) @@ -634,9 +695,10 @@ def evaluate_regressors_by_problem( regressor_names, dataset_names, save_path, - resamples=None, + resamples=1, load_train_results=False, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, verbose=False, @@ -674,12 +736,19 @@ def evaluate_regressors_by_problem( length as load_path. save_path : str The path to save the evaluation results to. - resamples : int or list of int, default=None - The resamples to evaluate. If int, evaluates resamples 0 to resamples-1. + resamples : int or list of int, default=1 + The resamples to evaluate. + If int, evaluates resamples 0 to resamples-1. + if None, treats resample as empty i.e. {split}Resample.csv. load_train_results : bool, default=False Whether to load train results as well as test results. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -781,6 +850,7 @@ def evaluate_regressors_by_problem( regressor_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=names, ) @@ -790,6 +860,7 @@ def evaluate_forecasters( forecaster_results, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, estimator_names=None, ): @@ -808,6 +879,11 @@ def evaluate_forecasters( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. estimator_names : list of str, default=None @@ -819,6 +895,7 @@ def evaluate_forecasters( ForecasterResults.statistics, save_path, error_on_missing, + continue_on_missing, eval_name, estimator_names, ) @@ -828,6 +905,7 @@ def evaluate_forecasters_from_file( load_paths, save_path, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, estimator_names=None, @@ -847,6 +925,11 @@ def evaluate_forecasters_from_file( The path to save the evaluation results to. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -871,6 +954,7 @@ def evaluate_forecasters_from_file( forecaster_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=estimator_names, ) @@ -881,8 +965,9 @@ def evaluate_forecasters_by_problem( forecaster_names, dataset_names, save_path, - resamples=None, + resamples=1, error_on_missing=True, + continue_on_missing=False, eval_name=None, verify_results=True, verbose=False, @@ -920,10 +1005,17 @@ def evaluate_forecasters_by_problem( length as load_path. save_path : str The path to save the evaluation results to. - resamples : int or list of int, default=None - The resamples to evaluate. If int, evaluates resamples 0 to resamples-1. + resamples : int or list of int, default=1 + The resamples to evaluate. + If int, evaluates resamples 0 to resamples-1. + if None, treats resample as empty i.e. {split}Resample.csv. error_on_missing : bool, default=True Whether to raise an error if results are missing. + continue_on_missing : bool, default=False + Whether to continue the evaluation if results are missing. + If False, removes datasets with missing results from the evaluation. + If True, keeps all datasets but does not include summary results, figures + or p-values. Treats any missing stat as NaN. eval_name : str, default=None The name of the evaluation, used in save_path. verify_results : bool, default=True @@ -1019,6 +1111,7 @@ def evaluate_forecasters_by_problem( forecaster_results, save_path, error_on_missing=error_on_missing, + continue_on_missing=continue_on_missing, eval_name=eval_name, estimator_names=names, ) @@ -1029,6 +1122,7 @@ def _evaluate_estimators( statistics, save_path, error_on_missing, + continue_with_missing, eval_name, estimator_names, ): @@ -1053,22 +1147,18 @@ def _evaluate_estimators( for dataset_name in results_dict[estimator_name]: datasets.add(dataset_name) for split in results_dict[estimator_name][dataset_name]: - split_fail = False if split == "train": has_train = True elif split == "test": has_test = True else: - split_fail = True + raise ValueError( + "Results must have a split of either 'train' or 'test' " + f"to be evaluated. Unknown split {split} found for " + f"{estimator_name} on {dataset_name}." + ) for resample in results_dict[estimator_name][dataset_name][split]: - if split_fail: - raise ValueError( - "Results must have a split of either 'train' or 'test' " - f"to be evaluated. Missing for {estimator_name} on " - f"{dataset_name} resample {resample}." - ) - if resample is not None: resamples.add(resample) else: @@ -1087,11 +1177,16 @@ def _evaluate_estimators( has_dataset_test = np.zeros( (len(estimators), len(datasets), len(resamples)), dtype=bool ) + stored_n_jobs = None for estimator_name in results_dict: for dataset_name in results_dict[estimator_name]: + stored_data_transform = None + for split in results_dict[estimator_name][dataset_name]: - for resample in results_dict[estimator_name][dataset_name][split]: + for resample, result in results_dict[estimator_name][dataset_name][ + split + ].items(): if split == "train": has_dataset_train[estimators.index(estimator_name)][ datasets.index(dataset_name) @@ -1101,6 +1196,47 @@ def _evaluate_estimators( datasets.index(dataset_name) ][resamples.index(resample)] = True + n_jobs = re.search( + r"['\"]n_jobs['\"]\s*:\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+))\s*,", + result.parameter_info, + ) + n_jobs = int(n_jobs.group(1)) if n_jobs is not None else 1 + if stored_n_jobs is None: + stored_n_jobs = n_jobs + else: + if n_jobs != stored_n_jobs: + warnings.warn( + f"Number of jobs for {estimator_name} on " + f"{dataset_name} {split} resample {resample} is " + f"{n_jobs}, which is different to the first result " + f"with {stored_n_jobs}. This may cause " + f"inconsistencies in timing results.", + stacklevel=0, + ) + + data_transformers = re.search( + r"Data transformers:\s*(.*?)\.", result.description + ) + data_transformers = ( + data_transformers.group(1) + if data_transformers is not None + else "None" + ) + if stored_data_transform is None: + stored_data_transform = data_transformers + else: + if data_transformers != stored_data_transform: + warnings.warn( + f"Data transformers for {estimator_name} on " + f"{dataset_name} {split} resample {resample} are " + f"{data_transformers}, which is different to the " + f"first result for {estimator_name}: " + f"{stored_data_transform}. Double check your results" + f"as there should likely not be grouped as the" + f"same estimator.", + stacklevel=0, + ) + msg = "\n\n" missing = False splits = [] @@ -1129,7 +1265,7 @@ def _evaluate_estimators( if error_on_missing: print(msg + "\n") # noqa: T201 raise ValueError("Missing results, exiting evaluation.") - else: + elif not continue_with_missing: if has_test and has_train: has_both = has_dataset_train.all(axis=(0, 2)) & has_dataset_test.all( axis=(0, 2) @@ -1151,10 +1287,9 @@ def _evaluate_estimators( ] msg += "\nMissing results, continuing evaluation with available datasets.\n" - print(msg) # noqa: T201 else: msg += "All results present, continuing evaluation.\n" - print(msg) # noqa: T201 + print(msg) # noqa: T201 print(f"Estimators ({len(estimators)}): {estimators}\n") # noqa: T201 print(f"Datasets ({len(datasets)}): {datasets}\n") # noqa: T201 @@ -1175,10 +1310,12 @@ def _evaluate_estimators( var, save_path, eval_name, + continue_with_missing and missing, ) stats.append((average, rank, stat, ascending, split)) - _summary_evaluation(stats, estimators, save_path, eval_name) + if not continue_with_missing or not missing: + _summary_evaluation(stats, estimators, save_path, eval_name) def _create_directory_for_statistic( @@ -1193,6 +1330,7 @@ def _create_directory_for_statistic( variable_name, save_path, eval_name, + has_missing, ): os.makedirs(f"{save_path}/{statistic_name}/all_resamples/", exist_ok=True) @@ -1203,15 +1341,39 @@ def _create_directory_for_statistic( for n, dataset_name in enumerate(datasets): for j, resample in enumerate(resamples): - er = results_dict[estimator_name][dataset_name][split][resample] - er.calculate_statistics() - est_stats[n, j] = ( - er.__dict__[variable_name] - if not is_timing - else ( - time_to_milliseconds(er.__dict__[variable_name], er.time_unit) + try: + er = results_dict[estimator_name][dataset_name][split][resample] + + er.calculate_statistics() + est_stats[n, j] = ( + er.__dict__[variable_name] + if not is_timing + else ( + time_to_milliseconds( + er.__dict__[variable_name], er.time_unit + ) + ) ) - ) + + if est_stats[n, j] == -1: + warnings.warn( + f"Statistic {statistic_name} for {estimator_name} on " + f"{dataset_name} {split} resample {resample} is -1, " + f"which indicates it was not recorded or calculated " + f"correctly. Be careful using averaged values with this " + f"statistic.", + stacklevel=0, + ) + except KeyError: + est_stats[n, j] = np.nan + + if not has_missing: + raise ValueError( + f"Missing {statistic_name} for {estimator_name} on " + f"{dataset_name} {split} resample {resample}. Should not " + f"be able to reach here as files are processed in prior " + f"function!" + ) average_stats[n, i] = np.mean(est_stats[n, :]) @@ -1246,28 +1408,33 @@ def _create_directory_for_statistic( for i, dataset_name in enumerate(datasets): file.write(f"{dataset_name},{','.join([str(n) for n in ranks[i]])}\n") - p_values = wilcoxon_test(average_stats, estimators, lower_better=not higher_better) - with open( - f"{save_path}/{statistic_name}/{statistic_name.lower()}_p_values.csv", "w" - ) as file: - file.write(f"Estimators:,{','.join(estimators)}\n") - for i, estimator_name in enumerate(estimators): - file.write(f"{estimator_name},{','.join([str(n) for n in p_values[i]])}\n") - - try: - _figures_for_statistic( - average_stats, - estimators, - statistic_name, - higher_better, - save_path, - eval_name, - ) - except ValueError as e: - warnings.warn( - f"Error during figure creation for {statistic_name}: {e}", - stacklevel=2, + if not has_missing: + p_values = wilcoxon_test( + average_stats, estimators, lower_better=not higher_better ) + with open( + f"{save_path}/{statistic_name}/{statistic_name.lower()}_p_values.csv", "w" + ) as file: + file.write(f"Estimators:,{','.join(estimators)}\n") + for i, estimator_name in enumerate(estimators): + file.write( + f"{estimator_name},{','.join([str(n) for n in p_values[i]])}\n" + ) + + try: + _figures_for_statistic( + average_stats, + estimators, + statistic_name, + higher_better, + save_path, + eval_name, + ) + except ValueError as e: + warnings.warn( + f"Error during figure creation for {statistic_name}: {e}", + stacklevel=2, + ) return average_stats, ranks diff --git a/tsml_eval/evaluation/storage/classifier_results.py b/tsml_eval/evaluation/storage/classifier_results.py index 5442b417..171beb6a 100644 --- a/tsml_eval/evaluation/storage/classifier_results.py +++ b/tsml_eval/evaluation/storage/classifier_results.py @@ -40,7 +40,7 @@ class ClassifierResults(EstimatorResults): description : str, default="" Additional description of the classification experiment. Appended to the end of the first line of the results file. - parameters : str, default="No parameter info" + parameter_info : str, default="No parameter info" Information about parameters used in the classifier and other build information. Written to the second line of the results file. fit_time : float, default=-1.0 @@ -113,7 +113,7 @@ def __init__( resample_id=None, time_unit="nanoseconds", description="", - parameters="No parameter info", + parameter_info="No parameter info", fit_time=-1.0, predict_time=-1.0, benchmark_time=-1.0, @@ -163,7 +163,7 @@ def __init__( resample_id=resample_id, time_unit=time_unit, description=description, - parameters=parameters, + parameter_info=parameter_info, fit_time=fit_time, predict_time=predict_time, benchmark_time=benchmark_time, @@ -416,7 +416,7 @@ def load_classifier_results(file_path, calculate_stats=True, verify_values=True) resample_id=None if line1[3] == "None" else int(line1[3]), time_unit=line1[4].lower(), description=",".join(line1[5:]).strip(), - parameters=lines[1].strip(), + parameter_info=lines[1].strip(), fit_time=float(line3[1]), predict_time=float(line3[2]), benchmark_time=float(line3[3]), diff --git a/tsml_eval/evaluation/storage/clusterer_results.py b/tsml_eval/evaluation/storage/clusterer_results.py index c3c2b508..06bbf16f 100644 --- a/tsml_eval/evaluation/storage/clusterer_results.py +++ b/tsml_eval/evaluation/storage/clusterer_results.py @@ -38,7 +38,7 @@ class ClustererResults(EstimatorResults): description : str, default="" Additional description of the clustering experiment. Appended to the end of the first line of the results file. - parameters : str, default="No parameter info" + parameter_info : str, default="No parameter info" Information about parameters used in the clusterer and other build information. Written to the second line of the results file. fit_time : float, default=-1.0 @@ -103,7 +103,7 @@ def __init__( resample_id=None, time_unit="nanoseconds", description="", - parameters="No parameter info", + parameter_info="No parameter info", fit_time=-1.0, predict_time=-1.0, benchmark_time=-1.0, @@ -146,7 +146,7 @@ def __init__( resample_id=resample_id, time_unit=time_unit, description=description, - parameters=parameters, + parameter_info=parameter_info, fit_time=fit_time, predict_time=predict_time, benchmark_time=benchmark_time, @@ -358,7 +358,7 @@ def load_clusterer_results(file_path, calculate_stats=True, verify_values=True): resample_id=None if line1[3] == "None" else int(line1[3]), time_unit=line1[4].lower(), description=",".join(line1[5:]).strip(), - parameters=lines[1].strip(), + parameter_info=lines[1].strip(), fit_time=float(line3[1]), predict_time=float(line3[2]), benchmark_time=float(line3[3]), diff --git a/tsml_eval/evaluation/storage/estimator_results.py b/tsml_eval/evaluation/storage/estimator_results.py index a372a4d9..2790f5eb 100644 --- a/tsml_eval/evaluation/storage/estimator_results.py +++ b/tsml_eval/evaluation/storage/estimator_results.py @@ -22,7 +22,7 @@ class EstimatorResults(ABC): description : str, default="" Additional description of the experiment. Appended to the end of the first line of the results file. - parameters : str, default="No parameter info" + parameter_info : str, default="No parameter info" Information about parameters used in the estimator and other build information. Written to the second line of the results file. fit_time : float, default=-1.0 @@ -45,7 +45,7 @@ def __init__( resample_id=-1, time_unit="nanoseconds", description="", - parameters="No parameter info", + parameter_info="No parameter info", fit_time=-1.0, predict_time=-1.0, benchmark_time=-1.0, @@ -60,7 +60,7 @@ def __init__( self.description = description # Line 2 - self.parameter_info = parameters + self.parameter_info = parameter_info # Line 3 self.fit_time = fit_time diff --git a/tsml_eval/evaluation/storage/forecaster_results.py b/tsml_eval/evaluation/storage/forecaster_results.py index f9753f3d..87cdf665 100644 --- a/tsml_eval/evaluation/storage/forecaster_results.py +++ b/tsml_eval/evaluation/storage/forecaster_results.py @@ -30,7 +30,7 @@ class ForecasterResults(EstimatorResults): description : str, default="" Additional description of the forecasting experiment. Appended to the end of the first line of the results file. - parameters : str, default="No parameter info" + parameter_info : str, default="No parameter info" Information about parameters used in the forecaster and other build information. Written to the second line of the results file. fit_time : float, default=-1.0 @@ -77,7 +77,7 @@ def __init__( random_seed=None, time_unit="nanoseconds", description="", - parameters="No parameter info", + parameter_info="No parameter info", fit_time=-1.0, predict_time=-1.0, benchmark_time=-1.0, @@ -108,7 +108,7 @@ def __init__( resample_id=random_seed, time_unit=time_unit, description=description, - parameters=parameters, + parameter_info=parameter_info, fit_time=fit_time, predict_time=predict_time, benchmark_time=benchmark_time, @@ -285,7 +285,7 @@ def load_forecaster_results(file_path, calculate_stats=True, verify_values=True) random_seed=None if line1[3] == "None" else int(line1[3]), time_unit=line1[4].lower(), description=",".join(line1[5:]).strip(), - parameters=lines[1].strip(), + parameter_info=lines[1].strip(), fit_time=float(line3[1]), predict_time=float(line3[2]), benchmark_time=float(line3[3]), diff --git a/tsml_eval/evaluation/storage/regressor_results.py b/tsml_eval/evaluation/storage/regressor_results.py index 2aa8d9c9..e1b6ed48 100644 --- a/tsml_eval/evaluation/storage/regressor_results.py +++ b/tsml_eval/evaluation/storage/regressor_results.py @@ -36,7 +36,7 @@ class RegressorResults(EstimatorResults): description : str, default="" Additional description of the regression experiment. Appended to the end of the first line of the results file. - parameters : str, default="No parameter info" + parameter_info : str, default="No parameter info" Information about parameters used in the regressor and other build information. Written to the second line of the results file. fit_time : float, default=-1.0 @@ -101,7 +101,7 @@ def __init__( resample_id=None, time_unit="nanoseconds", description="", - parameters="No parameter info", + parameter_info="No parameter info", fit_time=-1.0, predict_time=-1.0, benchmark_time=-1.0, @@ -143,7 +143,7 @@ def __init__( resample_id=resample_id, time_unit=time_unit, description=description, - parameters=parameters, + parameter_info=parameter_info, fit_time=fit_time, predict_time=predict_time, benchmark_time=benchmark_time, @@ -334,6 +334,16 @@ def load_regressor_results(file_path, calculate_stats=True, verify_values=True): if pred_descriptions is not None: pred_descriptions.append(",".join(line[5:]).strip()) + # compatibility with old results files + if len(line3) > 5: + error_estimate_method = line3[5] + error_estimate_time = float(line3[6]) + build_plus_estimate_time = float(line3[7]) + else: + error_estimate_method = "N/A" + error_estimate_time = -1.0 + build_plus_estimate_time = -1.0 + rr = RegressorResults( dataset_name=line1[0], regressor_name=line1[1], @@ -341,14 +351,14 @@ def load_regressor_results(file_path, calculate_stats=True, verify_values=True): resample_id=None if line1[3] == "None" else int(line1[3]), time_unit=line1[4].lower(), description=",".join(line1[5:]).strip(), - parameters=lines[1].strip(), + parameter_info=lines[1].strip(), fit_time=float(line3[1]), predict_time=float(line3[2]), benchmark_time=float(line3[3]), memory_usage=float(line3[4]), - error_estimate_method=line3[5], - error_estimate_time=float(line3[6]), - build_plus_estimate_time=float(line3[7]), + error_estimate_method=error_estimate_method, + error_estimate_time=error_estimate_time, + build_plus_estimate_time=build_plus_estimate_time, target_labels=target_labels, predictions=predictions, pred_times=pred_times, From b19d18d767e9b1d8166b781a8ec1472a37c5612d Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 24 Sep 2025 22:35:24 +0100 Subject: [PATCH 14/17] workflow update --- .github/workflows/monthly_github_maintenance.yml | 4 ++-- .github/workflows/weekly_github_maintenance.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/monthly_github_maintenance.yml b/.github/workflows/monthly_github_maintenance.yml index cb22b54b..391603c0 100644 --- a/.github/workflows/monthly_github_maintenance.yml +++ b/.github/workflows/monthly_github_maintenance.yml @@ -1,9 +1,9 @@ -name: GitHub Maintenance +name: Monthly GitHub Maintenance on: schedule: # every 1st of the month at 01:00 AM UTC - - cron: "0 1 1 1 *" + - cron: "0 1 1 * *" workflow_dispatch: concurrency: diff --git a/.github/workflows/weekly_github_maintenance.yml b/.github/workflows/weekly_github_maintenance.yml index 8555cfd5..c9946995 100644 --- a/.github/workflows/weekly_github_maintenance.yml +++ b/.github/workflows/weekly_github_maintenance.yml @@ -1,4 +1,4 @@ -name: GitHub Maintenance +name: Weekly GitHub Maintenance on: schedule: From e51b5d84203e680e33aebde60bf6f231ddc3e2ae Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 29 Oct 2025 13:41:05 +0000 Subject: [PATCH 15/17] table util and original mcm params --- .../multiple_estimator_evaluation.py | 2 + tsml_eval/utils/publications.py | 58 ++++++++++++++++++- tsml_eval/utils/tests/test_publications.py | 38 +++++++++++- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/tsml_eval/evaluation/multiple_estimator_evaluation.py b/tsml_eval/evaluation/multiple_estimator_evaluation.py index 4f2221e4..228748e9 100644 --- a/tsml_eval/evaluation/multiple_estimator_evaluation.py +++ b/tsml_eval/evaluation/multiple_estimator_evaluation.py @@ -1499,6 +1499,8 @@ def _figures_for_statistic( save_path=f"{save_path}/{statistic_name}/figures/{eval_name}" f"_{statistic_name.lower()}_mcm", formats="pdf", + pvalue_test_params={"zero_method": "pratt", "alternative": "two-sided"}, + pvalue_correction=None, show_symetry=True, higher_stat_better=higher_better, used_statistic=statistic_name, diff --git a/tsml_eval/utils/publications.py b/tsml_eval/utils/publications.py index df87f4c8..c17561f7 100644 --- a/tsml_eval/utils/publications.py +++ b/tsml_eval/utils/publications.py @@ -9,6 +9,8 @@ import os import shutil +import pandas as pd + def extract_publication_csv_from_evaluation(stats, eval_path, write_path): """Extract the CSV files from the evaluation directory to a new directory. @@ -53,7 +55,7 @@ def extract_publication_csv_from_evaluation(stats, eval_path, write_path): def parameter_table_from_estimator_selector(selection_function, estimator_names): - """Create a table of estimator names and their parameters. + """Create a table of estimator names and their parameters in LaTeX format. Parameters ---------- @@ -61,6 +63,11 @@ def parameter_table_from_estimator_selector(selection_function, estimator_names) The function that selects the estimator. estimator_names : list of str The names of the estimators. + + Returns + ------- + str + The LaTeX table string. """ parameters = [] for estimator_name in estimator_names: @@ -77,5 +84,52 @@ def parameter_table_from_estimator_selector(selection_function, estimator_names) for key, value in params.items(): table += f"{key}: {value}, " table += " \\\\ \n" - return table + + +def results_table_from_evaluation_csv( + eval_csv_path: str, + bold_best: bool = True, + round_digits: int = 4, + rank_columns: bool = False, + higher_is_better: bool = True, +) -> str: + """Create a table of results from an evaluation CSV file in LaTeX format. + + Parameters + ---------- + eval_csv_path : str + Path to the evaluation CSV file. + bold_best : bool, default=True + Bold the highest rounded value(s) per column. + round_digits : int, default=4 + Decimal places for rounding (drives display, best, and ranking). + rank_columns : bool, default=False + Append competition rank per column in brackets, e.g. ``0.9123 (1)``. + higher_is_better : bool, default=True + Whether higher values are better for determining the best score. + + Returns + ------- + str + The LaTeX table string. + """ + df = pd.read_csv(eval_csv_path) + df.set_index(df.columns[0], inplace=True) + df.index.name = None + df = df.round(round_digits) + best = df.eq(df.max(axis=0) if higher_is_better else df.min(axis=0)) + ranks = df.rank(method="min", ascending=False if higher_is_better else True) + + out = pd.DataFrame(index=df.index, columns=df.columns, dtype="object") + for c in df.columns: + for idx, score in df[c].items(): + cell = f"{score}" + if rank_columns: + cell += f" ({int(ranks.at[idx, c])})" + if bold_best and bool(best.at[idx, c]): + cell = r"\textbf{" + cell + "}" + out.at[idx, c] = cell + + col_format = "l" + "r" * len(out.columns) + return out.to_latex(index=True, escape=False, column_format=col_format) diff --git a/tsml_eval/utils/tests/test_publications.py b/tsml_eval/utils/tests/test_publications.py index ddf364df..e1253ded 100644 --- a/tsml_eval/utils/tests/test_publications.py +++ b/tsml_eval/utils/tests/test_publications.py @@ -2,8 +2,13 @@ import os +from tsml_eval.experiments import get_classifier_by_name from tsml_eval.testing.testing_utils import _TEST_EVAL_PATH, _TEST_OUTPUT_PATH -from tsml_eval.utils.publications import extract_publication_csv_from_evaluation +from tsml_eval.utils.publications import ( + extract_publication_csv_from_evaluation, + parameter_table_from_estimator_selector, + results_table_from_evaluation_csv, +) def test_extract_publication_csv_from_evaluation(): @@ -36,3 +41,34 @@ def test_extract_publication_csv_from_evaluation(): os.remove(f"{_TEST_OUTPUT_PATH}/eval_result_files_test/1NN-DTW_accuracy.csv") os.remove(f"{_TEST_OUTPUT_PATH}/eval_result_files_test/ROCKET_accuracy.csv") os.remove(f"{_TEST_OUTPUT_PATH}/eval_result_files_test/TSF_accuracy.csv") + + +def test_parameter_table_from_estimator_selector(): + """Test creating a parameter table from an estimator selector.""" + table = parameter_table_from_estimator_selector( + get_classifier_by_name, ["ROCKET", "TSF", "1NN-DTW"] + ) + assert isinstance(table, str) + + +def test_results_table_from_evaluation_csv(): + """Test creating a results table from evaluation CSV files.""" + table = results_table_from_evaluation_csv( + f"{_TEST_EVAL_PATH}/classification/Accuracy/accuracy_mean.csv" + ) + assert isinstance(table, str) + table2 = results_table_from_evaluation_csv( + f"{_TEST_EVAL_PATH}/classification/Accuracy/accuracy_mean.csv", + bold_best=False, + round_digits=6, + rank_columns=True, + ) + assert isinstance(table2, str) + table3 = results_table_from_evaluation_csv( + f"{_TEST_EVAL_PATH}/classification/LogLoss/logloss_mean.csv", + bold_best=True, + round_digits=3, + rank_columns=True, + higher_is_better=False, + ) + assert isinstance(table3, str) From f23aae36d4802ae6a4b3db1cf26d80eb959065b1 Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Wed, 29 Oct 2025 14:23:32 +0000 Subject: [PATCH 16/17] skipif --- tsml_eval/utils/publications.py | 3 +++ tsml_eval/utils/tests/test_publications.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/tsml_eval/utils/publications.py b/tsml_eval/utils/publications.py index c17561f7..7d0c8dc2 100644 --- a/tsml_eval/utils/publications.py +++ b/tsml_eval/utils/publications.py @@ -10,6 +10,7 @@ import shutil import pandas as pd +from aeon.utils.validation._dependencies import _check_soft_dependencies def extract_publication_csv_from_evaluation(stats, eval_path, write_path): @@ -114,6 +115,8 @@ def results_table_from_evaluation_csv( str The LaTeX table string. """ + _check_soft_dependencies("jinja2") + df = pd.read_csv(eval_csv_path) df.set_index(df.columns[0], inplace=True) df.index.name = None diff --git a/tsml_eval/utils/tests/test_publications.py b/tsml_eval/utils/tests/test_publications.py index e1253ded..a3f38863 100644 --- a/tsml_eval/utils/tests/test_publications.py +++ b/tsml_eval/utils/tests/test_publications.py @@ -2,6 +2,9 @@ import os +import pytest +from aeon.utils.validation._dependencies import _check_soft_dependencies + from tsml_eval.experiments import get_classifier_by_name from tsml_eval.testing.testing_utils import _TEST_EVAL_PATH, _TEST_OUTPUT_PATH from tsml_eval.utils.publications import ( @@ -51,6 +54,10 @@ def test_parameter_table_from_estimator_selector(): assert isinstance(table, str) +@pytest.mark.skipif( + not _check_soft_dependencies("jinja2", severity="none"), + reason="skip test if required soft dependency tsfresh not available", +) def test_results_table_from_evaluation_csv(): """Test creating a results table from evaluation CSV files.""" table = results_table_from_evaluation_csv( From 3ada799154c81c123b2bd817945e99520c4c01ac Mon Sep 17 00:00:00 2001 From: MatthewMiddlehurst Date: Tue, 25 Nov 2025 21:09:20 +0000 Subject: [PATCH 17/17] table fix --- .github/workflows/monthly_github_maintenance.yml | 1 - tsml_eval/utils/publications.py | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/monthly_github_maintenance.yml b/.github/workflows/monthly_github_maintenance.yml index 391603c0..95eed0f6 100644 --- a/.github/workflows/monthly_github_maintenance.yml +++ b/.github/workflows/monthly_github_maintenance.yml @@ -33,6 +33,5 @@ jobs: days-before-delete: 455 comment-updates: true tag-committer: true - stale-branch-label: "stale branch" compare-branches: "info" pr-check: true diff --git a/tsml_eval/utils/publications.py b/tsml_eval/utils/publications.py index 7d0c8dc2..ccc02297 100644 --- a/tsml_eval/utils/publications.py +++ b/tsml_eval/utils/publications.py @@ -120,6 +120,15 @@ def results_table_from_evaluation_csv( df = pd.read_csv(eval_csv_path) df.set_index(df.columns[0], inplace=True) df.index.name = None + + def escape_underscores(s): + if isinstance(s, str): + return s.replace("_", r"\_") + return s + + df.index = df.index.map(escape_underscores) + df.columns = df.columns.map(escape_underscores) + df = df.round(round_digits) best = df.eq(df.max(axis=0) if higher_is_better else df.min(axis=0)) ranks = df.rank(method="min", ascending=False if higher_is_better else True)