From 03a687646e601cfee90f110df684aa4fd42513a0 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 12:12:35 +0100 Subject: [PATCH 01/12] Create light_benchmark.py --- .github/workflows/light_benchmark.py | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/light_benchmark.py diff --git a/.github/workflows/light_benchmark.py b/.github/workflows/light_benchmark.py new file mode 100644 index 00000000..70b63eb2 --- /dev/null +++ b/.github/workflows/light_benchmark.py @@ -0,0 +1,34 @@ +name: Light Benchmark + +on: + push: + paths: + - 'pyriemann_qiskit/**' + - 'examples/**' + - '.github/workflows/light_benchmark.yml' + pull_request: + paths: + - 'pyriemann_qiskit/**' + - 'examples/**' + - '.github/workflows/light_benchmark.yml' + +jobs: + light_benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: light_benchmark.yml + - name: Install dependencies + run: | + apt-get --allow-releaseinfo-change update + python -m pip install --upgrade pip + apt-get -y install --fix-missing git-core + apt-get -y install build-essential + pip install -r doc/requirements.txt + - name: Run benchmark script + run: | + python benchmarks/light_benchmark.py From 31f4ec39470159c4d4e82969202fa2e536b5e610 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 12:25:18 +0100 Subject: [PATCH 02/12] push skeleton of light_benchmark.py --- ...light_benchmark.py => light_benchmark.yml} | 0 benchmarks/light_benchmark.py | 93 +++++++++++++++++++ 2 files changed, 93 insertions(+) rename .github/workflows/{light_benchmark.py => light_benchmark.yml} (100%) create mode 100644 benchmarks/light_benchmark.py diff --git a/.github/workflows/light_benchmark.py b/.github/workflows/light_benchmark.yml similarity index 100% rename from .github/workflows/light_benchmark.py rename to .github/workflows/light_benchmark.yml diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py new file mode 100644 index 00000000..c1cfdb9c --- /dev/null +++ b/benchmarks/light_benchmark.py @@ -0,0 +1,93 @@ +""" +==================================================================== +Light Benchmark +==================================================================== + +This benchmark is a non-regression performance test, intended +to run on Ci with each PRs. + +""" +# Author: Gregoire Cattan +# Modified from plot_classify_P300_bi.py of pyRiemann +# License: BSD (3-clause) + +from pyriemann.estimation import XdawnCovariances +from pyriemann.tangentspace import TangentSpace +from sklearn.pipeline import make_pipeline +from sklearn.model_selection import train_test_split +from sklearn.metrics import balanced_accuracy_score +from matplotlib import pyplot as plt +import warnings +import seaborn as sns +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA +from moabb import set_log_level +from moabb.datasets import bi2012 +from moabb.evaluations import WithinSessionEvaluation +from moabb.paradigms import P300 +from pyriemann_qiskit.pipelines import ( + QuantumClassifierWithDefaultRiemannianPipeline, +) +from sklearn.decomposition import PCA + +print(__doc__) + +############################################################################## +# getting rid of the warnings about the future +warnings.simplefilter(action="ignore", category=FutureWarning) +warnings.simplefilter(action="ignore", category=RuntimeWarning) + +warnings.filterwarnings("ignore") + +set_log_level("info") + +############################################################################## +# Create Pipelines +# ---------------- +# +# Pipelines must be a dict of sklearn pipeline transformer. + +############################################################################## +# We have to do this because the classes are called 'Target' and 'NonTarget' +# but the evaluation function uses a LabelEncoder, transforming them +# to 0 and 1 +labels_dict = {"Target": 1, "NonTarget": 0} + +paradigm = P300(resample=128) + +dataset = bi2012() # MOABB provides several other P300 datasets + +X, y = dataset.get_data(subjects=[1]) + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) + +pipelines = {} + +pipelines["RG+QuantumSVM"] = QuantumClassifierWithDefaultRiemannianPipeline( + shots=512, + nfilter=2, + dim_red=PCA(n_components=5), +) + +pipelines["RG+LDA"] = make_pipeline( + # applies XDawn and calculates the covariance matrix, output it matrices + XdawnCovariances( + nfilter=2, + classes=[labels_dict["Target"]], + estimator="lwf", + xdawn_estimator="scm", + ), + TangentSpace(), + PCA(n_components=10), + LDA(solver="lsqr", shrinkage="auto"), # you can use other classifiers +) + +scores = {} + +for key, pipeline in pipelines: + pipeline.fit(X_train, y_train) + y_pred = pipeline.predict(X_test) + score = balanced_accuracy_score(y_test, y_pred) + scores[key] = score + + +print("Scores: ", scores) From ce6d68d7121b52a10207da275c823f927d0c92a6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 11:25:25 +0000 Subject: [PATCH 03/12] [pre-commit.ci] auto fixes from pre-commit.com hooks --- benchmarks/light_benchmark.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index c1cfdb9c..e5eca232 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -58,7 +58,9 @@ X, y = dataset.get_data(subjects=[1]) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.33, random_state=42 +) pipelines = {} From 935d5425699c12ec6a68c8132b89816d7874626c Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 12:28:19 +0100 Subject: [PATCH 04/12] use pip to install dependencies --- .github/workflows/light_benchmark.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/light_benchmark.yml b/.github/workflows/light_benchmark.yml index 70b63eb2..439d9f56 100644 --- a/.github/workflows/light_benchmark.yml +++ b/.github/workflows/light_benchmark.yml @@ -24,11 +24,7 @@ jobs: key: light_benchmark.yml - name: Install dependencies run: | - apt-get --allow-releaseinfo-change update - python -m pip install --upgrade pip - apt-get -y install --fix-missing git-core - apt-get -y install build-essential - pip install -r doc/requirements.txt + pip install .[docs] - name: Run benchmark script run: | python benchmarks/light_benchmark.py From f132d144afc1d2690e103019631578580a4cbe98 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 12:43:00 +0100 Subject: [PATCH 05/12] use paradigm to get data --- benchmarks/light_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index e5eca232..1940a83c 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -56,7 +56,7 @@ dataset = bi2012() # MOABB provides several other P300 datasets -X, y = dataset.get_data(subjects=[1]) +X, y, _ = paradigm.get_data(dataset, subjects=[1]) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=42 From e1c27c5930028e5eb0db4e4aab654a638f5d0236 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 13:03:07 +0100 Subject: [PATCH 06/12] fix bug with loop+dictionnary --- benchmarks/light_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 1940a83c..28ec2ffe 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -85,7 +85,7 @@ scores = {} -for key, pipeline in pipelines: +for key, pipeline in pipelines.items(): pipeline.fit(X_train, y_train) y_pred = pipeline.predict(X_test) score = balanced_accuracy_score(y_test, y_pred) From 8d753a5b8b70320cad61daa7829d00c69a981580 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 13:09:31 +0100 Subject: [PATCH 07/12] fix label encoding --- benchmarks/light_benchmark.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 28ec2ffe..08a96a83 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -16,6 +16,7 @@ from sklearn.pipeline import make_pipeline from sklearn.model_selection import train_test_split from sklearn.metrics import balanced_accuracy_score +from sklearn.preprocessing import LabelEncoder from matplotlib import pyplot as plt import warnings import seaborn as sns @@ -47,10 +48,6 @@ # Pipelines must be a dict of sklearn pipeline transformer. ############################################################################## -# We have to do this because the classes are called 'Target' and 'NonTarget' -# but the evaluation function uses a LabelEncoder, transforming them -# to 0 and 1 -labels_dict = {"Target": 1, "NonTarget": 0} paradigm = P300(resample=128) @@ -58,6 +55,8 @@ X, y, _ = paradigm.get_data(dataset, subjects=[1]) +y = LabelEncoder().fit_transform(y) + X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=42 ) @@ -74,7 +73,6 @@ # applies XDawn and calculates the covariance matrix, output it matrices XdawnCovariances( nfilter=2, - classes=[labels_dict["Target"]], estimator="lwf", xdawn_estimator="scm", ), From 07e5bc8f43962c945c2bba2f2041588864bf73c4 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 13:18:40 +0100 Subject: [PATCH 08/12] run benchmarl on main too --- .github/workflows/light_benchmark.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/light_benchmark.yml b/.github/workflows/light_benchmark.yml index 439d9f56..051dba2d 100644 --- a/.github/workflows/light_benchmark.yml +++ b/.github/workflows/light_benchmark.yml @@ -25,6 +25,14 @@ jobs: - name: Install dependencies run: | pip install .[docs] - - name: Run benchmark script + - name: Run benchmark script (PR) + run: | + python benchmarks/light_benchmark.py + - uses: actions/checkout@v4 + ref: 'main' + - name: Install dependencies + run: | + pip install .[docs] + - name: Run benchmark script (main) run: | python benchmarks/light_benchmark.py From 12752c5944e577516fc64bd2fa5c9e3cf6860eb8 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 13:26:15 +0100 Subject: [PATCH 09/12] syntax error --- .github/workflows/light_benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/light_benchmark.yml b/.github/workflows/light_benchmark.yml index 051dba2d..62f9688d 100644 --- a/.github/workflows/light_benchmark.yml +++ b/.github/workflows/light_benchmark.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 ref: 'main' - name: Install dependencies - run: | + run: | pip install .[docs] - name: Run benchmark script (main) run: | From f6ae8da73bba819f41d127530bdd1ade54473411 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 13:27:24 +0100 Subject: [PATCH 10/12] missing `with` --- .github/workflows/light_benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/light_benchmark.yml b/.github/workflows/light_benchmark.yml index 62f9688d..3a5fbc06 100644 --- a/.github/workflows/light_benchmark.yml +++ b/.github/workflows/light_benchmark.yml @@ -29,7 +29,8 @@ jobs: run: | python benchmarks/light_benchmark.py - uses: actions/checkout@v4 - ref: 'main' + with: + ref: 'main' - name: Install dependencies run: | pip install .[docs] From 1b732b1acf6b5dfc8881f3f03bfa4314a63fd615 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 14:52:56 +0100 Subject: [PATCH 11/12] little clean up --- benchmarks/light_benchmark.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 08a96a83..4addb7c2 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -17,13 +17,10 @@ from sklearn.model_selection import train_test_split from sklearn.metrics import balanced_accuracy_score from sklearn.preprocessing import LabelEncoder -from matplotlib import pyplot as plt import warnings -import seaborn as sns from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA from moabb import set_log_level from moabb.datasets import bi2012 -from moabb.evaluations import WithinSessionEvaluation from moabb.paradigms import P300 from pyriemann_qiskit.pipelines import ( QuantumClassifierWithDefaultRiemannianPipeline, @@ -70,7 +67,6 @@ ) pipelines["RG+LDA"] = make_pipeline( - # applies XDawn and calculates the covariance matrix, output it matrices XdawnCovariances( nfilter=2, estimator="lwf", @@ -78,7 +74,7 @@ ), TangentSpace(), PCA(n_components=10), - LDA(solver="lsqr", shrinkage="auto"), # you can use other classifiers + LDA(solver="lsqr", shrinkage="auto"), ) scores = {} From d6ce2430f8c6f051ea893cb4f595a566ce71afa2 Mon Sep 17 00:00:00 2001 From: gcattan Date: Sat, 16 Dec 2023 14:54:32 +0100 Subject: [PATCH 12/12] fix dockerfile --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index c34983f7..9d1466c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,15 +25,15 @@ RUN mkdir /home/mne_data ## Workaround for firestore RUN pip install protobuf==4.25.1 -RUN pip install google_cloud_firestore==2.14.0rc1 +RUN pip install google_cloud_firestore==2.14.0 ### Missing __init__ file in protobuf RUN touch /usr/local/lib/python3.9/site-packages/protobuf-4.25.1-py3.9-linux-x86_64.egg/google/__init__.py ## google.cloud.location is never used in these files, and is missing in path. -RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0rc1-py3.9.egg/google/cloud/firestore_v1/services/firestore/client.py' -RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0rc1-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/base.py' -RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0rc1-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/grpc.py' -RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0rc1-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/grpc_asyncio.py' -RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0rc1-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/rest.py' -RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0rc1-py3.9.egg/google/cloud/firestore_v1/services/firestore/async_client.py' +RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0-py3.9.egg/google/cloud/firestore_v1/services/firestore/client.py' +RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/base.py' +RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/grpc.py' +RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/grpc_asyncio.py' +RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0-py3.9.egg/google/cloud/firestore_v1/services/firestore/transports/rest.py' +RUN sed -i 's/from google.cloud.location import locations_pb2//g' '/usr/local/lib/python3.9/site-packages/google_cloud_firestore-2.14.0-py3.9.egg/google/cloud/firestore_v1/services/firestore/async_client.py' ENTRYPOINT [ "python", "/examples/ERP/classify_P300_bi.py" ]