diff --git a/projects/dendrites/permutedMNIST/analyze_results.py b/projects/dendrites/permutedMNIST/analyze_results.py index ab7c272bb..b69618b3d 100644 --- a/projects/dendrites/permutedMNIST/analyze_results.py +++ b/projects/dendrites/permutedMNIST/analyze_results.py @@ -41,7 +41,29 @@ def key_func(x): return s -def parse_one_experiment(exp, state, df): +def parse(best_result, results, trial_checkpoint, df_entries, exp, tag): + config = trial_checkpoint["config"] + model_args = config["model_args"] + kw_percent_on = model_args["kw_percent_on"] + weight_sparsity = model_args.get("weight_sparsity", 0.0) + dendrite_weight_sparsity = model_args.get("dendrite_weight_sparsity", 0.0) + num_segments = model_args.get("num_segments") + dim_context = model_args["dim_context"] + epochs = config["epochs"] + num_tasks = config["num_tasks"] + lr = config["optimizer_args"]["lr"] + momentum = config["optimizer_args"].get("momentum", 0.0) + iteration = results["training_iteration"] + + # This list must match the column headers in collect_results + df_entries.append( + [exp, kw_percent_on, weight_sparsity, dendrite_weight_sparsity, num_segments, + dim_context, epochs, num_tasks, lr, momentum, config["seed"], best_result, + iteration, "{} {}".format(exp, tag)] + ) + + +def parse_one_experiment(exp, state, df, outmethod): """ Parse the trials in one experiment and append data to the given dataframe. @@ -72,39 +94,35 @@ def parse_one_experiment(exp, state, df): results = trial_checkpoint["results"] if results is None: continue - - # For each checkpoint select the iteration with the best accuracy as - # the best epoch - best_results = max(results, - key=lambda x: x.get("mean_accuracy", 0.0)) - best_result = best_results["mean_accuracy"] - if best_result > 0.0: - # Get the trial parameters we care about - config = trial_checkpoint["config"] - model_args = config["model_args"] - kw_percent_on = model_args["kw_percent_on"] - weight_sparsity = model_args.get("weight_sparsity", 0.0) - dendrite_weight_sparsity = model_args.get( - "dendrite_weight_sparsity", 0.0) - num_segments = model_args.get("num_segments") - dim_context = model_args["dim_context"] - epochs = config["epochs"] - num_tasks = config["num_tasks"] - lr = config["optimizer_args"]["lr"] - momentum = config["optimizer_args"].get("momentum", 0.0) - - # This list must match the column headers in collect_results - df_entries.append([ - exp, kw_percent_on, weight_sparsity, - dendrite_weight_sparsity, num_segments, dim_context, - epochs, num_tasks, lr, momentum, - config["seed"], best_result, - "{} {}".format(exp, tag) - ]) + if outmethod == "best": + # For each checkpoint select the iteration with the best + # accuracy as the best epoch + print("using parsing method : best") + best_results = max( + results, key=lambda x: x.get("mean_accuracy", 0.0) + ) + best_result = best_results["mean_accuracy"] + if best_result > 0.0: + parse(best_result, results, trial_checkpoint, df_entries, + exp, tag) + elif outmethod == "lasttask": + print("using parsing method : lasttask") + last_results = results[-1] + last_result = last_results["mean_accuracy"] + if last_result > 0.0: + parse(last_result, last_results, trial_checkpoint, + df_entries, exp, tag) + elif outmethod == "all": + print("using parsing method : all") + for i, _ in enumerate(results): + i_results = results[i] + i_result = i_results["mean_accuracy"] + if i_result > 0.0: + parse(i_result, i_results, trial_checkpoint, + df_entries, exp, tag) except Exception: - print("Problem with checkpoint group" + tag + " in " + exp - + " ...skipping") + print(f"Problem with checkpoint group {tag} in {exp} ...skipping") continue # Create new dataframe from the entries with same dimensions as df @@ -112,7 +130,7 @@ def parse_one_experiment(exp, state, df): return df.append(df2) -def collect_results(configs, basefilename): +def collect_results(configs, basefilename, outmethod): """ Parse the results for each specified experiment in each config file. Creates a dataframe containing one row for every trial for every network configuration in @@ -126,12 +144,9 @@ def collect_results(configs, basefilename): """ # The results table - columns = ["Experiment name", - "Activation sparsity", "FF weight sparsity", - "Dendrite weight sparsity", "Num segments", - "Dim context", "Epochs", "Num tasks", "LR", "Momentum", "Seed", - "Accuracy", "ID" - ] + columns = ["Experiment name", "Activation sparsity", "FF weight sparsity", + "Dendrite weight sparsity", "Num segments", "Dim context", "Epochs", + "Num tasks", "LR", "Momentum", "Seed", "Accuracy", "Iteration", "ID"] df = pd.DataFrame(columns=columns) for exp in configs: @@ -153,10 +168,10 @@ def collect_results(configs, basefilename): print("Could not locate experiment state for " + exp + " ...skipping") continue - df = parse_one_experiment(exp, states, df) + df = parse_one_experiment(exp, states, df, outmethod) - df.to_csv(basefilename + ".csv") - df.to_pickle(basefilename + ".pkl") + df.to_csv(f"{basefilename}_{outmethod}.csv") + df.to_pickle(f"{basefilename}_{outmethod}.pkl") def analyze_experiment_data(filename_df, output_filename): @@ -168,15 +183,26 @@ def analyze_experiment_data(filename_df, output_filename): :param output_filename: filename to use to save the csv """ df = pd.read_pickle(filename_df) - - # Create a dataframe containing one row per configuration. The accuracy - df_id = df.groupby(["ID"]).agg( + df_id = df.groupby(["ID", "Seed"]).agg( num_trials=("ID", "count"), ff_weight_sparsity=("FF weight sparsity", "first"), activation_sparsity=("Activation sparsity", "first"), + num_segments=("Num segments", "first"), mean_accuracy=("Accuracy", "mean"), stdev=("Accuracy", "std"), ) + # keep consistent columns names + df_id.rename( + columns={ + "num_trials": "ID", + "ff_weight_sparsity": "FF weight sparsity", + "activation_sparsity": "Activation sparsity", + "num_segments": "Num segments", + "mean_accuracy": "Accuracy", + "stdev": "std", + }, + inplace=True, + ) print(df_id) df_id.to_csv(output_filename) @@ -188,6 +214,8 @@ def analyze_experiment_data(filename_df, output_filename): parser.add_argument("-f", dest="format", default="grid", help="Table format", choices=["grid", "latex_raw"]) parser.add_argument("-n", dest="name", default="temp", help="Base filename") + parser.add_argument("-o", dest="outmethod", default="best", + help="Keep only considered task/run: best, last, or all") args = parser.parse_args() # Get configuration values @@ -195,6 +223,9 @@ def analyze_experiment_data(filename_df, output_filename): for name in args.experiments: configs[name] = copy.deepcopy(CONFIGS[name]) - collect_results(configs, args.name) + collect_results(configs, args.name, args.outmethod) - analyze_experiment_data(args.name + ".pkl", args.name + "_analysis.csv") + analyze_experiment_data( + f"{args.name}_{args.outmethod}.pkl", + f"{args.name}_{args.outmethod}_analysis.csv", + ) diff --git a/projects/dendrites/permutedMNIST/experiments/__init__.py b/projects/dendrites/permutedMNIST/experiments/__init__.py index a65e71fac..953a32e68 100644 --- a/projects/dendrites/permutedMNIST/experiments/__init__.py +++ b/projects/dendrites/permutedMNIST/experiments/__init__.py @@ -23,6 +23,7 @@ from .batch import CONFIGS as BATCH from .batch_mnist import CONFIGS as BATCH_MNIST from .centroid import CONFIGS as CENTROID +from .hyperparameter_search import CONFIGS as HYPERPARAMETERSEARCH from .no_dendrites import CONFIGS as NO_DENDRITES from .si_centroid import CONFIGS as SI_CENTROID from .sp_context import CONFIGS as SP_CONTEXT @@ -39,7 +40,8 @@ CONFIGS.update(BATCH) CONFIGS.update(BATCH_MNIST) CONFIGS.update(CENTROID) +CONFIGS.update(HYPERPARAMETERSEARCH) CONFIGS.update(NO_DENDRITES) CONFIGS.update(SI_CENTROID) -CONFIGS.update(SP_PROTO) CONFIGS.update(SP_CONTEXT) +CONFIGS.update(SP_PROTO) diff --git a/projects/dendrites/permutedMNIST/experiments/base.py b/projects/dendrites/permutedMNIST/experiments/base.py index e12e04b17..c7f2e51d0 100644 --- a/projects/dendrites/permutedMNIST/experiments/base.py +++ b/projects/dendrites/permutedMNIST/experiments/base.py @@ -38,9 +38,11 @@ from nupic.research.frameworks.vernon import mixins -class PermutedMNISTExperiment(mixins.RezeroWeights, - mixins.PermutedMNISTTaskIndices, - DendriteContinualLearningExperiment): +class PermutedMNISTExperiment( + mixins.RezeroWeights, + mixins.PermutedMNISTTaskIndices, + DendriteContinualLearningExperiment, +): pass @@ -99,7 +101,7 @@ class PermutedMNISTExperiment(mixins.RezeroWeights, BASE_10_TASKS = deepcopy(DEFAULT_BASE) BASE_10_TASKS.update( dataset_args=dict( - num_tasks=10, # NUM_TASKS + num_tasks=10, # NUM_TASKS root=os.path.expanduser("~/nta/results/data/"), dim_context=1024, seed=42, @@ -113,23 +115,25 @@ class PermutedMNISTExperiment(mixins.RezeroWeights, input_size=784, output_size=10, hidden_sizes=[2048, 2048], - num_segments=10, # NUM_TASKS + num_segments=10, # NUM_TASKS dim_context=1024, kw=True, dendrite_weight_sparsity=0.0, weight_sparsity=tune.sample_from( - lambda spec: np.random.choice([0.95, 0.90, 0.8])), + lambda spec: np.random.choice([0.95, 0.90, 0.8]) + ), ), - num_tasks=10, # NUM_TASKS - num_classes=100, # 10 * NUM_TASKS + num_tasks=10, # NUM_TASKS + num_classes=100, # 10 * NUM_TASKS seed=tune.sample_from(lambda spec: np.random.randint(2, 10000)), num_samples=20, optimizer_class=tune.grid_search([torch.optim.Adam, torch.optim.SGD]), optimizer_args=dict( lr=tune.sample_from( - lambda spec: np.random.choice([0.01, 0.005, 0.001, 0.0005])), + lambda spec: np.random.choice([0.01, 0.005, 0.001, 0.0005]) + ), ), ) @@ -140,14 +144,16 @@ class PermutedMNISTExperiment(mixins.RezeroWeights, input_size=784, output_size=10, hidden_sizes=[2048, 2048], - num_segments=10, # NUM_TASKS + num_segments=10, # NUM_TASKS dim_context=1024, kw=True, kw_percent_on=tune.sample_from( - lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0])), + lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0]) + ), dendrite_weight_sparsity=0.0, weight_sparsity=tune.sample_from( - lambda spec: np.random.choice([0.90, 0.8, 0.7, 0.5, 0.25, 0.0])), + lambda spec: np.random.choice([0.90, 0.8, 0.7, 0.5, 0.25, 0.0]) + ), ), epochs=tune.sample_from(lambda spec: np.random.randint(1, 3)), @@ -163,7 +169,7 @@ class PermutedMNISTExperiment(mixins.RezeroWeights, BASE_50_SPARSITY_SEARCH = deepcopy(BASE_10_SPARSITY_SEARCH) BASE_50_SPARSITY_SEARCH.update( dataset_args=dict( - num_tasks=50, # NUM_TASKS + num_tasks=50, # NUM_TASKS root=os.path.expanduser("~/nta/results/data/"), dim_context=1024, seed=42, @@ -174,14 +180,16 @@ class PermutedMNISTExperiment(mixins.RezeroWeights, input_size=784, output_size=10, hidden_sizes=[2048, 2048], - num_segments=50, # NUM_TASKS + num_segments=50, # NUM_TASKS dim_context=1024, kw=True, kw_percent_on=tune.sample_from( - lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0])), + lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0]) + ), dendrite_weight_sparsity=0.0, weight_sparsity=tune.sample_from( - lambda spec: np.random.choice([0.90, 0.8, 0.5, 0.0])), + lambda spec: np.random.choice([0.90, 0.8, 0.5, 0.0]) + ), ), tasks_to_validate=(0, 1, 40, 49, 50), # Tasks on which to run validate diff --git a/projects/dendrites/permutedMNIST/experiments/hyperparameter_search.py b/projects/dendrites/permutedMNIST/experiments/hyperparameter_search.py new file mode 100644 index 000000000..b83b63597 --- /dev/null +++ b/projects/dendrites/permutedMNIST/experiments/hyperparameter_search.py @@ -0,0 +1,158 @@ +# Numenta Platform for Intelligent Computing (NuPIC) +# Copyright (C) 2021, Numenta, Inc. Unless you have an agreement +# with Numenta, Inc., for a separate license for this software code, the +# following terms and conditions apply: +# +# This program is free software you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see htt"://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# + +""" +Experiment file to run hyperparameter searches on different types of dendritic +networks in a continual learning setting using permutedMNIST as benchmark. +This config file was used to generate the data in CNS 2021's poster and +Bernstein Conference 2021 submission. +""" + +import os +from copy import deepcopy + +import numpy as np +import ray.tune as tune +import torch +import torch.nn.functional as F + +from nupic.research.frameworks.dendrites import DendriticMLP +from nupic.research.frameworks.dendrites.dendrite_cl_experiment import ( + DendriteContinualLearningExperiment, +) +from nupic.research.frameworks.pytorch.datasets import PermutedMNIST +from nupic.research.frameworks.vernon import mixins + +"""Permuted MNIST with DendriticMLP""" + + +class NbSegmentSearchExperiment( + mixins.RezeroWeights, + mixins.CentroidContext, + mixins.PermutedMNISTTaskIndices, + DendriteContinualLearningExperiment, +): + pass + + +BASE10 = dict( + experiment_class=NbSegmentSearchExperiment, + # Results path + local_dir=os.path.expanduser("~/nta/results/experiments/dendrites/"), + dataset_class=PermutedMNIST, + dataset_args=dict( + num_tasks=10, + # Consistent location outside of git repo + root=os.path.expanduser("~/nta/results/data/"), + seed=42, + download=False, # Change to True if running for the first time + ), + model_class=DendriticMLP, + model_args=dict( + input_size=784, + output_size=10, + hidden_sizes=[2048, 2048], + num_segments=tune.grid_search([2, 3, 5, 7, 10, 14, 20, 30, 50, 100]), + dim_context=784, + kw=True, + kw_percent_on=tune.grid_search( + [0.01, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 0.9, 0.99] + ), + dendrite_weight_sparsity=0.0, + weight_sparsity=tune.grid_search( + [0.01, 0.05, 0.1, 0.5, 0.7, 0.9, 0.95, 0.99] + ), + context_percent_on=0.1, + ), + batch_size=256, + val_batch_size=512, + epochs=3, + num_tasks=10, + tasks_to_validate=range(10), # Tasks on which to run validate + num_classes=10 * 10, + distributed=False, + seed=tune.sample_from(lambda spec: np.random.randint(2, 10000)), + num_samples=10, + loss_function=F.cross_entropy, + optimizer_class=torch.optim.Adam, + optimizer_args=dict(lr=5e-4), +) + +# varying only num segments +SEGMENT_SEARCH = deepcopy(BASE10) +SEGMENT_SEARCH["model_args"].update(kw_percent_on=0.1, weight_sparsity=0.5) + +# varying only kw sparsity +KW_SPARSITY_SEARCH = deepcopy(BASE10) +KW_SPARSITY_SEARCH["model_args"].update(num_segments=10, weight_sparsity=0.5) + +# varying only weights sparsity +W_SPARSITY_SEARCH = deepcopy(BASE10) +W_SPARSITY_SEARCH["model_args"].update(num_segments=10, kw_percent_on=0.1) + +TEST = deepcopy(BASE10) +TEST["model_args"].update( + kw_percent_on=0.1, weight_sparsity=0.5, num_segments=2 +) +TEST["num_samples"] = 1 + +# Idem on 50 tasks +BASE50 = deepcopy(BASE10) +BASE50["dataset_args"].update(num_tasks=50) +BASE50["tasks_to_validate"] = range(50) +BASE50["num_classes"] = 10 * 50 + +# Segment search +SEGMENT_SEARCH_50 = deepcopy(BASE50) +SEGMENT_SEARCH_50["model_args"].update(kw_percent_on=0.1, weight_sparsity=0.5) + +# kw sparsity search +KW_SPARSITY_SEARCH_50 = deepcopy(BASE50) +KW_SPARSITY_SEARCH_50["model_args"].update( + num_segments=50, weight_sparsity=0.5 +) + +# weight sparsity search +W_SPARSITY_SEARCH_50 = deepcopy(BASE50) +W_SPARSITY_SEARCH_50["model_args"].update(num_segments=50, kw_percent_on=0.1) + +TEST50 = deepcopy(BASE50) +TEST50["model_args"].update( + kw_percent_on=0.1, weight_sparsity=0.5, num_segments=2 +) +TEST50["num_samples"] = 1 + +OPTIMAL_50 = deepcopy(BASE50) +OPTIMAL_50["model_args"].update( + kw_percent_on=0.05, weight_sparsity=0.5, num_segments=100 +) + +# Export configurations in this file +CONFIGS = dict( + segment_search=SEGMENT_SEARCH, + kw_sparsity_search=KW_SPARSITY_SEARCH, + w_sparsity_search=W_SPARSITY_SEARCH, + segment_search_50=SEGMENT_SEARCH_50, + kw_sparsity_search_50=KW_SPARSITY_SEARCH_50, + w_sparsity_search_50=W_SPARSITY_SEARCH_50, + test=TEST, + test50=TEST50, + optimal_50=OPTIMAL_50, +) diff --git a/projects/dendrites/permutedMNIST/hyperparameter_search/Makefile b/projects/dendrites/permutedMNIST/hyperparameter_search/Makefile new file mode 100644 index 000000000..aa2929eb2 --- /dev/null +++ b/projects/dendrites/permutedMNIST/hyperparameter_search/Makefile @@ -0,0 +1,75 @@ +:::bash + +.PHONY: clean all + +all: figs/hyperparameter_search_panel.png figs/hyperparameter_search_panel_along_tasks.png + +data_hyperparameter_search/segment_search_lasttask.csv: ../analyze_results.py + python ../analyze_results.py segment_search -n segment_search -o lasttask && \ + mv segment_search_lasttask.pkl segment_search_lasttask.csv segment_search_lasttask_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/segment_search_50_lasttask.csv: ../analyze_results.py + python ../analyze_results.py segment_search_50 -n segment_search_50 -o lasttask && \ + mv segment_search_50_lasttask.pkl segment_search_50_lasttask.csv segment_search_50_lasttask_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/kw_sparsity_search_lasttask.csv: ../analyze_results.py + python ../analyze_results.py kw_sparsity_search -n kw_sparsity_search -o lasttask && \ + mv kw_sparsity_search_lasttask.pkl kw_sparsity_search_lasttask.csv kw_sparsity_search_lasttask_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/kw_sparsity_search_50_lasttask.csv: ../analyze_results.py + python ../analyze_results.py kw_sparsity_search_50 -n kw_sparsity_search_50 -o lasttask && \ + mv kw_sparsity_search_50_lasttask.pkl kw_sparsity_search_50_lasttask.csv kw_sparsity_search_50_lasttask_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/w_sparsity_search_lasttask.csv: ../analyze_results.py + python ../analyze_results.py w_sparsity_search -n w_sparsity_search -o lasttask && \ + mv w_sparsity_search_lasttask.pkl w_sparsity_search_lasttask.csv w_sparsity_search_lasttask_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/w_sparsity_search_50_lasttask.csv: ../analyze_results.py + python ../analyze_results.py w_sparsity_search_50 -n w_sparsity_search_50 -o lasttask && \ + mv w_sparsity_search_50_lasttask.pkl w_sparsity_search_50_lasttask.csv w_sparsity_search_50_lasttask_analysis.csv data_hyperparameter_search/ + +figs/hyperparameter_search_panel.png: hyperparameters_figures.py \ + data_hyperparameter_search/segment_search_lasttask.csv \ + data_hyperparameter_search/segment_search_50_lasttask.csv \ + data_hyperparameter_search/kw_sparsity_search_lasttask.csv \ + data_hyperparameter_search/kw_sparsity_search_50_lasttask.csv \ + data_hyperparameter_search/w_sparsity_search_lasttask.csv \ + data_hyperparameter_search/w_sparsity_search_50_lasttask.csv + python hyperparameters_figures.py + +data_hyperparameter_search/segment_search_all.csv: ../analyze_results.py + python ../analyze_results.py segment_search -n segment_search -o all && \ + mv segment_search_all.pkl segment_search_all.csv segment_search_all_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/segment_search_50_all.csv: ../analyze_results.py + python ../analyze_results.py segment_search_50 -n segment_search_50 -o all && \ + mv segment_search_50_all.pkl segment_search_50_all.csv segment_search_50_all_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/kw_sparsity_search_all.csv: ../analyze_results.py + python ../analyze_results.py kw_sparsity_search -n kw_sparsity_search -o all && \ + mv kw_sparsity_search_all.pkl kw_sparsity_search_all.csv kw_sparsity_search_all_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/kw_sparsity_search_50_all.csv: ../analyze_results.py + python ../analyze_results.py kw_sparsity_search_50 -n kw_sparsity_search_50 -o all && \ + mv kw_sparsity_search_50_all.pkl kw_sparsity_search_50_all.csv kw_sparsity_search_50_all_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/w_sparsity_search_all.csv: ../analyze_results.py + python ../analyze_results.py w_sparsity_search -n w_sparsity_search -o all && \ + mv w_sparsity_search_all.pkl w_sparsity_search_all.csv w_sparsity_search_all_analysis.csv data_hyperparameter_search/ + +data_hyperparameter_search/w_sparsity_search_50_all.csv: ../analyze_results.py + python ../analyze_results.py w_sparsity_search_50 -n w_sparsity_search_50 -o all && \ + mv w_sparsity_search_50_all.pkl w_sparsity_search_50_all.csv w_sparsity_search_50_all_analysis.csv data_hyperparameter_search/ + +figs/hyperparameter_search_panel_along_tasks.png: hyperparameters_figures.py \ + data_hyperparameter_search/segment_search_all.csv \ + data_hyperparameter_search/segment_search_50_all.csv \ + data_hyperparameter_search/kw_sparsity_search_all.csv \ + data_hyperparameter_search/kw_sparsity_search_50_all.csv \ + data_hyperparameter_search/w_sparsity_search_all.csv \ + data_hyperparameter_search/w_sparsity_search_50_all.csv + python hyperparameters_figures.py \ + +clean: + rm -rf data_hyperparameter_search/* + rm -rf figs/* diff --git a/projects/dendrites/permutedMNIST/hyperparameter_search/README.md b/projects/dendrites/permutedMNIST/hyperparameter_search/README.md new file mode 100644 index 000000000..b7cc8cecd --- /dev/null +++ b/projects/dendrites/permutedMNIST/hyperparameter_search/README.md @@ -0,0 +1,21 @@ +These files +- `hyperparameters_figures.py` +- `cns2021_figure1c.py` + +are able to generate summary figures of hyperparameter search in the config file `../experiments/hyperparameter_search.py` + +To reproduce the figures: + +1. Run the config file `hyperparameter_search.py` to generate the data. You need to run the following configs: + + - segment_search + - kw_sparsity_search + - w_sparsity_search + - segment_search_50 + - kw_sparsity_search_50 + - w_sparsity_search_50 + + +2. Once the data has been generated you just need to run the makefile using `make`. You will need to run `make` twice. The first time will error out in the middle but that is expected (it's due to the fact that the plots are in the same python script and not in two separated files). + +Once you are done use `make clean` to delete the generated csv, pkl, and png files. diff --git a/projects/dendrites/permutedMNIST/hyperparameter_search/cns2021_figure1c.py b/projects/dendrites/permutedMNIST/hyperparameter_search/cns2021_figure1c.py new file mode 100644 index 000000000..ea5164ce8 --- /dev/null +++ b/projects/dendrites/permutedMNIST/hyperparameter_search/cns2021_figure1c.py @@ -0,0 +1,99 @@ +# Numenta Platform for Intelligent Computing (NuPIC) +# Copyright (C) 2021, Numenta, Inc. Unless you have an agreement +# with Numenta, Inc., for a separate license for this software code, the +# following terms and conditions apply: +# +# This program is free software you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see htt"://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# ---------------------------------------------------------------------- + +import matplotlib.gridspec as gridspec +import matplotlib.pyplot as plt +import pandas as pd +import ptitprince as pt +import seaborn as sns + +sns.set(style="whitegrid", font_scale=1) + + +def cns_figure_1c(): + """ + CNS 2021 abstract figure 1C. + """ + data_folder = "cns2021_figure1c_data/" + savefigs = True + + df_path1 = f"{data_folder}nb_segment_search2.csv" + df1 = pd.read_csv(df_path1) + + df_path1bis = f"{data_folder}nb_segment_search3.csv" + df1bis = pd.read_csv(df_path1bis) + + df_path2 = f"{data_folder}kw_sparsity_search.csv" + df2 = pd.read_csv(df_path2) + + relevant_columns = ["Activation sparsity", "FF weight sparsity", "Num segments", + "Accuracy"] + + df1 = df1[relevant_columns] + df1bis = df1bis[relevant_columns] + df2 = df2[relevant_columns] + + df1 = pd.concat([df1, df1bis]) + + gs = gridspec.GridSpec(1, 2) + fig = plt.figure(figsize=(10, 8)) + + ax1 = fig.add_subplot(gs[:, 0]) + ax2 = fig.add_subplot(gs[:, 1]) + + x1 = "Num segments" + x2 = "Activation sparsity" + + y = "Accuracy" + ort = "v" + pal = "Set2" + sigma = 0.2 + fig.suptitle( + "Impact of the number of dendritic segments or the\n \ + activation sparsity on 10-tasks permuted MNIST performance", + fontsize=16, + ) + + pt.RainCloud(x=x1, y=y, data=df1, palette=pal, bw=sigma, width_viol=0.6, ax=ax1, + orient=ort, move=0.2, pointplot=True, alpha=0.65) + pt.RainCloud(x=x2, y=y, data=df2, palette=pal, bw=sigma, width_viol=0.6, ax=ax2, + orient=ort, move=0.2, pointplot=True, alpha=0.65) + ax1.set_ylim([0.9, 0.96]) + ax1.set_ylabel("Mean accuracy", fontsize=16) + ax1.set_xlabel("Number of dendritic segments", fontsize=16) + ax1.set_xticklabels(["2", "3", "5", "7", "10", "14", "20"], fontsize=14) + ax1.set_yticklabels( + ["0.90", "0.91", "0.92", "0.93", "0.94", "0.95", "0.96"], fontsize=14 + ) + ax2.set_ylim([0.60, 1]) + ax2.set_ylabel("") + ax2.set_xticklabels( + ["0.99", "0.95", "0.9", "0.8", "0.6", "0.4", "0.2", "0.1"], fontsize=14 + ) + ax2.set_yticklabels(["0.60", "0.65", "0.70", "0.75", "0.80", "0.85", "0.90", + "0.95", "1.0"], fontsize=14) + ax2.set_xlabel("Activation sparsity", fontsize=16) + + if savefigs: + plt.savefig("cns2021_figure1c.png", bbox_inches="tight", dpi=1200) + + +if __name__ == "__main__": + cns_figure_1c() diff --git a/projects/dendrites/permutedMNIST/hyperparameter_search/hyperparameters_figures.py b/projects/dendrites/permutedMNIST/hyperparameter_search/hyperparameters_figures.py new file mode 100755 index 000000000..a396d97cf --- /dev/null +++ b/projects/dendrites/permutedMNIST/hyperparameter_search/hyperparameters_figures.py @@ -0,0 +1,266 @@ +# Numenta Platform for Intelligent Computing (NuPIC) +# Copyright (C) 2021, Numenta, Inc. Unless you have an agreement +# with Numenta, Inc., for a separate license for this software code, the +# following terms and conditions apply: +# +# This program is free software you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see htt"://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# ---------------------------------------------------------------------- + +import os + +import matplotlib.gridspec as gridspec +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import pandas as pd +import ptitprince as pt +import seaborn as sns + +sns.set(style="ticks", font_scale=1.3) + + +def hyperparameter_search_panel(): + """ + Plots a 6 panels figure on 2 rows x 3 columns + Rows contains figures representing hyperparameters search for 10 and 50 + permutedMNIST tasks resulting from hyperparameter_search.py config file. + Columns 1 is the number of dendritic segments, columns 2 the activation + sparsity and column 3 the weight sparsity. + """ + + df_path1 = f"{experiment_folder}segment_search_lasttask.csv" + df1 = pd.read_csv(df_path1) + + df_path2 = f"{experiment_folder}kw_sparsity_search_lasttask.csv" + df2 = pd.read_csv(df_path2) + + df_path3 = f"{experiment_folder}w_sparsity_search_lasttask.csv" + df3 = pd.read_csv(df_path3) + + df_path1_50 = f"{experiment_folder}segment_search_50_lasttask.csv" + df1_50 = pd.read_csv(df_path1_50) + + df_path2_50 = f"{experiment_folder}kw_sparsity_search_50_lasttask.csv" + df2_50 = pd.read_csv(df_path2_50) + + df_path3_50 = f"{experiment_folder}w_sparsity_search_50_lasttask.csv" + df3_50 = pd.read_csv(df_path3_50) + + relevant_columns = ["Activation sparsity", "FF weight sparsity", "Num segments", + "Accuracy"] + + df1 = df1[relevant_columns] + df2 = df2[relevant_columns] + df3 = df3[relevant_columns] + df1_50 = df1_50[relevant_columns] + df2_50 = df2_50[relevant_columns] + df3_50 = df3_50[relevant_columns] + + # Figure 1 'Impact of the different hyperparameters on performance + # full cross product of hyperparameters + gs = gridspec.GridSpec(2, 3) + fig = plt.figure(figsize=(14, 10)) + + ax1 = fig.add_subplot(gs[0, 0]) + ax2 = fig.add_subplot(gs[0, 1]) + ax3 = fig.add_subplot(gs[0, 2]) + ax1_50 = fig.add_subplot(gs[1, 0]) + ax2_50 = fig.add_subplot(gs[1, 1]) + ax3_50 = fig.add_subplot(gs[1, 2]) + + x1 = "Num segments" + x2 = "Activation sparsity" + x3 = "FF weight sparsity" + + y = "Accuracy" + ort = "v" + pal = sns.color_palette(n_colors=6) + sigma = 0.2 + fig.suptitle( + "Impact of the different hyperparameters on performance", fontsize=12 + ) + + pt.RainCloud(x=x1, y=y, data=df1, palette=pal, bw=sigma, width_viol=0.6, ax=ax1, + orient=ort, move=0.2, pointplot=True, alpha=0.65) + pt.RainCloud(x=x1, y=y, data=df1_50, palette=pal, bw=sigma, width_viol=0.6, + ax=ax1_50, orient=ort, move=0.2, pointplot=True, alpha=0.65) + pt.RainCloud(x=x2, y=y, data=df2, palette=pal, bw=sigma, width_viol=0.6, ax=ax2, + orient=ort, move=0.2, pointplot=True, alpha=0.65) + pt.RainCloud(x=x2, y=y, data=df2_50, palette=pal, bw=sigma, width_viol=0.6, + ax=ax2_50, orient=ort, move=0.2, pointplot=True, alpha=0.65) + pt.RainCloud(x=x3, y=y, data=df3, palette=pal, bw=sigma, width_viol=0.6, ax=ax3, + orient=ort, move=0.2, pointplot=True, alpha=0.65) + pt.RainCloud(x=x3, y=y, data=df3_50, palette=pal, bw=sigma, width_viol=0.6, + ax=ax3_50, orient=ort, move=0.2, pointplot=True, alpha=0.65) + ax1.set_ylabel("Mean accuracy", fontsize=16) + ax1.set_xlabel("Number of dendritic segments", fontsize=16) + ax1_50.set_ylabel("Mean accuracy", fontsize=16) + ax1_50.set_xlabel("Number of dendritic segments", fontsize=16) + + ax2.set(ylabel="") + ax2.set_xlabel("Activation density", fontsize=16) + + ax2_50.set(ylabel="") + ax2_50.set_xlabel("Activation density", fontsize=16) + + ax3.set(ylabel="") + ax3.set_xlabel("FF Weight density", fontsize=16) + ax3_50.set(ylabel="") + ax3_50.set_xlabel("FF Weight density", fontsize=16) + + # Add 10 tasks and 50 tasks labels on the left + plt.figtext(-0.02, 0.7, "10 TASKS", fontsize=16) + plt.figtext(-0.02, 0.28, "50 TASKS", fontsize=16) + + fig.suptitle( + """Impact of different hyperparameters on \n 10-tasks and 50-tasks + permuted MNIST performance""", + fontsize=16, + ) + if savefigs: + plt.savefig( + f"{figs_dir}/hyperparameter_search_panel.png", bbox_inches="tight" + ) + + +def performance_across_tasks(): + """ + Similar representation as previous function (hyperparameter_search_panel) + but in this case, it represents the performance along the number of tasks + and we plot all the hyperparameters on the same figure for each figures. + """ + + df_path1 = f"{experiment_folder}segment_search_all.csv" + df1 = pd.read_csv(df_path1) + + df_path2 = f"{experiment_folder}kw_sparsity_search_all.csv" + df2 = pd.read_csv(df_path2) + + df_path3 = f"{experiment_folder}w_sparsity_search_all.csv" + df3 = pd.read_csv(df_path3) + + df_path1_50 = f"{experiment_folder}segment_search_50_all.csv" + df1_50 = pd.read_csv(df_path1_50) + + df_path2_50 = f"{experiment_folder}kw_sparsity_search_50_all.csv" + df2_50 = pd.read_csv(df_path2_50) + + df_path3_50 = f"{experiment_folder}w_sparsity_search_50_all.csv" + df3_50 = pd.read_csv(df_path3_50) + + gs = gridspec.GridSpec(2, 3) + fig = plt.figure(figsize=(14, 10)) + + ax1 = fig.add_subplot(gs[0, 0]) + ax2 = fig.add_subplot(gs[0, 1]) + ax3 = fig.add_subplot(gs[0, 2]) + ax1_50 = fig.add_subplot(gs[1, 0]) + ax2_50 = fig.add_subplot(gs[1, 1]) + ax3_50 = fig.add_subplot(gs[1, 2]) + + x1 = "Iteration" + hue1 = "Num segments" + hue2 = "Activation sparsity" + hue3 = "FF weight sparsity" + y = "Accuracy" + ort = "v" + pal = sns.color_palette(n_colors=10) + sigma = 0.2 + fig.suptitle( + """Performance along number of tasks with different + hyperpameter conditions""", + fontsize=16, + ) + + pt.RainCloud(x=x1, y=y, hue=hue1, data=df1, palette=pal, bw=sigma, width_viol=0.6, + ax=ax1, orient=ort, move=0.2, pointplot=True, alpha=0.65) + + l, h = ax1.get_legend_handles_labels() + ax1.legend(handles=l[0:10], labels=h[0:10], fontsize="8") + + pt.RainCloud(x=x1, y=y, hue=hue2, data=df2, palette=pal, bw=sigma, width_viol=0.6, + ax=ax2, orient=ort, move=0.2, pointplot=True, alpha=0.65) + + l, h = ax2.get_legend_handles_labels() + ax2.legend(handles=l[0:9], labels=h[0:9], fontsize="8") + + pt.RainCloud(x=x1, y=y, hue=hue3, data=df3, palette=pal, bw=sigma, width_viol=0.6, + ax=ax3, orient=ort, move=0.2, pointplot=True, alpha=0.65) + l, h = ax3.get_legend_handles_labels() + ax3.legend(handles=l[0:8], labels=h[0:8], fontsize="8") + + pt.RainCloud(x=x1, y=y, hue=hue1, data=df1_50, palette=pal, bw=sigma, + width_viol=0.6, ax=ax1_50, orient=ort, move=0.2, pointplot=True, + alpha=0.65) + l, h = ax1_50.get_legend_handles_labels() + labels = h[0:9] + ax1_50.legend( + handles=l[0:9], + labels=labels, + fontsize="8", + ) + + pt.RainCloud(x=x1, y=y, hue=hue2, data=df2_50, palette=pal, bw=sigma, + width_viol=0.6, ax=ax2_50, orient=ort, move=0.2, pointplot=True, + alpha=0.65) + l, h = ax2_50.get_legend_handles_labels() + ax2_50.legend(handles=l[0:8], labels=h[0:8], fontsize="8") + + pt.RainCloud(x=x1, y=y, hue=hue3, data=df3_50, palette=pal, bw=sigma, + width_viol=0.6, ax=ax3_50, orient=ort, move=0.2, pointplot=True, + alpha=0.65) + l, h = ax3_50.get_legend_handles_labels() + ax3_50.legend(handles=l[0:8], labels=h[0:8], fontsize="8") + + ax1.set_xlabel("") + ax1.set_ylabel("Mean Accuracy") + ax1.set_title("Number of segments") + ax1_50.set_xlabel("Tasks", fontsize=16) + ax1_50.set_ylabel("Mean Accuracy") + ax1_50.xaxis.set_major_locator(ticker.MultipleLocator(10)) + ax1_50.xaxis.set_major_formatter(ticker.ScalarFormatter()) + ax2.set_title("Activation density") + ax2.set_xlabel("") + ax2.set_ylabel("") + ax2_50.set_xlabel("Tasks", fontsize=16) + ax2_50.set_ylabel("") + ax2_50.xaxis.set_major_locator(ticker.MultipleLocator(10)) + ax2_50.xaxis.set_major_formatter(ticker.ScalarFormatter()) + ax3.set_xlabel("") + ax3.set_ylabel("") + ax3.set_title("FF weight density") + ax3_50.set_xlabel("Tasks", fontsize=16) + ax3_50.set_ylabel("") + ax3_50.xaxis.set_major_locator(ticker.MultipleLocator(10)) + ax3_50.xaxis.set_major_formatter(ticker.ScalarFormatter()) + + plt.figtext(-0.01, 0.7, " 10 TASKS", fontsize=14) + plt.figtext(-0.01, 0.28, " 50 TASKS", fontsize=14) + + if savefigs: + plt.savefig(f"{figs_dir}/hyperparameter_search_panel_along_tasks.png") + + +if __name__ == "__main__": + + savefigs = True + figs_dir = "figs/" + if savefigs: + if not os.path.isdir(f"{figs_dir}"): + os.makedirs(f"{figs_dir}") + + experiment_folder = "data_hyperparameter_search/" + + hyperparameter_search_panel() + performance_across_tasks() diff --git a/projects/dendrites/permutedMNIST/run_dendritic_network.py b/projects/dendrites/permutedMNIST/run_dendritic_network.py index 7926d4981..0f4003bbc 100644 --- a/projects/dendrites/permutedMNIST/run_dendritic_network.py +++ b/projects/dendrites/permutedMNIST/run_dendritic_network.py @@ -96,7 +96,7 @@ def evaluate_model(exp): correct += pred.eq(target.view_as(pred)).sum() total += len(data) - mean_acc = torch.true_divide(correct, total).item() if total > 0 else 0, + mean_acc = torch.true_divide(correct, total).item() if total > 0 else 0 return mean_acc diff --git a/requirements.txt b/requirements.txt index 243437462..b4e93e728 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,3 +57,6 @@ wandb>=0.10.23 colorama==0.4.3 urllib3==1.25.11 pytorch-lightning==1.2.9 +# for graphs +ptitprince==0.2.5 +seaborn==0.11.1