Skip to content

Commit

Permalink
Merge pull request #544 from jeremyforest/num_segment_search
Browse files Browse the repository at this point in the history 
Dendrite hyperparameter search
  • Loading branch information
@jeremyforest
jeremyforest committed Aug 18, 2021
2 parents cc8a095 + 3291946 commit edff022
Show file tree
Hide file tree
Showing 10 changed files with 728 additions and 65 deletions.
125 changes: 78 additions & 47 deletions projects/dendrites/permutedMNIST/analyze_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,29 @@ def key_func(x):
return s


def parse_one_experiment(exp, state, df):
def parse(best_result, results, trial_checkpoint, df_entries, exp, tag):
config = trial_checkpoint["config"]
model_args = config["model_args"]
kw_percent_on = model_args["kw_percent_on"]
weight_sparsity = model_args.get("weight_sparsity", 0.0)
dendrite_weight_sparsity = model_args.get("dendrite_weight_sparsity", 0.0)
num_segments = model_args.get("num_segments")
dim_context = model_args["dim_context"]
epochs = config["epochs"]
num_tasks = config["num_tasks"]
lr = config["optimizer_args"]["lr"]
momentum = config["optimizer_args"].get("momentum", 0.0)
iteration = results["training_iteration"]

# This list must match the column headers in collect_results
df_entries.append(
[exp, kw_percent_on, weight_sparsity, dendrite_weight_sparsity, num_segments,
dim_context, epochs, num_tasks, lr, momentum, config["seed"], best_result,
iteration, "{} {}".format(exp, tag)]
)


def parse_one_experiment(exp, state, df, outmethod):
"""
Parse the trials in one experiment and append data to the given dataframe.
Expand Down Expand Up @@ -72,47 +94,43 @@ def parse_one_experiment(exp, state, df):
results = trial_checkpoint["results"]
if results is None:
continue

# For each checkpoint select the iteration with the best accuracy as
# the best epoch
best_results = max(results,
key=lambda x: x.get("mean_accuracy", 0.0))
best_result = best_results["mean_accuracy"]
if best_result > 0.0:
# Get the trial parameters we care about
config = trial_checkpoint["config"]
model_args = config["model_args"]
kw_percent_on = model_args["kw_percent_on"]
weight_sparsity = model_args.get("weight_sparsity", 0.0)
dendrite_weight_sparsity = model_args.get(
"dendrite_weight_sparsity", 0.0)
num_segments = model_args.get("num_segments")
dim_context = model_args["dim_context"]
epochs = config["epochs"]
num_tasks = config["num_tasks"]
lr = config["optimizer_args"]["lr"]
momentum = config["optimizer_args"].get("momentum", 0.0)

# This list must match the column headers in collect_results
df_entries.append([
exp, kw_percent_on, weight_sparsity,
dendrite_weight_sparsity, num_segments, dim_context,
epochs, num_tasks, lr, momentum,
config["seed"], best_result,
"{} {}".format(exp, tag)
])
if outmethod == "best":
# For each checkpoint select the iteration with the best
# accuracy as the best epoch
print("using parsing method : best")
best_results = max(
results, key=lambda x: x.get("mean_accuracy", 0.0)
)
best_result = best_results["mean_accuracy"]
if best_result > 0.0:
parse(best_result, results, trial_checkpoint, df_entries,
exp, tag)
elif outmethod == "lasttask":
print("using parsing method : lasttask")
last_results = results[-1]
last_result = last_results["mean_accuracy"]
if last_result > 0.0:
parse(last_result, last_results, trial_checkpoint,
df_entries, exp, tag)
elif outmethod == "all":
print("using parsing method : all")
for i, _ in enumerate(results):
i_results = results[i]
i_result = i_results["mean_accuracy"]
if i_result > 0.0:
parse(i_result, i_results, trial_checkpoint,
df_entries, exp, tag)

except Exception:
print("Problem with checkpoint group" + tag + " in " + exp
+ " ...skipping")
print(f"Problem with checkpoint group {tag} in {exp} ...skipping")
continue

# Create new dataframe from the entries with same dimensions as df
df2 = pd.DataFrame(df_entries, columns=df.columns)
return df.append(df2)


def collect_results(configs, basefilename):
def collect_results(configs, basefilename, outmethod):
"""
Parse the results for each specified experiment in each config file. Creates a
dataframe containing one row for every trial for every network configuration in
Expand All @@ -126,12 +144,9 @@ def collect_results(configs, basefilename):
"""

# The results table
columns = ["Experiment name",
"Activation sparsity", "FF weight sparsity",
"Dendrite weight sparsity", "Num segments",
"Dim context", "Epochs", "Num tasks", "LR", "Momentum", "Seed",
"Accuracy", "ID"
]
columns = ["Experiment name", "Activation sparsity", "FF weight sparsity",
"Dendrite weight sparsity", "Num segments", "Dim context", "Epochs",
"Num tasks", "LR", "Momentum", "Seed", "Accuracy", "Iteration", "ID"]
df = pd.DataFrame(columns=columns)

for exp in configs:
Expand All @@ -153,10 +168,10 @@ def collect_results(configs, basefilename):
print("Could not locate experiment state for " + exp + " ...skipping")
continue

df = parse_one_experiment(exp, states, df)
df = parse_one_experiment(exp, states, df, outmethod)

df.to_csv(basefilename + ".csv")
df.to_pickle(basefilename + ".pkl")
df.to_csv(f"{basefilename}_{outmethod}.csv")
df.to_pickle(f"{basefilename}_{outmethod}.pkl")


def analyze_experiment_data(filename_df, output_filename):
Expand All @@ -168,15 +183,26 @@ def analyze_experiment_data(filename_df, output_filename):
:param output_filename: filename to use to save the csv
"""
df = pd.read_pickle(filename_df)

# Create a dataframe containing one row per configuration. The accuracy
df_id = df.groupby(["ID"]).agg(
df_id = df.groupby(["ID", "Seed"]).agg(
num_trials=("ID", "count"),
ff_weight_sparsity=("FF weight sparsity", "first"),
activation_sparsity=("Activation sparsity", "first"),
num_segments=("Num segments", "first"),
mean_accuracy=("Accuracy", "mean"),
stdev=("Accuracy", "std"),
)
# keep consistent columns names
df_id.rename(
columns={
"num_trials": "ID",
"ff_weight_sparsity": "FF weight sparsity",
"activation_sparsity": "Activation sparsity",
"num_segments": "Num segments",
"mean_accuracy": "Accuracy",
"stdev": "std",
},
inplace=True,
)
print(df_id)
df_id.to_csv(output_filename)

Expand All @@ -188,13 +214,18 @@ def analyze_experiment_data(filename_df, output_filename):
parser.add_argument("-f", dest="format", default="grid",
help="Table format", choices=["grid", "latex_raw"])
parser.add_argument("-n", dest="name", default="temp", help="Base filename")
parser.add_argument("-o", dest="outmethod", default="best",
help="Keep only considered task/run: best, last, or all")
args = parser.parse_args()

# Get configuration values
configs = {}
for name in args.experiments:
configs[name] = copy.deepcopy(CONFIGS[name])

collect_results(configs, args.name)
collect_results(configs, args.name, args.outmethod)

analyze_experiment_data(args.name + ".pkl", args.name + "_analysis.csv")
analyze_experiment_data(
f"{args.name}_{args.outmethod}.pkl",
f"{args.name}_{args.outmethod}_analysis.csv",
)
4 changes: 3 additions & 1 deletion projects/dendrites/permutedMNIST/experiments/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from .batch import CONFIGS as BATCH
from .batch_mnist import CONFIGS as BATCH_MNIST
from .centroid import CONFIGS as CENTROID
from .hyperparameter_search import CONFIGS as HYPERPARAMETERSEARCH
from .no_dendrites import CONFIGS as NO_DENDRITES
from .si_centroid import CONFIGS as SI_CENTROID
from .sp_context import CONFIGS as SP_CONTEXT
Expand All @@ -39,7 +40,8 @@
CONFIGS.update(BATCH)
CONFIGS.update(BATCH_MNIST)
CONFIGS.update(CENTROID)
CONFIGS.update(HYPERPARAMETERSEARCH)
CONFIGS.update(NO_DENDRITES)
CONFIGS.update(SI_CENTROID)
CONFIGS.update(SP_PROTO)
CONFIGS.update(SP_CONTEXT)
CONFIGS.update(SP_PROTO)
40 changes: 24 additions & 16 deletions projects/dendrites/permutedMNIST/experiments/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,11 @@
from nupic.research.frameworks.vernon import mixins


class PermutedMNISTExperiment(mixins.RezeroWeights,
mixins.PermutedMNISTTaskIndices,
DendriteContinualLearningExperiment):
class PermutedMNISTExperiment(
mixins.RezeroWeights,
mixins.PermutedMNISTTaskIndices,
DendriteContinualLearningExperiment,
):
pass


Expand Down Expand Up @@ -99,7 +101,7 @@ class PermutedMNISTExperiment(mixins.RezeroWeights,
BASE_10_TASKS = deepcopy(DEFAULT_BASE)
BASE_10_TASKS.update(
dataset_args=dict(
num_tasks=10, # NUM_TASKS
num_tasks=10, # NUM_TASKS
root=os.path.expanduser("~/nta/results/data/"),
dim_context=1024,
seed=42,
Expand All @@ -113,23 +115,25 @@ class PermutedMNISTExperiment(mixins.RezeroWeights,
input_size=784,
output_size=10,
hidden_sizes=[2048, 2048],
num_segments=10, # NUM_TASKS
num_segments=10, # NUM_TASKS
dim_context=1024,
kw=True,
dendrite_weight_sparsity=0.0,
weight_sparsity=tune.sample_from(
lambda spec: np.random.choice([0.95, 0.90, 0.8])),
lambda spec: np.random.choice([0.95, 0.90, 0.8])
),
),

num_tasks=10, # NUM_TASKS
num_classes=100, # 10 * NUM_TASKS
num_tasks=10, # NUM_TASKS
num_classes=100, # 10 * NUM_TASKS
seed=tune.sample_from(lambda spec: np.random.randint(2, 10000)),
num_samples=20,

optimizer_class=tune.grid_search([torch.optim.Adam, torch.optim.SGD]),
optimizer_args=dict(
lr=tune.sample_from(
lambda spec: np.random.choice([0.01, 0.005, 0.001, 0.0005])),
lambda spec: np.random.choice([0.01, 0.005, 0.001, 0.0005])
),
),
)

Expand All @@ -140,14 +144,16 @@ class PermutedMNISTExperiment(mixins.RezeroWeights,
input_size=784,
output_size=10,
hidden_sizes=[2048, 2048],
num_segments=10, # NUM_TASKS
num_segments=10, # NUM_TASKS
dim_context=1024,
kw=True,
kw_percent_on=tune.sample_from(
lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0])),
lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0])
),
dendrite_weight_sparsity=0.0,
weight_sparsity=tune.sample_from(
lambda spec: np.random.choice([0.90, 0.8, 0.7, 0.5, 0.25, 0.0])),
lambda spec: np.random.choice([0.90, 0.8, 0.7, 0.5, 0.25, 0.0])
),
),

epochs=tune.sample_from(lambda spec: np.random.randint(1, 3)),
Expand All @@ -163,7 +169,7 @@ class PermutedMNISTExperiment(mixins.RezeroWeights,
BASE_50_SPARSITY_SEARCH = deepcopy(BASE_10_SPARSITY_SEARCH)
BASE_50_SPARSITY_SEARCH.update(
dataset_args=dict(
num_tasks=50, # NUM_TASKS
num_tasks=50, # NUM_TASKS
root=os.path.expanduser("~/nta/results/data/"),
dim_context=1024,
seed=42,
Expand All @@ -174,14 +180,16 @@ class PermutedMNISTExperiment(mixins.RezeroWeights,
input_size=784,
output_size=10,
hidden_sizes=[2048, 2048],
num_segments=50, # NUM_TASKS
num_segments=50, # NUM_TASKS
dim_context=1024,
kw=True,
kw_percent_on=tune.sample_from(
lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0])),
lambda spec: np.random.choice([0.05, 0.1, 0.2, 0.0])
),
dendrite_weight_sparsity=0.0,
weight_sparsity=tune.sample_from(
lambda spec: np.random.choice([0.90, 0.8, 0.5, 0.0])),
lambda spec: np.random.choice([0.90, 0.8, 0.5, 0.0])
),
),

tasks_to_validate=(0, 1, 40, 49, 50), # Tasks on which to run validate
Expand Down
Loading

0 comments on commit edff022

Please sign in to comment.