Code to Reproduce LassoNet experiments of Table 1

examples/miceprotein.py

@@ -36,64 +36,63 @@
 print("Lambda =", model.best_lambda_)
 plot_cv(model, X_test, y_test)
 plt.savefig("miceprotein-cv.png")
-1 / 0
 
 model = LassoNetClassifier()
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein.png")
 
 model = LassoNetClassifier(dropout=0.5)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_dropout.png")
 
 model = LassoNetClassifier(hidden_dims=(100, 100))
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_deep.png")
 
 model = LassoNetClassifier(hidden_dims=(100, 100), gamma=0.01)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_deep_l2_weak.png")
 
 model = LassoNetClassifier(hidden_dims=(100, 100), gamma=0.1)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_deep_l2_strong.png")
 
 model = LassoNetClassifier(hidden_dims=(100, 100), gamma=1)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_deep_l2_super_strong.png")
 
 model = LassoNetClassifier(hidden_dims=(100, 100), dropout=0.5)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_deep_dropout.png")
 
 model = LassoNetClassifier(hidden_dims=(100, 100), backtrack=True, dropout=0.5)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_deep_dropout_backtrack.png")
 
 model = LassoNetClassifier(batch_size=64)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_64.png")
 
 model = LassoNetClassifier(backtrack=True)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_backtrack.png")
 
 model = LassoNetClassifier(batch_size=64, backtrack=True)
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_backtrack_64.png")
 
 model = LassoNetClassifier(class_weight=[0.1, 0.2, 0.3, 0.1, 0.3, 0, 0, 0])
-path = model.path(X_train, y_train)
+path = model.path(X_train, y_train, return_state_dicts=True)
 plot_path(model, path, X_test, y_test)
 plt.savefig("miceprotein_weighted.png")

experiments/README.MD

@@ -1,5 +1,5 @@
 - The data to reproduce Table 1 are available in [this Google Drive repository](https://drive.google.com/open?id=1quiURu7w0nU3Pxcc448xRgfeI80okLsS). Alternatively, you can download all the data sets (except MNIST and MNIST-Fashion) directly from the [UCI Repository](https://archive.ics.uci.edu/ml/datasets.php).
 - `data-utils.py` contains starter code to load the 6 datasets in Table 1 of [the paper](https://arxiv.org/abs/1907.12207).
 - You will need to download the files, unzip them and modify the `/home/lemisma/datasets` path in `data-utils.py` to point to your local copy.
+- `run.py` contains the necessary code to reproduce the results in Table 1. This script allows you to specify the dataset and other parameters. To run the script, navigate to the directory containing `run.py` and use the following command: `python run.py`.
 
-We may release additional software to reproduce the experiments, if there is strong user demand.

experiments/data_utils.py

@@ -67,21 +67,6 @@ def load_isolet():
     return (x_train, y_train - 1), (x_test, y_test - 1)
 
 
-def load_activity():
-    x_train = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_X_train.txt'), delimiter = ',', encoding = 'UTF-8')
-    x_test = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_X_test.txt'), delimiter = ',', encoding = 'UTF-8')
-    y_train = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_y_train.txt'), delimiter = ',', encoding = 'UTF-8')
-    y_test = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_y_test.txt'), delimiter = ',', encoding = 'UTF-8')
-
-    X = MinMaxScaler(feature_range=(0,1)).fit_transform(np.concatenate((x_train, x_test)))
-    x_train = X[: len(y_train)]
-    x_test = X[len(y_train):]
-
-    print(x_train.shape, y_train.shape)
-    print(x_test.shape, y_test.shape)
-    return (x_train, y_train), (x_test, y_test)
-
-
 import numpy as np
 def load_epileptic():
     filling_value = -100000
@@ -165,13 +150,13 @@ def load_data(fashion = False, digit = None, normalize = False):
 
 def load_mnist():
     train, test = load_data(fashion = False, normalize = True)
-    x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.6)
+    x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.2)
     return (x_train, y_train), (x_test, y_test)
 
 
 def load_fashion():
     train, test = load_data(fashion = True, normalize = True)
-    x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.6)
+    x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.2)
     return (x_train, y_train), (x_test, y_test)
 
 def load_mnist_two_digits(digit1, digit2):
@@ -213,4 +198,21 @@ def load_activity():
 
     print(x_train.shape, y_train.shape)
     print(x_test.shape, y_test.shape)
-    return (x_train, y_train), (x_test, y_test)
+    return (x_train, y_train), (x_test, y_test)
+
+def load_dataset(dataset):
+    if dataset == 'MNIST':
+        return load_mnist()
+    elif dataset == 'MNIST-Fashion':
+        return load_fashion()
+    if dataset == 'MICE':
+        return load_mice()
+    elif dataset == 'COIL':
+        return load_coil()
+    elif dataset == 'ISOLET':
+        return load_isolet()
+    elif dataset == 'Activity':
+        return load_activity()
+    else:
+        print('Please specify a valid dataset')
+        return None

experiments/run.py

@@ -0,0 +1,47 @@
+from sklearn.model_selection import train_test_split
+from lassonet import LassoNetClassifier
+from lassonet.utils import eval_on_path
+from data_utils import load_dataset
+import torch
+import pickle
+
+seed = None
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+batch_size = 256
+K = 50 # Number of features to select
+n_epochs = 1000
+dataset = 'ISOLET'
+
+# Load dataset and split the data
+(X_train_valid, y_train_valid), (X_test, y_test) = load_dataset(dataset)
+X_train, X_val, y_train, y_val = train_test_split(X_train_valid, y_train_valid, test_size=0.125, random_state=seed)
+
+# Set the dimensions of the hidden layers
+data_dim = X_test.shape[1]
+hidden_dim = (data_dim//3,) 
+
+# Initialize the LassoNetClassifier model and compute the path
+lasso_model = LassoNetClassifier(M=10, hidden_dims=hidden_dim, verbose=1, torch_seed=seed, random_state=seed, device=device, n_iters=n_epochs, batch_size=batch_size)
+path = lasso_model.path(X_train, y_train, X_val=X_val, y_val=y_val)
+
+# Select the features
+desired_save = next(save for save in path if save.selected.sum().item() <= K)
+SELECTED_FEATURES = desired_save.selected
+print("Number of selected features:", SELECTED_FEATURES.sum().item())
+
+# Select the features from the training, validation, and test data
+X_train_selected = X_train[:, SELECTED_FEATURES]
+X_val_selected = X_val[:, SELECTED_FEATURES]
+X_test_selected = X_test[:, SELECTED_FEATURES]
+
+# Initialize another LassoNetClassifier for retraining with the selected features
+lasso_sparse = LassoNetClassifier(M=10, hidden_dims=hidden_dim, verbose=1, torch_seed=seed, random_state=seed, device=device, n_iters=n_epochs)
+path_sparse = lasso_sparse.path(X_train_selected, y_train, X_val=X_val_selected, y_val=y_val, lambda_seq=[0], return_state_dicts=True)[:1]
+
+# Evaluate the model on the test data
+score = eval_on_path(lasso_sparse, path_sparse, X_test_selected, y_test)
+print("Test accuracy:", score)
+
+# Save the path
+with open(f'{dataset}_path.pkl', 'wb') as f:
+    pickle.dump(path_sparse, f)