Release 0.5.2

.gitignore

@@ -4,6 +4,7 @@ docs/build/*
 *.py[cod]
 
 # C extensions
+*.c
 *.so
 
 # Packages
@@ -46,3 +47,5 @@ download
 *.pkl
 num_run
 number_submission
+.pypirc
+dmypy.json

.travis.yml

@@ -33,7 +33,7 @@ matrix:
   - os: linux
     env: DISTRIB="conda" COVERAGE="true" DOCPUSH="true" PYTHON="3.6"
   - os: linux
-    env: DISTRIB="conda" $TEST_DIST="true" PYTHON="3.7"
+    env: DISTRIB="conda" TEST_DIST="true" PYTHON="3.7"
   - os: linux
     env: DISTRIB="conda" EXAMPLES="true" PYTHON=3.7"
   - os: linux

autosklearn/__version__.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.5.1"
+__version__ = "0.5.2"

autosklearn/automl.py

@@ -1057,6 +1057,9 @@ def predict_proba(self, X, batch_size=None, n_jobs=1):
 
 
 class AutoMLRegressor(BaseAutoML):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
     def fit(
         self,
         X: np.ndarray,

autosklearn/ensemble_builder.py

@@ -257,13 +257,15 @@ def read_ensemble_preds(self):
 
         if self.shared_mode is False:
             pred_path = os.path.join(
-                    self.dir_ensemble,
-                    'predictions_ensemble_%s_*.npy' % self.seed)
+                glob.escape(self.dir_ensemble),
+                'predictions_ensemble_%s_*.npy' % self.seed,
+            )
         # pSMAC
         else:
             pred_path = os.path.join(
-                    self.dir_ensemble,
-                    'predictions_ensemble_*_*.npy')
+                glob.escape(self.dir_ensemble),
+                'predictions_ensemble_*_*.npy',
+            )
 
         y_ens_files = glob.glob(pred_path)
         # no validation predictions so far -- no files
@@ -453,13 +455,21 @@ def get_valid_test_preds(self, selected_keys: list):
 
         for k in selected_keys:
             valid_fn = glob.glob(
-                os.path.join(self.dir_valid, 'predictions_valid_%d_%d.npy'
-                                    % (self.read_preds[k]["seed"],
-                                       self.read_preds[k]["num_run"])))
+                os.path.join(
+                    glob.escape(self.dir_valid),
+                    'predictions_valid_%d_%d.npy' % (
+                        self.read_preds[k]["seed"],
+                        self.read_preds[k]["num_run"])
+                )
+            )
             test_fn = glob.glob(
-                os.path.join(self.dir_test, 'predictions_test_%d_%d.npy' %
-                                   (self.read_preds[k]["seed"],
-                                    self.read_preds[k]["num_run"])))
+                os.path.join(
+                    glob.escape(self.dir_test),
+                    'predictions_test_%d_%d.npy' % (
+                        self.read_preds[k]["seed"],
+                        self.read_preds[k]["num_run"])
+                )
+            )
 
             # TODO don't read valid and test if not changed
             if len(valid_fn) == 0:
@@ -636,11 +646,11 @@ def predict(self, set_: str,
 
     def _read_np_fn(self, fp):
         if self.precision is "16":
-            predictions = np.load(fp).astype(dtype=np.float16)
+            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float16)
         elif self.precision is "32":
-            predictions = np.load(fp).astype(dtype=np.float32)
+            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float32)
         elif self.precision is "64":
-            predictions = np.load(fp).astype(dtype=np.float64)
+            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float64)
         else:
-            predictions = np.load(fp)
+            predictions = np.load(fp, allow_pickle=True)
         return predictions

autosklearn/util/backend.py

@@ -244,7 +244,7 @@ def get_smac_output_directory_for_run(self, seed):
 
     def get_smac_output_glob(self, smac_run_id: Union[str, int] = 1) -> str:
         return os.path.join(
-            self.temporary_directory,
+            glob.escape(self.temporary_directory),
             'smac3-output',
             'run_%s' % str(smac_run_id),
         )
@@ -265,7 +265,7 @@ def save_targets_ensemble(self, targets):
         # number of times where we erronously keep a lock on the ensemble
         # targets file although the process already was killed
         try:
-            existing_targets = np.load(filepath)
+            existing_targets = np.load(filepath, allow_pickle=True)
             if existing_targets.shape[0] > targets.shape[0] or \
                     (existing_targets.shape == targets.shape and
                          np.allclose(existing_targets, targets)):
@@ -278,7 +278,7 @@ def save_targets_ensemble(self, targets):
         with lockfile.LockFile(lock_path):
             if os.path.exists(filepath):
                 with open(filepath, 'rb') as fh:
-                    existing_targets = np.load(fh)
+                    existing_targets = np.load(fh, allow_pickle=True)
                     if existing_targets.shape[0] > targets.shape[0] or \
                             (existing_targets.shape == targets.shape and
                              np.allclose(existing_targets, targets)):
@@ -299,7 +299,7 @@ def load_targets_ensemble(self):
         lock_path = filepath + '.lock'
         with lockfile.LockFile(lock_path):
             with open(filepath, 'rb') as fh:
-                targets = np.load(fh)
+                targets = np.load(fh, allow_pickle=True)
 
         return targets
 
@@ -346,8 +346,9 @@ def save_model(self, model, idx, seed):
     def list_all_models(self, seed):
         model_directory = self.get_model_dir()
         if seed >= 0:
-            model_files = glob.glob(os.path.join(model_directory,
-                                                 '%s.*.model' % seed))
+            model_files = glob.glob(
+                os.path.join(glob.escape(model_directory), '%s.*.model' % seed)
+            )
         else:
             model_files = os.listdir(model_directory)
             model_files = [os.path.join(model_directory, mf)
@@ -408,9 +409,11 @@ def load_ensemble(self, seed):
             self.logger.warning('Directory %s does not exist' % ensemble_dir)
             return None
 
+        print(seed)
         if seed >= 0:
-            indices_files = glob.glob(os.path.join(ensemble_dir,
-                                                   '%s.*.ensemble' % seed))
+            indices_files = glob.glob(
+                os.path.join(glob.escape(ensemble_dir), '%s.*.ensemble' % seed)
+            )
             indices_files.sort()
         else:
             indices_files = os.listdir(ensemble_dir)
@@ -419,6 +422,7 @@ def load_ensemble(self, seed):
 
         with open(indices_files[-1], 'rb') as fh:
             ensemble_members_run_numbers = pickle.load(fh)
+        print(indices_files)
 
         return ensemble_members_run_numbers
 

doc/releases.rst

@@ -11,6 +11,22 @@
 Releases
 ========
 
+Version 0.5.2
+=============
+
+* FIX #669: Correctly handle arguments to the ``AutoMLRegressor``
+* FIX #667: Auto-sklearn works with numpy 1.16.3 again.
+* ADD #676: Allow brackets [ ] inside the temporary and output directory paths.
+* ADD #424: (Experimental) scripts to reproduce the results from the original Auto-sklearn paper.
+
+Contributors
+************
+
+* Jin Woo Ahn
+* Herilalaina Rakotoarison
+* Matthias Feurer
+* yazanobeidi
+
 Version 0.5.1
 =============
 

scripts/2015_nips_paper/Readme.md

@@ -0,0 +1,34 @@
+## Reproduce results of Efficient and Robust Automated Machine Learning (Feurer et al.)
+This folder contains all necessary scripts in order to reproduce the results shown in
+Figure 3 of Efficient and Robust Automated Machine Learning (Feurer et al.). The scripts
+can be modified to include different datasets, change the runtime, etc. The scripts only
+only handles classification tasks, and balanced accuracy is used as the score metric.
+
+### 1. Creating commands.txt
+To run the experiment, first create commands.txt by running:
+```bash
+cd setup
+bash create_commands.sh
+```
+The script can be modified to run experiments with different settings, i.e. 
+different runtime and/or different tasks.
+
+### 2. Executing commands.txt
+Run each commands in commands.txt:
+```bash
+cd run
+bash run_commands.sh
+```
+Each command line in commands.txt first executes model fitting, and then creating the
+single best and ensemble trajectories. Therefore, the commands can be run in parallel
+on a cluster by modifying run_commands.sh.
+
+### 3. Plotting the results
+To plot the results, run:
+```bash
+cd plot
+bash plot_ranks.py
+```
+
+
+

scripts/2015_nips_paper/plot/plot_ranks.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+import csv
+import sys
+import os
+
+import numpy as np
+
+import pandas as pd
+import matplotlib.pyplot as plt
+
+
+def read_csv(fn, has_header=True, data_type=str):
+    """
+    Function which reads the csv files containing trajectories
+    of the auto-sklearn runs.
+    """
+    data = list()
+    header = None
+    with open(fn, 'r') as csvfile:
+        csv_reader = csv.reader(csvfile, delimiter=',', quotechar='|')
+        for row in csv_reader:
+            if header is None and has_header:
+                header = row
+                continue
+            data.append(list(map(data_type, [i.strip() for i in row])))
+    return header, data
+
+
+def fill_trajectory(performance_list, time_list):
+    # Create n series objects.
+    series_list = []
+    for n in range(len(time_list)):
+        series_list.append(pd.Series(data=performance_list[n], index=time_list[n]))
+
+    # Concatenate to one Series with NaN vales.
+    series = pd.concat(series_list, axis=1)
+
+    # Fill missing performance values (NaNs) with last non-NaN value.
+    series = series.fillna(method='ffill')
+
+    # return the trajectories over seeds (series object)
+    return series
+
+
+def main():
+    # name of the file where the plot is stored
+    saveto = "../plot.png"
+    # runtime of each experiment
+    max_runtime = 3600
+    # folder where all trajectories are stored.
+    working_directory = "../log_output"
+
+    # list of models
+    model_list = ['vanilla', 'ensemble', 'metalearning', 'meta_ensemble']
+
+    # list of seeds
+    seed_dir = os.path.join(working_directory, 'vanilla')
+    seed_list = [seed for seed in os.listdir(seed_dir)]
+
+    # list of tasks
+    vanilla_task_dir = os.path.join(seed_dir, seed_list[0])
+    task_list = [task_id for task_id in os.listdir(vanilla_task_dir)]
+
+    # Step 1. Merge all trajectories into one Dataframe object.
+    #####################################################################################
+    all_trajectories = []
+
+    for model in model_list:
+        trajectories = []
+        for task_id in task_list:
+            csv_files = []
+
+            for seed in seed_list:
+                # collect all csv files of different seeds for current model and
+                # current task.
+                if model in ['vanilla', 'ensemble']:
+                    csv_file = os.path.join(working_directory,
+                                            'vanilla',
+                                            seed,
+                                            task_id,
+                                            "score_{}.csv".format(model)
+                                            )
+
+                elif model in ['metalearning', 'meta_ensemble']:
+                    csv_file = os.path.join(working_directory,
+                                            'metalearning',
+                                            seed,
+                                            task_id,
+                                            "score_{}.csv".format(model),
+                                            )
+                csv_files.append(csv_file)
+
+            performance_list = []
+            time_list = []
+
+            # Get data from csv
+            for fl in csv_files:
+                _, csv_data = read_csv(fl, has_header=True)
+                csv_data = np.array(csv_data)
+                # Replace too high values with args.maxsize
+                data = [min([sys.maxsize, float(i.strip())]) for i in
+                        csv_data[:, 2]]  # test trajectories are stored in third column
+
+                time_steps = [float(i.strip()) for i in csv_data[:, 0]]
+                assert time_steps[0] == 0
+
+                performance_list.append(data)
+                time_list.append(time_steps)
+
+            # trajectory is the pd.Series object containing all seed runs of the
+            # current model and current task.
+            trajectory = fill_trajectory(performance_list, time_list)
+            trajectories.append(trajectory)
+
+        # list[list[pd.Series]]
+        all_trajectories.append(trajectories)
+
+    # Step 2. Compute average ranks of the trajectories.
+    #####################################################################################
+    all_rankings = []
+    n_iter = 500  # number of bootstrap samples to use for estimating the ranks.
+    n_tasks = len(task_list)
+
+    for i in range(n_iter):
+        pick = np.random.choice(all_trajectories[0][0].shape[1],
+                                size=(len(model_list)))
+
+        for j in range(n_tasks):
+            all_trajectories_tmp = pd.DataFrame(
+                {model_list[k]: at[j].iloc[:, pick[k]] for
+                 k, at in enumerate(all_trajectories)}
+            )
+            all_trajectories_tmp = all_trajectories_tmp.fillna(method='ffill', axis=0)
+            r_tmp = all_trajectories_tmp.rank(axis=1)
+            all_rankings.append(r_tmp)
+
+    final_ranks = []
+    for i, model in enumerate(model_list):
+        ranks_for_model = []
+        for ranking in all_rankings:
+            ranks_for_model.append(ranking.loc[:, model])
+        ranks_for_model = pd.DataFrame(ranks_for_model)
+        ranks_for_model = ranks_for_model.fillna(method='ffill', axis=1)
+        final_ranks.append(ranks_for_model.mean(skipna=True))
+
+    # Step 3. Plot the average ranks over time.
+    #####################################################################################
+    for i, model in enumerate(model_list):
+        X_data = []
+        y_data = []
+        for x, y in final_ranks[i].iteritems():
+            X_data.append(x)
+            y_data.append(y)
+        X_data.append(max_runtime)
+        y_data.append(y)
+        plt.plot(X_data, y_data, label=model)
+        plt.xlabel('time [sec]')
+        plt.ylabel('average rank')
+        plt.legend()
+    plt.savefig(saveto)
+
+
+if __name__ == "__main__":
+    main()