Allow timeseries script to run on more than one node #436

agent/slycat-agent-compute-timeseries.py

@@ -37,6 +37,7 @@
 parser.add_argument("--cluster-type", default="average", choices=["single", "complete", "average", "weighted"], help="Hierarchical clustering method.  Default: %(default)s")
 parser.add_argument("--cluster-metric", default="euclidean", choices=["euclidean"], help="Hierarchical clustering distance metric.  Default: %(default)s")
 parser.add_argument("--hash", default=None, help="Unique identifier for the output folder.")
+parser.add_argument("--profile", default=None, help="Name of the IPython profile to use")
 arguments = parser.parse_args()
 
 if arguments.cluster_sample_count < 1:
@@ -45,7 +46,7 @@
 _numSamples = arguments.cluster_sample_count
 
 try:
-  pool = IPython.parallel.Client()
+  pool = IPython.parallel.Client(profile=arguments.profile)[:]
 except:
   raise Exception("A running IPython parallel cluster is required to run this script.")
 
@@ -163,7 +164,7 @@ def get_time_range(directory, timeseries_index):
 
   #connection.update_model(mid, message="Collecting timeseries statistics.")
   print("Collecting timeseries statistics.")
-  time_ranges = pool[:].map_sync(get_time_range, list(itertools.repeat(arguments.directory, timeseries_count)), range(timeseries_count))
+  time_ranges = pool.map_sync(get_time_range, list(itertools.repeat(arguments.directory, timeseries_count)), range(timeseries_count))
 
   # For each cluster ...
   for index, (name, storage) in enumerate(sorted(clusters.items())):
@@ -204,7 +205,7 @@ def uniform_pla(directory, min_time, max_time, bin_count, timeseries_index, attr
       bin_counts = list(itertools.repeat(_numSamples, len(storage)))
       timeseries_indices = [timeseries for timeseries, attribute in storage]
       attribute_indices = [attribute for timeseries, attribute in storage]
-      waveforms = pool[:].map_sync(uniform_pla, directories, min_times, max_times, bin_counts, timeseries_indices, attribute_indices)
+      waveforms = pool.map_sync(uniform_pla, directories, min_times, max_times, bin_counts, timeseries_indices, attribute_indices)
     elif arguments.cluster_sample_type == "uniform-paa":
       def uniform_paa(directory, min_time, max_time, bin_count, timeseries_index, attribute_index):
         import h5py
@@ -235,7 +236,7 @@ def uniform_paa(directory, min_time, max_time, bin_count, timeseries_index, attr
       bin_counts = list(itertools.repeat(_numSamples, len(storage)))
       timeseries_indices = [timeseries for timeseries, attribute in storage]
       attribute_indices = [attribute for timeseries, attribute in storage]
-      waveforms = pool[:].map_sync(uniform_paa, directories, min_times, max_times, bin_counts, timeseries_indices, attribute_indices)
+      waveforms = pool.map_sync(uniform_paa, directories, min_times, max_times, bin_counts, timeseries_indices, attribute_indices)
 
     # Compute a distance matrix comparing every series to every other ...
     observation_count = len(waveforms)

packages/slycat/web/server/remote.py

@@ -362,20 +362,15 @@ def create_distance_matrix(fn_id, params):
         return arr
 
       def compute_timeseries(fn_id, params):
-        # arr = list(ipython_parallel_setup_arr)
-        arr = [
-          "module load %s" % module_name,
-          "ipcluster start -n %s &" % ntasks_per_node,
-          "sleep 1m"
-        ]
+        arr = list(ipython_parallel_setup_arr)
 
         if params["timeseries_file"]:
           arr.append("slycat-xyce-timeseries-push1.py --timeseries-file=%s --force %s %s" % (params["timeseries_file"], params["in_directory"], params["directory"]))
 
         # uncomment this line for production
-        arr.append("python $SLYCAT_HOME/agent/slycat-agent-compute-timeseries.py %s --cluster-sample-count %s --cluster-sample-type %s --cluster-type %s --cluster-metric %s --hash %s" % (params["directory"], params["cluster_sample_count"], params["cluster_sample_type"], params["cluster_type"], params["cluster_metric"], uid))
+        arr.append("python $SLYCAT_HOME/agent/slycat-agent-compute-timeseries.py %s --cluster-sample-count %s --cluster-sample-type %s --cluster-type %s --cluster-metric %s --hash %s --profile ${profile}" % (params["directory"], params["cluster_sample_count"], params["cluster_sample_type"], params["cluster_type"], params["cluster_metric"], uid))
         # uncomment this line for local development
-        # arr.append("python slycat-agent-compute-timeseries.py %s --cluster-sample-count %s --cluster-sample-type %s --cluster-type %s --cluster-metric %s --hash %s" % (params["directory"], params["cluster_sample_count"], params["cluster_sample_type"], params["cluster_type"], params["cluster_metric"], uid))
+        # arr.append("python slycat-agent-compute-timeseries.py %s --cluster-sample-count %s --cluster-sample-type %s --cluster-type %s --cluster-metric %s --hash %s --profile ${profile}" % (params["directory"], params["cluster_sample_count"], params["cluster_sample_type"], params["cluster_type"], params["cluster_metric"], uid))
 
         return arr
 

web-server/plugins/slycat-timeseries-model/wizard-ui.html

@@ -71,7 +71,7 @@ <h3 class="modal-title">New Timeseries Model</h3>
 
     <div data-bind="visible:tab() == 3">
       <form class="compute form-horizontal" role="form">
-      <slycat-remote-interface params="agent_functions: ['timeseries-model'], agent_functions_params: { 'timeseries_file': timeseries_file(), 'in_directory': in_directory(), 'directory': directory(), 'cluster_sample_count': cluster_sample_count(), 'cluster_sample_type': $('#timeseries-wizard-cluster-sample-type').val(), 'cluster_type': $('#timeseries-wizard-cluster-type').val(), 'cluster_metric': $('#timeseries-wizard-cluster-metric').val() }, model_type: 'timeseries', mid: model._id(), restrictions: [{'nnodes': 1}], suggestions: [{'ntasks_per_node': 8}]"></slycat-remote-interface>
+      <slycat-remote-interface params="agent_functions: ['timeseries-model'], agent_functions_params: { 'timeseries_file': timeseries_file(), 'in_directory': in_directory(), 'directory': directory(), 'cluster_sample_count': cluster_sample_count(), 'cluster_sample_type': $('#timeseries-wizard-cluster-sample-type').val(), 'cluster_type': $('#timeseries-wizard-cluster-type').val(), 'cluster_metric': $('#timeseries-wizard-cluster-metric').val() }, model_type: 'timeseries', mid: model._id(), suggestions: [{'ntasks_per_node': 8}]"></slycat-remote-interface>
       </form>
     </div>