Develop IDMTools docker image for SSMT (#604)

* SSMT docker image * Add version file to be used in versioning the docker image Add make rules to COMPS Finish Docker image * Update Image to use COMPS base image. (Still needs the FROM) * Update Image to use COMPS base image docker file. We need to later actually move to a true derived image * Fix build version for ssmt images * Add minor version of image as well * filter patch versions * Add version to image * Pass version to image so it will be on the environment when running * Fix import error on COMPS SSMT image * add missing init file * fix import issue for CompsCLI for ssmt * add matplotlib package for ssmt example analyzers * Add init * Fix analyze_ssmt.py and ssmt_analysis.py to allow to pass Platform * fix ssmt examples, TODO, still need to fix all integration tests Co-authored-by: Sharon Chen <38014764+shchen-idmod@users.noreply.github.com>
InstituteforDiseaseModeling · Feb 19, 2020 · eff6bef · eff6bef
1 parent 94259ed
commit eff6bef
Show file tree

Hide file tree

Showing 26 changed files with 592 additions and 55 deletions.
diff --git a/examples/ssmt/generic_ssmt/files/MyAnalyzer.py b/examples/ssmt/generic_ssmt/files/MyAnalyzer.py
@@ -1,4 +1,5 @@
 import json
+import os
 
 try:
     # use idmtools image
@@ -8,7 +9,6 @@
     from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
 
 import matplotlib as mpl
-
 mpl.use('Agg')
 
 
@@ -21,7 +21,10 @@ def select_simulation_data(self, data, simulation):
         return data[self.filenames[0]]["Channels"]["Statistical Population"]["Data"]
 
     def finalize(self, all_data):
-        with open("results.json", "w") as fp:
+        # output directory to store json and image
+        output_dir = os.path.join(self.working_dir, "output")
+
+        with open(os.path.join(output_dir, "population.json"), "w") as fp:
             json.dump({s.id: v for s, v in all_data.items()}, fp)
 
         import matplotlib.pyplot as plt
@@ -32,4 +35,24 @@ def finalize(self, all_data):
         for pop in list(all_data.values()):
             ax.plot(pop)
         ax.legend([s.id for s in all_data.keys()])
-        fig.savefig("Figure.png")
+        fig.savefig(os.path.join(output_dir, "population.png"))
+
+    def initialize(self):
+        if not os.path.exists(os.path.join(self.working_dir, "output")):
+            os.mkdir(os.path.join(self.working_dir, "output"))
+
+    # idmtools analyzer
+    def map(self, data: 'Any', item: 'IItem') -> 'Any':
+        return data[self.filenames[0]]["Channels"]["Statistical Population"]["Data"]
+
+    def reduce(self, all_data: dict) -> 'Any':
+        output_dir = os.path.join(self.working_dir, "output")
+        with open(os.path.join(output_dir, "population.json"), "w") as fp:
+            json.dump({str(s.uid): v for s, v in all_data.items()}, fp)
+        import matplotlib.pyplot as plt
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+        for pop in list(all_data.values()):
+            ax.plot(pop)
+        ax.legend([str(s.uid) for s in all_data.keys()])
+        fig.savefig(os.path.join(output_dir, "population.png"))
diff --git a/examples/ssmt/generic_ssmt/files/idmtools.ini b/examples/ssmt/generic_ssmt/files/idmtools.ini
@@ -0,0 +1,102 @@
+[COMMON]
+# Number of threads idm-tools will use for analysis and other multi-threaded activities
+max_threads = 16
+
+# How many simulations per threads during simulation creation
+sims_per_thread = 20
+
+# Maximum number of LOCAL simulation ran simultaneously
+max_local_sims = 6
+
+# Maxium number of workers processing in parallel
+max_workers = 16
+
+# Maxium batch size to retrieve simulations
+batch_size = 10
+
+
+[COMPS]
+type = COMPS
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+max_workers = 16
+batch_size = 10
+work_item_type = DockerWorker
+docker_image = ubuntu18.04_python3.6_dtk-tools1.0b6
+plugin_key = 1.0.0.0_RELEASE
+
+
+[COMPS2]
+type = COMPS
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+max_workers = 16
+batch_size = 10
+work_item_type = DockerWorker
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+plugin_key = 1.0.0.0_RELEASE
+
+
+[SSMT]
+type = SSMT
+endpoint = https://comps2.idmod.org
+#endpoint = http://idmtvapp17
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+work_item_type = DockerWorker
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+#docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0.7
+plugin_key = 1.0.0.0_RELEASE
+
+
+[Local]
+type = Local
+
+# Path to the model executable
+exe_path = \\internal.idm.ctr\IDM-Test\home\mafisher\bin\Eradication.exe
+
+[Test]
+type = Test
+
+
+[Slurm]
+type = Slurm
+job_directory = /data
+mode = ssh
+remote_host = 192.168.208.6
+remote_user = test
+key_file = /home/clinton/development/work/idmtools/idmtools_platform_slurm/dockerized_slurm/id_rsa
+
+
+[BADTYPE]
+type = Bad
+
+
+[NOTYPE]
+
+
+[Logging]
+level = DEBUG
+console = on
+
+
+# This is a test we used to validate loading local from section block
+[Custom_Local]
+type = Local
diff --git a/examples/ssmt/generic_ssmt/files/run_analysis.py b/examples/ssmt/generic_ssmt/files/run_analysis.py
@@ -1,7 +1,8 @@
 import os
 import sys
 
-from MyAnalyzer import PopulationAnalyzer
+from idmtools.core import ItemType
+from idmtools.core.platform_factory import Platform
 
 try:
     # use idmtool image
@@ -10,11 +11,12 @@
     # use dtk-tools image
     from simtools.Analysis.AnalyzeManager import AnalyzeManager
 
-
-
 sys.path.append(os.path.dirname(__file__))
+from MyAnalyzer import PopulationAnalyzer
 
 if __name__ == "__main__":
+    platform = Platform('SSMT')
     analyzers = [PopulationAnalyzer()]
-    am = AnalyzeManager(exp_list=["8bb8ae8f-793c-ea11-a2be-f0921c167861"], analyzers=analyzers)
+    exp_id = "8bb8ae8f-793c-ea11-a2be-f0921c167861" # COMPS2 exp_id
+    am = AnalyzeManager(platform=platform, ids=[(exp_id, ItemType.EXPERIMENT)], analyzers=analyzers)
     am.analyze()
diff --git a/examples/ssmt/generic_ssmt/run_ssmt_analysis.py b/examples/ssmt/generic_ssmt/run_ssmt_analysis.py
@@ -8,8 +8,8 @@
 user_files = FileList(root='files')
 
 if __name__ == "__main__":
-    platform = Platform('COMPS2')
+    platform = Platform('SSMT')
     wi = SSMTWorkItem(item_name=wi_name, command=command, user_files=user_files,
-                      related_experiments=["8bb8ae8f-793c-ea11-a2be-f0921c167861"])
+                      related_experiments=["8bb8ae8f-793c-ea11-a2be-f0921c167861"]) # COMPS2 exp_id
     wim = WorkItemManager(wi, platform)
     wim.process(check_status=True)
diff --git a/examples/ssmt/hello_world/run_ssmt_task.py b/examples/ssmt/hello_world/run_ssmt_task.py
@@ -9,7 +9,7 @@
 tags = {'test': 123}
 
 if __name__ == "__main__":
-    platform = Platform('COMPS2')
+    platform = Platform('SSMT')
     wi = SSMTWorkItem(item_name=wi_name, command=command, user_files=user_files, tags=tags)
     wim = WorkItemManager(wi, platform)
     wim.process(check_status=True)
diff --git a/examples/ssmt/idmtools.ini b/examples/ssmt/idmtools.ini
@@ -45,13 +45,14 @@ exclusive = False
 max_workers = 16
 batch_size = 10
 work_item_type = DockerWorker
-docker_image = ubuntu18.04_python3.6_dtk-tools1.0b6
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
 plugin_key = 1.0.0.0_RELEASE
 
 
 [SSMT]
 type = SSMT
 endpoint = https://comps2.idmod.org
+#endpoint = http://idmtvapp17
 environment = Bayesian
 priority = Lowest
 simulation_root = $COMPS_PATH(USER)\output
@@ -60,7 +61,8 @@ num_retires = 0
 num_cores = 1
 exclusive = False
 work_item_type = DockerWorker
-docker_image = ubuntu18.04_python3.6_dtk-tools1.0b6
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+#docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0.7
 plugin_key = 1.0.0.0_RELEASE
 
 

diff --git a/examples/ssmt/simple_analysis/analyzers/AdultVectorsAnalyzer.py b/examples/ssmt/simple_analysis/analyzers/AdultVectorsAnalyzer.py
@@ -9,7 +9,6 @@
     from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
 
 import matplotlib as mpl
-
 mpl.use('Agg')
 
 
@@ -43,8 +42,20 @@ def finalize(self, all_data):
         fig.savefig(os.path.join(output_dir, "adult_vectors.png"))
 
     def map(self, data: 'Any', item: 'IItem') -> 'Any':
-        return None
+        return data[self.filenames[0]]["Channels"]["Adult Vectors"]["Data"]
 
     def reduce(self, all_data: dict) -> 'Any':
-        pass
+        output_dir = os.path.join(self.working_dir, "output")
+        with open(os.path.join(output_dir, "adult_vectors.json"), "w") as fp:
+            json.dump({str(s.uid): v for s, v in all_data.items()}, fp)
+
+        import matplotlib.pyplot as plt
+
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+
+        for pop in list(all_data.values()):
+            ax.plot(pop)
+        ax.legend([str(s.uid) for s in all_data.keys()])
+        fig.savefig(os.path.join(output_dir, "adult_vectors.png"))
 
diff --git a/examples/ssmt/simple_analysis/analyzers/PopulationAnalyzer.py b/examples/ssmt/simple_analysis/analyzers/PopulationAnalyzer.py
@@ -9,10 +9,8 @@
     from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
 
 import matplotlib as mpl
-
 mpl.use('Agg')
 
-
 class PopulationAnalyzer(BaseAnalyzer):
 
     def __init__(self, title='idm'):
@@ -43,8 +41,22 @@ def finalize(self, all_data):
         ax.legend([s.id for s in all_data.keys()])
         fig.savefig(os.path.join(output_dir, "population.png"))
 
+    def initialize(self):
+        if not os.path.exists(os.path.join(self.working_dir, "output")):
+            os.mkdir(os.path.join(self.working_dir, "output"))
+
+    # idmtools analyzer
     def map(self, data: 'Any', item: 'IItem') -> 'Any':
-        return None
+        return data[self.filenames[0]]["Channels"]["Statistical Population"]["Data"]
 
-    def reduce(self, all_data: dict) -> 'Any':
-        pass
+    def reduce(self, all_data: dict) ->'Any':
+        output_dir = os.path.join(self.working_dir, "output")
+        with open(os.path.join(output_dir, "population.json"), "w") as fp:
+            json.dump({str(s.uid): v for s, v in all_data.items()}, fp)
+        import matplotlib.pyplot as plt
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+        for pop in list(all_data.values()):
+            ax.plot(pop)
+        ax.legend([str(s.uid) for s in all_data.keys()])
+        fig.savefig(os.path.join(output_dir, "population.png"))
diff --git a/examples/ssmt/simple_analysis/idmtools.ini b/examples/ssmt/simple_analysis/idmtools.ini
@@ -0,0 +1,102 @@
+[COMMON]
+# Number of threads idm-tools will use for analysis and other multi-threaded activities
+max_threads = 16
+
+# How many simulations per threads during simulation creation
+sims_per_thread = 20
+
+# Maximum number of LOCAL simulation ran simultaneously
+max_local_sims = 6
+
+# Maxium number of workers processing in parallel
+max_workers = 16
+
+# Maxium batch size to retrieve simulations
+batch_size = 10
+
+
+[COMPS]
+type = COMPS
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+max_workers = 16
+batch_size = 10
+work_item_type = DockerWorker
+docker_image = ubuntu18.04_python3.6_dtk-tools1.0b6
+plugin_key = 1.0.0.0_RELEASE
+
+
+[COMPS2]
+type = COMPS
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+max_workers = 16
+batch_size = 10
+work_item_type = DockerWorker
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+plugin_key = 1.0.0.0_RELEASE
+
+
+[SSMT]
+type = SSMT
+endpoint = https://comps2.idmod.org
+#endpoint = http://idmtvapp17
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+work_item_type = DockerWorker
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+#docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0.7
+plugin_key = 1.0.0.0_RELEASE
+
+
+[Local]
+type = Local
+
+# Path to the model executable
+exe_path = \\internal.idm.ctr\IDM-Test\home\mafisher\bin\Eradication.exe
+
+[Test]
+type = Test
+
+
+[Slurm]
+type = Slurm
+job_directory = /data
+mode = ssh
+remote_host = 192.168.208.6
+remote_user = test
+key_file = /home/clinton/development/work/idmtools/idmtools_platform_slurm/dockerized_slurm/id_rsa
+
+
+[BADTYPE]
+type = Bad
+
+
+[NOTYPE]
+
+
+[Logging]
+level = DEBUG
+console = on
+
+
+# This is a test we used to validate loading local from section block
+[Custom_Local]
+type = Local