Ssmt test fix (#612)

* #575: fix experiment.simulations type issue

* #575: initial function implementation

* #575: code cleanup

* #575: add unit tests

* add missing __init__.py file for ssmt

* #575: add new feature: skip sims

* #575: add more detailed description for the filter method

* #575: move tests to comps' tests

* Update test_filter_simulations.py

* forgot test files, adding...

* fix typo in comment

* Update names for test

* update ssmt tests

* update tests

* update werkzeug version

* .gitignore

* #575: add unit tests

* $(git log --format=%B --reverse HEAD..HEAD@{1})

* fix werkzeug package issue

* add pytest mark for ssmt tests

Co-authored-by: ZDu-IDM <zdu@idmod.org>

.gitignore

@@ -377,27 +377,20 @@ examples/vistools/**/workitem_output/
 
 idmtools_core/tests/data
 idmtools_core/tests/reports/
-idmtools_platform_comps/tests/inputs/analyzers/population.csv
-idmtools_platform_comps/tests/inputs/analyzers/population.png
-idmtools_platform_comps/tests/inputs/analyzers/timeseries.csv
-idmtools_platform_comps/tests/inputs/analyzers/timeseries.png
-idmtools_platform_comps/tests/test_vistools_work_item/output_simulation/
-idmtools_platform_comps/tests/test_vistools_work_item/output_workitem/
+idmtools_platform_comps/tests/inputs/population.csv
+idmtools_platform_comps/tests/inputs/population.png
+idmtools_platform_comps/tests/inputs/timeseries.csv
+idmtools_platform_comps/tests/inputs/timeseries.png
 idmtools_platform_comps/**/task.diskcache/
-idmtools_platform_comps/tests/test_ssmt/output/
-idmtools_platform_comps/tests/output/
+idmtools_platform_comps/tests/output*/
 idmtools_platform_local/internals/ui/static/*
-idmtools_platform_comps/tests/test_vistools/output_workitem/
-idmtools_platform_comps/tests/test_vistools/output_simulation/
-idmtools_platform_comps/tests/test_ssmt/output_ssmt/
-idmtools_platform_comps/tests/test_ssmt/inputs/climate/dtk_*
-idmtools_platform_comps/tests/output_tag/
-idmtools_platform_comps/tests/output_csv/
-idmtools_platform_comps/tests/output_ssmt/
+idmtools_platform_comps/tests/test_vistools/output*/
+idmtools_platform_comps/tests/test_ssmt/output*/
+idmtools_platform_comps/tests/inputs/climate/*.json
+idmtools_platform_comps/tests/inputs/climate/*.bin
 idmtools_platform_local/idmtools_webui/build
 idmtools_platform_local/idmtools_platform_local/internals/ui/static/
 idmtools_test/idmtools_test/data/
-
 idmtools_platform_local/idmtools_webui/yarn.lock
 idmtools_model_emod/tests/reload_sim_download/
 idmtools_model_emod/tests/serialized_file_download/

idmtools_core/idmtools/ssmt/ssmt_analysis.py

@@ -43,7 +43,9 @@ def analyze(self, check_status=True):
         # save pickle file as a temp file
         temp_dir = tempfile.mkdtemp()
         temp_file = os.path.join(temp_dir, "analyzer_args.pkl")
-        pickle.dump(args_dict, open(temp_file, 'wb'))
+        file = open(temp_file, 'wb')
+        pickle.dump(args_dict, file)
+        file.close()
 
         # Add analyzer args pickle as additional file
         self.additional_files.add_file(temp_file)

idmtools_core/requirements.txt

@@ -6,4 +6,5 @@ pandas==0.24.2
 pipreqs==0.4.9
 pluggy~=0.12
 PyYaml~=5.1
-tabulate~=0.8.3
+tabulate~=0.8.3
+Werkzeug==0.16.1

idmtools_platform_comps/Makefile

@@ -52,6 +52,9 @@ test-emod: ## Run our emod tests
 test-python: ## Run our python tests
 	$(FULL_TEST_CMD) -m "python"'
 
+test-ssmt: ## Run our python tests
+	$(FULL_TEST_CMD) -m "ssmt"'
+
 coverage: ## Generate a code-coverage report
 	@make clean
 	# We have to run in our tests folder to use the proper config

idmtools_platform_comps/tests/idmtools.ini

@@ -34,10 +34,9 @@ exclusive = False
 max_workers = 16
 batch_size = 10
 work_item_type = DockerWorker
-docker_image = ubuntu18.04_python3.6_dtk-tools1.0b6
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
 plugin_key = 1.0.0.0_RELEASE
-test1 = 1
-test2 = 2
+
 
 [SSMT]
 type = SSMT
@@ -50,7 +49,7 @@ num_retires = 0
 num_cores = 1
 exclusive = False
 work_item_type = DockerWorker
-docker_image = ubuntu18.04_python3.6_dtk-tools1.0b6
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
 plugin_key = 1.0.0.0_RELEASE
 
 [Logging]

idmtools_platform_comps/tests/test_ssmt/inputs/AdultVectorsAnalyzer.py → idmtools_platform_comps/tests/inputs/AdultVectorsAnalyzer.py

@@ -1,15 +1,8 @@
 import json
 import os
 
-try:
-    # use idmtools image
-    from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
-except ImportError:
-    # use dtk-tools image
-    from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
-
+from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
 import matplotlib as mpl
-
 mpl.use('Agg')
 
 
@@ -43,8 +36,20 @@ def finalize(self, all_data):
         fig.savefig(os.path.join(output_dir, "adult_vectors.png"))
 
     def map(self, data: 'Any', item: 'IItem') -> 'Any':
-        return None
+        return data[self.filenames[0]]["Channels"]["Adult Vectors"]["Data"]
 
     def reduce(self, all_data: dict) -> 'Any':
-        pass
+        output_dir = os.path.join(self.working_dir, "output")
+        with open(os.path.join(output_dir, "adult_vectors.json"), "w") as fp:
+            json.dump({str(s.uid): v for s, v in all_data.items()}, fp)
+
+        import matplotlib.pyplot as plt
+
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+
+        for pop in list(all_data.values()):
+            ax.plot(pop)
+        ax.legend([str(s.uid) for s in all_data.keys()])
+        fig.savefig(os.path.join(output_dir, "adult_vectors.png"))
 

idmtools_platform_comps/tests/test_ssmt/inputs/CSVAnalyzer.py → idmtools_platform_comps/tests/inputs/CSVAnalyzer.py

@@ -3,12 +3,8 @@
 # First, import some necessary system and idmtools packages.
 import os
 import pandas as pd
-try:
-    # use idmtools image
-    from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
-except ImportError:
-    # use dtk-tools image
-    from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
+from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
+
 
 # Create a class for the analyzer
 class CSVAnalyzer(BaseAnalyzer):
@@ -33,7 +29,7 @@ def map(self, data, simulation):
 
     # In reduce, we are printing the simulation and result data filtered in map
     def reduce(self, all_data):
-
+        output_dir = os.path.join(self.working_dir, "output")
         # Let's hope the first simulation is representative
         first_sim = next(iter(all_data.keys()))  # Iterate over the dataframe keys
         exp_id = str(first_sim.experiment.uid)  # Set the exp id from the first sim data
@@ -43,10 +39,8 @@ def reduce(self, all_data):
                             names=['SimId'])  # Label the index keys you create with the names option
         results.index = results.index.droplevel(1)  # Remove default index
 
-        # Make a directory labeled the exp id to write the csv results to
-        os.makedirs(exp_id, exist_ok=True)
         # NOTE: If running twice with different filename, the output files will collide
-        results.to_csv(os.path.join(exp_id, self.__class__.__name__+'.csv'))
+        results.to_csv(os.path.join(output_dir, 'aggregated_c.csv'))
 
     #dtk-tools analyzer
     def select_simulation_data(self, data, simulation):

idmtools_platform_comps/tests/test_ssmt/inputs/PopulationAnalyzer.py → idmtools_platform_comps/tests/inputs/PopulationAnalyzer.py

@@ -2,22 +2,12 @@
 import os
 import pandas as pd
 
-# uncomment following lines with idmtools image of ssmt
-# from idmtools.analysis.analyze_manager import AnalyzeManager
-# from idmtools.core import ItemType
-# from idmtools.core.platform_factory import Platform
-
-try:
-    # use idmtools image
-    from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
-except ImportError:
-    # use dtk-tools image
-    from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
-
+from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
 import matplotlib as mpl
 
 mpl.use('Agg')
 
+
 class PopulationAnalyzer(BaseAnalyzer):
 
     def __init__(self, working_dir=".", name='idm'):

idmtools_platform_comps/tests/test_ssmt/inputs/SimpleAnalyzer.py → idmtools_platform_comps/tests/inputs/SimpleAnalyzer.py

@@ -1,13 +1,7 @@
 import json
 import os
 
-try:
-    # use idmtools image
-    from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
-except ImportError:
-    # use dtk-tools image
-    from simtools.Analysis.BaseAnalyzers.BaseAnalyzer import BaseAnalyzer
-
+from idmtools.entities.ianalyzer import IAnalyzer as BaseAnalyzer
 
 class SimpleAnalyzer(BaseAnalyzer):
 

idmtools_platform_comps/tests/inputs/idmtools.ini

@@ -0,0 +1,64 @@
+[COMMON]
+# Number of threads idm-tools will use for analysis and other multi-threaded activities
+max_threads = 16
+
+# How many simulations per threads during simulation creation
+sims_per_thread = 20
+
+# Maximum number of LOCAL simulation ran simultaneously
+max_local_sims = 6
+
+[COMPS]
+type = COMPS
+endpoint = https://comps.idmod.org
+environment = Belegost
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+test1 = 1
+test2 = 2
+
+[COMPS2]
+type = COMPS
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+max_workers = 16
+batch_size = 10
+work_item_type = DockerWorker
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+plugin_key = 1.0.0.0_RELEASE
+
+
+[SSMT]
+type = SSMT
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retires = 0
+num_cores = 1
+exclusive = False
+work_item_type = DockerWorker
+docker_image = idm-docker-staging.packages.idmod.org/idmtools_comps_ssmt_worker:0.3.0
+plugin_key = 1.0.0.0_RELEASE
+
+[Logging]
+level = DEBUG
+console = on
+
+# This is a test we used to validate loading local from section block
+[Custom_Local]
+type = Local
+
+# Path to the model executable
+exe_path = \\internal.idm.ctr\IDM-Test\home\mafisher\bin\Eradication.exe

idmtools_platform_comps/tests/test_ssmt/inputs/run_PopulationAnalyzer.py → idmtools_platform_comps/tests/inputs/run_PopulationAnalyzer.py

@@ -2,23 +2,20 @@
 import sys
 from sys import argv
 
-sys.path.append(os.path.dirname(__file__))
-
-try:
-    # use idmtool image
-    from idmtools.analysis.analyze_manager import AnalyzeManager
-except ImportError:
-    # use dtk-tools image
-    from simtools.Analysis.AnalyzeManager import AnalyzeManager
+from idmtools.core import ItemType
+from idmtools.core.platform_factory import Platform
+from idmtools.analysis.analyze_manager import AnalyzeManager
 
+sys.path.append(os.path.dirname(__file__))
 from PopulationAnalyzer import PopulationAnalyzer
 
 if __name__ == "__main__":
+    platform = Platform('SSMT')
     analyzers = [PopulationAnalyzer()]
     if len(sys.argv) >= 1:
         print(argv[1])
         expid = argv[1]
     else:
         expid = '8bb8ae8f-793c-ea11-a2be-f0921c167861'
-    am = AnalyzeManager(exp_list=[expid], analyzers=analyzers)
+    am = AnalyzeManager(platform=platform, ids=[(expid, ItemType.EXPERIMENT)], analyzers=analyzers)
     am.analyze()

idmtools_platform_comps/tests/test_ssmt/inputs/run_multiple_analyers.py → idmtools_platform_comps/tests/inputs/run_multiple_analyers.py

@@ -5,19 +5,19 @@
 """
 import os
 import sys
-from sys import argv
 
-sys.path.append(os.path.dirname(__file__))
+from idmtools.core import ItemType
+from idmtools.core.platform_factory import Platform
+
+from idmtools.analysis.analyze_manager import AnalyzeManager
 
-from simtools.Analysis.AnalyzeManager import AnalyzeManager
+sys.path.append(os.path.dirname(__file__))
 from PopulationAnalyzer import PopulationAnalyzer
 from AdultVectorsAnalyzer import AdultVectorsAnalyzer
 
 if __name__ == "__main__":
+    platform = Platform('SSMT')
     analyzers = [PopulationAnalyzer(), AdultVectorsAnalyzer()]
-    if len(sys.argv) > 1:
-        expid = argv[1]
-    else:
-        expid = '8bb8ae8f-793c-ea11-a2be-f0921c167861'
-    am = AnalyzeManager(exp_list=[expid], analyzers=analyzers)
-    am.analyze()
+    exp_id = "8bb8ae8f-793c-ea11-a2be-f0921c167861" # COMPS2 exp_id
+    am = AnalyzeManager(platform=platform, ids=[(exp_id, ItemType.EXPERIMENT)], analyzers=analyzers)
+    am.analyze()

idmtools_platform_comps/tests/inputs/run_multiple_exps.py

@@ -0,0 +1,25 @@
+import os
+import sys
+from sys import argv
+
+
+from idmtools.analysis.analyze_manager import AnalyzeManager
+from idmtools.core import ItemType
+from idmtools.core.platform_factory import Platform
+
+sys.path.append(os.path.dirname(__file__))
+from PopulationAnalyzer import PopulationAnalyzer
+
+if __name__ == "__main__":
+    platform = Platform('SSMT')
+
+    analyzers = [PopulationAnalyzer()]
+    if len(sys.argv) > 1:
+        expid = []
+        expid.append((argv[1], ItemType.EXPERIMENT))
+        expid.append((argv[2], ItemType.EXPERIMENT))
+    else:
+        expid = [('8bb8ae8f-793c-ea11-a2be-f0921c167861',ItemType.EXPERIMENT),
+                 ('4ea96af7-1549-ea11-a2be-f0921c167861',ItemType.EXPERIMENT)]
+    am = AnalyzeManager(platform=platform, ids=expid, analyzers=analyzers)
+    am.analyze()