#1193 assetize outputs

.github/workflows/pr-all-test-ubuntu-setup.yml

@@ -5,12 +5,12 @@ name: "pr-all-test-ubuntu-setup"
 on: [pull_request]
 
 jobs:
-  pr-run-all-test-ubuntu-setup:
+  pr-test-core-linux:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [self-hosted]
-        python-version: [3.6.12, 3.7.5, 3.8.2]
+        os: [self-linux]
+        python-version: ['3.6.x', '3.7.x', '3.8.x']
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v2
@@ -26,28 +26,87 @@ jobs:
             models:
               - 'idmtools_models/**'
             comps:
-              - 'idmtools_comps_platform/**'
+              - 'idmtools_platform_comps/**'
             local:
-              - 'idmtools_local_platform/**'
+              - 'idmtools_platform_local/**'
+      - uses: actions/cache@v2
+        if: steps.filter.outputs.core == 'true'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/*_requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
       - name: ${{ matrix.os }} Python ${{ matrix.python-version }}
-        uses: MaksimZhukov/setup-python@main
+        if: steps.filter.outputs.core == 'true'
+        uses: actions/setup-python@v2.1.4
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
       - name: Install Python dependencies
+        if: steps.filter.outputs.core == 'true'
         run: |
           pip install idm-buildtools flake8 wheel --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
-      - name: Run setup-dev
-        # only run if full install for core, local changes, or models
-        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.local == 'true' ||  steps.filter.outputs.models == 'true'
-        run: |
-          make setup-dev
-      - name: Run setup-dev
+      - name: Run setup-dev-no-docker
         # only run if core, local, models wasn't changed
-        if: steps.filter.outputs.core == 'false' &&  steps.filter.outputs.local == 'false' &&  steps.filter.outputs.models == 'false'
+        if: steps.filter.outputs.core == 'true'
         run: |
           make setup-dev-no-docker
       - name: login to comps2
+        if: steps.filter.outputs.core == 'true'
+        run: |
+          python dev_scripts/create_auth_token_args.py --username $COMPS_USER --password $COMPS_PASSWORD
+      - name: run idmtools_core smoke tests
+        # only run if core changed
+        if: steps.filter.outputs.core == 'true'
+        run: |
+          cd idmtools_core
+          make test-all
+      - name: Upload idmtools_core smoke test results
+        uses: actions/upload-artifact@v2
+        if: failure() && steps.filter.outputs.core == 'true'
+        with:
+          name: idmtools_core_test_results
+          path: idmtools_core/tests/reports/
+  pr-test-cli-linux:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ self-linux ]
+        python-version: [ 3.6.12, 3.7.9, 3.8.6 ]
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: dev
+          filters: |
+            core:
+              - 'idmtools_core/**'
+            cli:
+              - 'idmtools_cli/**'
+      - uses: actions/cache@v2
+        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.cli == 'true'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/*_requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: ${{ matrix.os }} Python ${{ matrix.python-version }}
+        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.cli == 'true'
+        uses: actions/setup-python@v2.1.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Install Python dependencies
+        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.cli == 'true'
+        run: |
+          pip install idm-buildtools flake8 wheel --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+      - name: Run setup-dev-no-docker
+        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.cli == 'true'
+        run: |
+          make setup-dev-no-docker
+      - name: login to comps2
+        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.cli == 'true'
         run: |
           python dev_scripts/create_auth_token_args.py --username $COMPS_USER --password $COMPS_PASSWORD
       - name: run idmtools_cli smoke tests
@@ -61,19 +120,56 @@ jobs:
         if: failure() && (steps.filter.outputs.core == 'true' ||  steps.filter.outputs.cli == 'true')
         with:
           name: idmtools_cli_test_results
-          path: idmtools_cli/tests/test_results.xml
-      - name: run idmtools_core smoke tests
-        # only run if core changed
-        if: steps.filter.outputs.core == 'true'
-        run: |
-          cd idmtools_core
-          make test-all
-      - name: Upload idmtools_core smoke test results
-        uses: actions/upload-artifact@v2
-        if: failure() && steps.filter.outputs.core == 'true'
+          path: idmtools_cli/tests/reports/
+  pr-test-models-linux:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ self-linux ]
+        python-version: [ 3.6.12, 3.7.9, 3.8.6 ]
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
         with:
-          name: idmtools_core_test_results
-          path: idmtools_core/tests/test_results.xml
+          base: dev
+          filters: |
+            core:
+              - 'idmtools_core/**'
+            cli:
+              - 'idmtools_cli/**'
+            models:
+              - 'idmtools_models/**'
+            comps:
+              - 'idmtools_platform_comps/**'
+            local:
+              - 'idmtools_platform_local/**'
+      - uses: actions/cache@v2
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/*_requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: ${{ matrix.os }} Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2.1.4
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Install Python dependencies
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          pip install idm-buildtools flake8 wheel --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+      - name: Run setup-dev-no-docker
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          make setup-dev-no-docker
+      - name: login to comps2
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          python dev_scripts/create_auth_token_args.py --username $COMPS_USER --password $COMPS_PASSWORD
       - name: run idmtools_models tests
         # only run if core changed
         if: steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true'
@@ -85,7 +181,55 @@ jobs:
         if: failure() && (steps.filter.outputs.core == 'true' || steps.filter.outputs.models == 'true')
         with:
           name: idmtools_models_test_results
-          path: idmtools_models/tests/test_results.xml
+          path: idmtools_models/tests/results/
+  pr-test-comps-linux:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ self-linux ]
+        python-version: [ 3.6.12, 3.7.9, 3.8.6 ]
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: dev
+          filters: |
+            core:
+              - 'idmtools_core/**'
+            cli:
+              - 'idmtools_cli/**'
+            models:
+              - 'idmtools_models/**'
+            comps:
+              - 'idmtools_platform_comps/**'
+            local:
+              - 'idmtools_platform_local/**'
+      - uses: actions/cache@v2
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/*_requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: ${{ matrix.os }} Python ${{ matrix.python-version }}
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        uses: actions/setup-python@v2.1.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Install Python dependencies
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          pip install idm-buildtools flake8 wheel --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+      - name: Run setup-dev-no-docker
+        run: |
+          make setup-dev-no-docker
+      - name: login to comps2
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          python dev_scripts/create_auth_token_args.py --username $COMPS_USER --password $COMPS_PASSWORD
       - name: run idmtools_platform_comps tests
         # only run if core, comps, or models changed
         if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
@@ -97,7 +241,57 @@ jobs:
         if: failure() && (steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true')
         with:
           name: idmtools_platform_comps_test_results
-          path: idmtools_platform_comps/tests/test_results.xml
+          path: idmtools_platform_comps/tests/results/
+  pr-test-local-linux:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ self-docker ]
+        python-version: [ 3.6.12, 3.7.9, 3.8.6 ]
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: dev
+          filters: |
+            core:
+              - 'idmtools_core/**'
+            cli:
+              - 'idmtools_cli/**'
+            models:
+              - 'idmtools_models/**'
+            comps:
+              - 'idmtools_platform_comps/**'
+            local:
+              - 'idmtools_platform_local/**'
+      - uses: actions/cache@v2
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/*_requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: ${{ matrix.os }} Python ${{ matrix.python-version }}
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        uses: actions/setup-python@v2.1.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Install Python dependencies
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          pip install idm-buildtools flake8 wheel --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+      - name: Run setup-dev
+        # only run if full install for core, local changes, or models
+        if: steps.filter.outputs.core == 'true' ||  steps.filter.outputs.local == 'true' ||  steps.filter.outputs.models == 'true'
+        run: |
+          make setup-dev
+      - name: login to comps2
+        if: steps.filter.outputs.core == 'true' || steps.filter.outputs.comps == 'true' || steps.filter.outputs.models == 'true'
+        run: |
+          python dev_scripts/create_auth_token_args.py --username $COMPS_USER --password $COMPS_PASSWORD
       - name: run idmtools_platform_local smoke tests
         # only run if core and comps changed
         if: steps.filter.outputs.core == 'true' || steps.filter.outputs.local == 'true' || steps.filter.outputs.models == 'true'
@@ -109,4 +303,4 @@ jobs:
         if: failure() && (steps.filter.outputs.core == 'true' || steps.filter.outputs.local == 'true' || steps.filter.outputs.models == 'true')
         with:
           name: idmtools_platform_local_test_results
-          path: idmtools_platform_local/tests/test_results.xml
+          path: idmtools_platform_local/tests/results/

dev_scripts/serve_docs.py

@@ -17,7 +17,7 @@
 server = Server()
 dirs_to_watch = set()
 # Find all the directories within docs to watch
-for filepath in glob.iglob("**.rst", recursive=True):
+for filepath in glob.glob("**/*.rst", recursive=True):
     if 'idmtools_' not in filepath:
         d = os.path.dirname(filepath)
         if d not in dirs_to_watch:
@@ -26,6 +26,6 @@
 server.watch('docs/__static/*', shell('make html', cwd='docs'), delay=5)
 server.watch('docs/images/*', shell('make html', cwd='docs'), delay=5)
 server.watch('docs/*.txt', shell('make html', cwd='docs'), delay=5)
-server.watch('docs/*.py', shell('make html', cwd='docs'), delay=5)
+server.watch('*.py', shell('make html', cwd='docs'), delay=5)
 webbrowser.open("http://localhost:8000")
 server.serve(root='docs/_build/html', port=args.port)

dev_scripts/test_root.mk

@@ -19,7 +19,7 @@ TEST_EXTRA_OPTS?=
 DOCKER_REPO?=docker-staging
 NO_SPINNER?=1
 PARALLEL_TEST_COUNT?=8
-TEST_COMMAND =  py.test --junitxml=$(REPORT_DIR)/$(TEST_REPORT) --html=$(REPORT_DIR)/$(HTML_TEST_REPORT) --self-contained-html $(TEST_EXTRA_OPTS)
+TEST_COMMAND =  py.test --timeout=600 --junitxml=$(REPORT_DIR)/$(TEST_REPORT) --html=$(REPORT_DIR)/$(HTML_TEST_REPORT) --self-contained-html $(TEST_EXTRA_OPTS)
 COVERAGE_OPTS := --cov-config=.coveragerc --cov-branch --cov-append --cov=idmtools --cov=idmtoools_cli --cov=idmtools_models --cov=idmtools_platform_comps --cov=idmtools_platform_local --cov=../../idmtools_cli/
 
 

docs/analyzers/analyzers-ssmt.rst

@@ -8,7 +8,7 @@ The :py:class:`idmtools.analysis.platform_analysis.PlatformAnalysis` class is us
 
 The following example, run_ssmt_analysis.py, shows how to use :py:class:`idmtools.analysis.platform_analysis.PlatformAnalysis` for running analysis on |SSMT_s|:
 
-    .. literalinclude:: ../examples/ssmt/simple_analysis/run_ssmt_analysis.py
+    .. literalinclude:: ../../examples/ssmt/simple_analysis/run_ssmt_analysis.py
 
 In this example two analyzers are run on an existing experiment with the output results saved to an output directory. After you run the example you can see the results by using the returned SSMTWorkItem id and searching for it under **Work Items** in |COMPS_s|.
 

docs/cli/cli-comps.rst

@@ -0,0 +1,17 @@
+.. _COMPS CLI:
+
+=====
+COMPS
+=====
+
+The COMPS platform related commands can be accessed with either ``idmtools comps`` or ``comps-cli``. All comps command require a target configuration block or alias to use to configure the connection to COMPS. See the details of the top level command below for detailed help:
+
+.. command-output:: idmtools comps --help
+
+You can login to a comps environment by using the ``idmtools comps CONFIG_BLOCK login`` command. See the help below:
+
+.. command-output:: idmtools comps CONFIG_BLOCK login --help
+
+You can assetize outputs from the CLI by running ``idmtools comps CONFIG_BLOCK assetize-outputs``:
+
+.. command-output:: idmtools comps CONFIG_BLOCK assetize-outputs --help

docs/cli/cli_index.rst

@@ -10,6 +10,7 @@ CLI reference
    cli-simulations
    cli-experiments
    cli-platforms
+   cli-comps
    cli-examples
    cli-troubleshoot
 

docs/cli/wizard.rst

@@ -13,4 +13,4 @@ Use the ``block`` command to start the wizard that will guide you through the cr
 
 Here is a demo of the command in action
 
-.. image:: images/config-wizard.svg
+.. image:: /images/config-wizard.svg

docs/configuration.rst

@@ -14,4 +14,4 @@ The configuration of |IT_s| is set in the idmtools.ini file. This file is normal
 
 Below is an example configuration file:
 
-.. literalinclude:: ../examples/idmtools.ini
+.. literalinclude:: idmtools.ini