Merge branch 'dev' of https://github.com/janhybs/ci-hpc into dev

cfg/flow123d/metacentrum/config.yaml

@@ -28,32 +28,33 @@ install:
       cd flow123d
       cp config/config-cihpc-meta.cmake config.cmake
       make -j16
-      bin/flow123d --help
+      bin/flow123d --version
 
 
 test:
   - name: integration
     enabled: true
-    output: stdout
+    output: 'null'
 
     repeat:
-      exactly: 5          # we want to have 5 repetitions by default
-      no-less-than: 15    # but with access to the db, we want to have 15 in total
+      exactly: 15          # we want to have 15 repetitions by default
+      no-less-thann: 15     # but with access to the db, we want to have 15 in total
 
     parallel:
-      cpus: all           # we want to use as many cpus as there are on this node
+      cpus: <cpu-value>   # we want to use as many cpus as there are on this node
       prop: <cpu>         # by default each 'worker' will use <cpu> of cpus
 
     # container: docker run --rm -v $(pwd):$(pwd) -w $(pwd) flow123d/flow-libs-dev-dbg:user %s
 
     collect:
-      files: <flow123d-workdir>/bench_data/**/profiler_info*.json
-      move-to: <flow123d-workdir>/artifacts/<__project__.start.datetime>
-      repo: <flow123d-home>
       save-to-db: true
-      module: artifacts.collect.modules.flow123d_profiler_module
+      files:    <flow123d-workdir>/bench_data/**/test_results-<__unique__>/**/profiler_info*.json
+      move-to:  <flow123d-workdir>/artifacts/<__project__.start.datetime>
+      repo:     <flow123d-home>
+      module:   artifacts.collect.modules.flow123d_profiler_module
 
       extra:
+        cpu-value:  <cpu-value>
         test-name:  <benchmark>
         mesh:       <mesh>
         weak:       <weak|i>
@@ -63,17 +64,13 @@ test:
 
     shell: |
       echo "BENCHMARK: <benchmark>, INPUT-DIR: <mesh>, CPU: <cpu>"
-      # mpiexec -np <cpu> flow123d/build_tree/bin/flow123d \
-      #       -s bench_data/benchmarks/<benchmark>/transport.yaml \
-      #       -i <mesh> &
-
-      # which mpiexec.hydra
       python3 flow123d/bin/python/runtest.py \
           bench_data/benchmarks/<benchmark>/ \
           --cpu <cpu>                        \
           --status-file                      \
           --no-clean                         \
           --no-compare                       \
+          --random-output-dir <__unique__>   \
           -a -- --input_dir <mesh> &
 
       pid=$!
@@ -113,22 +110,22 @@ test:
         - cpu:        [1          ,     2          ,     3         ]
         - weak:       [0          ,     0          ,     0         ]
         - strong:     [1          ,     1          ,     1         ]
-      # ------------------------------------------------------------------------
-      # 02_cube_123d
-      # weak scale
-      - values:
-        - benchmark:  02_cube_123d
-        - mesh:       [1_15786_el,      2_29365_el,      3_47367_el,      4_58803_el ]
-        - cpu:        [1         ,      2         ,      3         ,      4          ]
-        - weak:       [1         ,      1         ,      1         ,      1          ]
-        - strong:     [0         ,      0         ,      0         ,      1          ]
-      # strong scale
-      - values:
-        - benchmark:  02_cube_123d
-        - mesh:       [4_58803_el ,     4_58803_el ,     4_58803_el]
-        - cpu:        [1          ,     2          ,     3         ]
-        - weak:       [0          ,     0          ,     0         ]
-        - strong:     [1          ,     1          ,     1         ]
+      # # ------------------------------------------------------------------------
+      # # 02_cube_123d
+      # # weak scale
+      # - values:
+      #   - benchmark:  02_cube_123d
+      #   - mesh:       [1_15786_el,      2_29365_el,      3_47367_el,      4_58803_el ]
+      #   - cpu:        [1         ,      2         ,      3         ,      4          ]
+      #   - weak:       [1         ,      1         ,      1         ,      1          ]
+      #   - strong:     [0         ,      0         ,      0         ,      1          ]
+      # # strong scale
+      # - values:
+      #   - benchmark:  02_cube_123d
+      #   - mesh:       [4_58803_el ,     4_58803_el ,     4_58803_el]
+      #   - cpu:        [1          ,     2          ,     3         ]
+      #   - weak:       [0          ,     0          ,     0         ]
+      #   - strong:     [1          ,     1          ,     1         ]
       # ------------------------------------------------------------------------
       # 03_mesh_read
       # weak scale

cfg/flow123d/metacentrum/variables.yaml

@@ -1,6 +1,7 @@
 
 flow123d-workdir: /storage/praha1/home/jan-hybs/projects/ci-hpc/projects/flow123d
 flow123d-home:    /storage/praha1/home/jan-hybs/projects/ci-hpc/projects/flow123d/flow123d
+cpu-value: [20, 10, 5, 2, 1]
 
 hybs:
   flow123d-workdir: /home/jan-hybs/projects/ci-hpc/projects/flow123d

ci-hpc/caching/__init__.py

@@ -0,0 +1,27 @@
+#!/bin/python3
+# author: Jan Hybs
+
+from utils.glob import global_configuration
+try:
+    import cachetools
+
+    def cached():
+        return cachetools.cached(**cache_configuration())
+
+
+    def cache_configuration():
+        if global_configuration.cache_type:
+            cache = getattr(cachetools, global_configuration.cache_type)
+            if global_configuration.cache_opts:
+                return dict(cache=cache(**global_configuration.cache_opts))
+            return dict(cache=cache())
+        return dict(cache=None)
+
+except ImportError:
+
+    def cached():
+        def decorator(func):
+            def wrapper(*args, **kwargs):
+                return func(*args, **kwargs)
+            return wrapper
+        return decorator

ci-hpc/main.py

@@ -277,28 +277,7 @@ def main():
     global_configuration.project_name = project_name
     global_configuration.project_cfg_dir = project_dir
     global_configuration.project_cfg = config_path
-
-    # load config
-    project_config = cfgutil.configure_file(config_path, variables)
-    logger.debug('yaml configuration: \n%s', pad_lines(utils.strings.to_yaml(project_config)), skip_format=True)
-
-    # specify some useful global arguments which will be available in the config file
-    global_args_extra = {
-        'project-name': project_name,
-        'project-dir': project_dir,
-        'arg': dict(
-            branch=defaultdict(lambda: 'master', **dict(args.git_branch)),
-            commit=defaultdict(lambda: '', **dict(args.git_commit)),
-        )
-    }
-
-    # parse config
-    project_definition = Project(project_name, **project_config)
-    project_definition.update_global_args(global_args_extra)
-
-    project = ProcessProject(project_definition)
-    logger.info('processing project %s, section %s', project_name, args.step)
-
+
     # -----------------------------------------------------------------
 
     if 'watch' in args.step:

ci-hpc/utils/config.py

@@ -2,10 +2,11 @@
 # author: Jan Hybs
 
 import os
-from copy import deepcopy
+from copy import deepcopy, copy
 
 import yaml
 import re
+import itertools
 
 from utils.logging import logger
 
@@ -142,10 +143,25 @@ def load_config(path, replace=True, hostname_conditions=True):
 
 def configure_file(path, variables, convert=yaml_load, start='<', stop='>'):
     content = read_file(path)
+    iterable_keys = list()
+    iterable_vals = list()
+    rest = dict()
     for k, v in variables.items():
-        content = content.replace('%s%s%s' % (start, k, stop), v)
-
-    return convert(content)
+        if isinstance(v, list):
+            iterable_keys.append(k)
+            iterable_vals.append(v)
+        else:
+            rest[k] = v
+
+    for g in itertools.product(*iterable_vals):
+        single_config = rest.copy()
+        single_config.update(dict(zip(iterable_keys, g)))
+
+        single_content = copy(content)
+        for k, v in single_config.items():
+            single_content = single_content.replace('%s%s%s' % (start, k, stop), str(v))
+
+        yield convert(single_content)
 
 
 def configure_string(string, vars):

ci-hpc/utils/glob.py

@@ -27,6 +27,13 @@ class global_configuration(object):
     log_path = os.path.join(__root__, 'ci-hpc.log')
     log_style = 'short'
 
+    # default cache is for 10 entries or 10 minutes
+    cache_type = 'TTLCache'  # or LRUCache or None
+    cache_opts = dict(
+        maxsize=10,
+        ttl=10*60
+    )
+
     # this file should be PROTECTED, as it may contain passwords and database connection details
     cfg_secret_path = os.path.join(__cfg__, 'secret.yaml')
 

ci-hpc/utils/parallels.py

@@ -3,7 +3,7 @@
 
 # value which can be considered as plentiful cpus available
 
-PLENTY_OF_CPUS = 16
+PLENTY_OF_CPUS = 64  # turn off for now
 
 
 def extract_cpus_from_worker(worker):

ci-hpc/visualisation/www/plot/cfg/project_config.py → ci-hpc/visualisation/cfg/project_config.py

@@ -217,6 +217,7 @@ def __bool__(self):
     def __repr__(self):
         return '<%s>' % self.name
 
+
 class ProjectConfigFields(object):
     _result = namedtuple('Result', ['duration', 'returncode'])
     _git = namedtuple('Git', ['datetime', 'commit', 'branch'])