Drop unittest for integration and system tests

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "spack"]
 	path = spack
-	url = https://github.com/dominichofer/spack.git
+	url = https://github.com/spack/spack.git

Jenkinsfile

@@ -61,7 +61,7 @@ pipeline {
                         steps {
                             sh """
                             source env/bin/activate
-                            pytest -v -n auto --maxprocesses=12 --scope \"""" + env.ghprbCommentBody + " parallel\"  test/system_test.py"
+                            pytest -v -n auto --maxprocesses=24 --scope \"""" + env.ghprbCommentBody + " parallel\"  test/system_test.py"
                         }
                     }
                     stage('System Tests Serial') {
@@ -70,7 +70,7 @@ pipeline {
                             mkdir -p log/${NODENAME}/system_test
                             touch log/${NODENAME}/system_test/serial_test_run
                             source env/bin/activate
-                            pytest -v -n auto --maxprocesses=12 --scope \"""" + env.ghprbCommentBody + " serial\" test/system_test.py"
+                            pytest -v -n auto --maxprocesses=24 --scope \"""" + env.ghprbCommentBody + " serial\" test/system_test.py"
                         }
                     }
                 }

README.md

@@ -24,6 +24,7 @@ Spack is the package manager used by C2SM and MeteoSwiss to install and deploy s
   * [spack-c2sm v0.18.1.1](https://C2SM.github.io/spack-c2sm/v0.18.1.1) [deprecated]
 
 **General infos about spack**
+  * [Official spack v0.21.1](https://spack.readthedocs.io/en/v0.21.1/) 
   * [Official spack v0.20.1](https://spack.readthedocs.io/en/v0.20.1/) 
   * [Official spack v0.18.1](https://spack.readthedocs.io/en/v0.18.1/) 
 

repos/c2sm/packages/fdb/package.py

@@ -1,28 +1,21 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
 from spack.package import *
+
 from spack.pkg.builtin.fdb import Fdb as SpackFdb
 
 
 class Fdb(SpackFdb):
-    """FDB (Fields DataBase) is a domain-specific object store developed at
-    ECMWF for storing, indexing and retrieving GRIB data."""
 
-    version("5.11.17",
-            sha256=
-            "375c6893c7c60f6fdd666d2abaccb2558667bd450100817c0e1072708ad5591e")
-
-    depends_on("ecbuild@3.7:", type="build", when="@5.11.6:")
+    # This section can be removed when the following commit
+    # https://github.com/spack/spack/commit/8871bd5ba5c58562b8c20baa00f125aeccba586f
+    # is included in a release on spack and this is used by spack-c2sm.
+    depends_on("eckit@1.24.4:", when="@5.11.22:")
 
+    # This section can be removed when the following PR
+    # https://github.com/spack/spack/pull/42874
+    # is included in a release on spack and this is used by spack-c2sm.
     @property
     def libs(self):
         return find_libraries("libfdb5",
                               root=self.prefix,
                               shared=True,
                               recursive=True)
-
-    def setup_build_environment(self, env):
-        env.set('CTEST_OUTPUT_ON_FAILURE', 1)

repos/c2sm/packages/flexpart-cosmo/package.py

@@ -43,6 +43,7 @@ def setup_build_environment(self, env):
     def build(self, spec, prefix):
 
         with working_dir(self.build_directory):
+            make.jobs = 1
             make('-f', self.makefile_file)
 
     def install(self, spec, prefix):

repos/c2sm/packages/infero/package.py

@@ -29,8 +29,8 @@ class Infero(CMakePackage):
     variant('tf_c', description='Enable tensorflow-c backend', default=False)
     variant('onnx', description='Enable ONNX backend', default=False)
 
-    depends_on('eckit@1.20.0:')
-    depends_on('fckit')
+    depends_on('eckit@1.20.2')
+    depends_on('fckit@0.9.0')
     depends_on('ecbuild', type=('build'))
     depends_on('tensorflowc', when='+tf_c')
     depends_on('onnx-runtime', when='+onnx')

repos/c2sm/packages/metkit/package.py

@@ -0,0 +1,10 @@
+from spack.package import *
+
+from spack.pkg.builtin.metkit import Metkit as SpackMetkit
+
+
+class Metkit(SpackMetkit):
+
+    # This file can be removed when this PR https://github.com/spack/spack/pull/42871
+    # is included in a release on spack and this is used by spack-c2sm.
+    depends_on("eckit@:1.21", when="@:1.10")

repos/c2sm/packages/py-cmake/package.py

@@ -25,6 +25,8 @@ class PyCmake(PythonPackage):
             sha256=
             '52b98c5ee70b5fa30a8623e96482227e065292f78794eb085fdf0fecb204b79b')
 
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
     depends_on('ninja', type='build')
     depends_on('py-scikit-build@0.12:', type='build')
     depends_on('py-setuptools@42:', type='build')

repos/c2sm/packages/py-cytoolz/package.py

@@ -17,13 +17,20 @@ class PyCytoolz(PythonPackage):
 
     maintainers = ['samkellerhals']
 
+    version('0.12.3',
+            sha256=
+            '4503dc59f4ced53a54643272c61dc305d1dbbfbd7d6bdf296948de9f34c3a282')
     version('0.12.0',
             sha256=
             'c105b05f85e03fbcd60244375968e62e44fe798c15a3531c922d531018d22412')
 
     depends_on('py-setuptools', type='build')
     depends_on('py-cython', type='build')
 
+    # py-cytoolz@0.12.0 not compatible with py-cython@3:, see
+    # https://www.layerzrozero.network/?_=%2Fpytoolz%2Fcytoolz%2Fissues%2F202%23w2n%2BddnGqHIZTHHkluHJC3Vn
+    depends_on('py-cython@:2', when='@0.12.0', type='build')
+
     depends_on('python@3.5:', type=('build', 'run'))
     depends_on('py-toolz@0.8.0:', type=('build', 'run'))
 

repos/c2sm/packages/py-frozendict/package.py

@@ -4,22 +4,19 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack import *
+from spack.pkg.builtin.py_frozendict import PyFrozendict as SpackPyFrozendict
 
 
-class PyFrozendict(PythonPackage):
-    """frozendict is a simple immutable dictionary."""
+class PyFrozendict(SpackPyFrozendict):
 
-    homepage = "https://github.com/Marco-Sulla/python-frozendict"
-
-    pypi = 'frozendict/frozendict-2.3.4.tar.gz'
-
-    maintainers = ['samkellerhals']
-
-    # FIXME: Add proper versions and checksums here.
-    version('2.3.4',
+    version('2.4.0',
             sha256=
-            '15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78')
-
-    depends_on('python@3.6:', type=('build', 'run'))
-
-    depends_on('py-setuptools', type='build')
+            'c26758198e403337933a92b01f417a8240c954f553e1d4b5e0f8e39d9c8e3f0a')
+
+    # TODO: remove this extension once we have a more recent
+    # version than v0.21.1
+    def setup_build_environment(self, env):
+        # C extension is not supported for 3.11+. See also
+        # https://github.com/Marco-Sulla/python-frozendict/issues/68
+        if self.spec.satisfies("^python@3.11:"):
+            env.set("FROZENDICT_PURE_PY", "1")

repos/c2sm/packages/py-gt4py/package.py

@@ -58,7 +58,7 @@ class PyGt4py(PythonPackage):
     depends_on('py-lark@1.1.2:', type=('build', 'run'))
     depends_on('py-mako@1.1:', type=('build', 'run'))
     depends_on('py-ninja@1.10:', type=('build', 'run'))
-    depends_on('py-numpy@1.24.2: ~blas ~lapack', type=('build', 'run'))
+    depends_on('py-numpy@1.24.2:', type=('build', 'run'))
     depends_on('py-packaging@20.0:', type=('build', 'run'))
 
     # versions later than 2.9.2 fail to pick to right Python version
@@ -68,7 +68,12 @@ class PyGt4py(PythonPackage):
 
     depends_on('py-nanobind@1.4.0:', when="@1.0.1.3:", type=('build', 'run'))
     depends_on('py-tabulate@0.8.10:', type=('build', 'run'))
-    depends_on('py-typing-extensions@4.2:4.6', type=('build', 'run'))
+    depends_on('py-typing-extensions@4.5.0',
+               when="@:1.0.3.5",
+               type=('build', 'run'))
+    depends_on('py-typing-extensions@4.10.0',
+               when="@1.0.3.6:",
+               type=('build', 'run'))
     depends_on('py-toolz@0.12.0:', type=('build', 'run'))
     depends_on('py-xxhash@1.4.4:3.0.9', type=('build', 'run'))
     depends_on('py-hypothesis@6.0.0:', type=('build', 'run'))

repos/c2sm/packages/py-typing-extensions/package.py

@@ -5,6 +5,9 @@
 class PyTypingExtensions(SpackPyTypingExtensions):
     """Backported and Experimental Type Hints for Python 3.7+"""
 
+    version("4.10.0",
+            sha256=
+            "b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb")
     version("4.5.0",
             sha256=
             "5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb")

repos/c2sm/packages/python/package.py

@@ -0,0 +1,5 @@
+from spack.pkg.builtin.python import Python as SpackPython
+
+
+class Python(SpackPython):
+    variant("ssl", default=False, description="Build ssl module")

src/__init__.py

@@ -2,4 +2,4 @@
 from .machine import machine_name
 from .spack_commands import log_with_spack
 from .github import GitHubRepo, Markdown, HTML
-from .scope import all_machines, all_packages, explicit_scope, package_triggers
+from .scope import all_machines, all_packages, explicit_scope, package_triggers, all_packages_underscore

src/report_tests.py

@@ -41,8 +41,6 @@ def __str__(self) -> str:
     # Trigger phrases that cause a test to get a special icon and comment.
     # List[(trigger, icon, comment)]
     triggers = [
-        ('AssertionError exception when releasing read lock', ':lock:',
-         'spack locking problem'),
         ('Timed out waiting for a write lock', ':lock:',
          'spack write lock problem'),
         ('Timed out waiting for a read lock', ':lock:',

src/scope.py

@@ -9,6 +9,7 @@
         os.path.join(spack_c2sm_path, 'repos/c2sm/packages')) if os.path.isdir(
             os.path.join(spack_c2sm_path, 'repos/c2sm/packages', name))
 ]
+all_packages_underscore = [p.replace('-', '_') for p in all_packages]
 
 
 def explicit_scope(scope: str) -> list:
@@ -24,14 +25,10 @@ def explicit_scope(scope: str) -> list:
 
 
 def package_triggers(scope: list) -> list:
-    """
-    Transforms ['package-name'] to ['packagenametest', 'test_package_name']
-    so they match with the naming convenction of testcases and tests.
-    """
-
-    active_packages = [x for x in all_packages if x in scope]  # intersection
-    active_testcases = [
-        x.replace('-', '').replace('_', '') + 'test' for x in active_packages
-    ]
-    active_tests = ['test_' + x.replace('-', '_') for x in active_packages]
-    return active_testcases + active_tests
+    triggers = []
+    for x, y in zip(all_packages, all_packages_underscore):
+        if x in scope:
+            triggers.append(x)
+            triggers.append(y)
+
+    return triggers

sysconfigs/balfrin/config.yaml

@@ -1,6 +1,4 @@
 config:
-  extensions:
-    - '$spack/../tools/spack-scripting'
   build_stage: '$spack/../spack-build_stage'
   test_stage: '$spack/../spack-test_stage'
   misc_cache: '$spack/../spack-misc_cache'

sysconfigs/daint/config.yaml

@@ -1,6 +1,4 @@
 config:
-  extensions:
-    - '$spack/../tools/spack-scripting'
   build_stage: '$spack/../spack-build_stage'
   test_stage: '$spack/../spack-test_stage'
   misc_cache: '$spack/../spack-misc_cache'

sysconfigs/daint/packages.yaml

@@ -58,7 +58,6 @@ packages:
   cosmo: # manually added
     variants: slave=daint cuda_arch=60
   cosmo-dycore: # manually added
-    compiler: [gcc@8.3.0]
     variants: slave=daint cuda_arch=60 data_path=/scratch/snx3000/jenkins/data/cosmo/
   cuda: # manually added
     buildable: false