Merge feature/capgen into main as of 2024-03-08

.github/workflows/capgen_unit_tests.yaml

@@ -0,0 +1,17 @@
+name: Capgen Unit Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [feature/capgen, main]
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: update repos and install dependencies
+      run: sudo apt-get update && sudo apt-get install -y build-essential gfortran cmake python3 git
+    - name: Run unit tests
+      run: cd test && ./run_fortran_tests.sh
+

CMakeLists.txt

@@ -7,7 +7,7 @@ project(ccpp_framework
 #------------------------------------------------------------------------------
 # Set package definitions
 set(PACKAGE "ccpp-framework")
-set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber")
+set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Dustin Swales" "Courtney Peverley")
 string(TIMESTAMP YEAR "%Y")
 
 #------------------------------------------------------------------------------

CODEOWNERS

@@ -3,7 +3,7 @@
 
 # These owners will be the default owners for everything in the repo.
 #*       @defunkt
-*       @climbfuji @grantfirl @gold2718 @mkavulich
+*       @climbfuji @gold2718 @dustinswales @mwaxmonsky @peverwhee @grantfirl @mkavulich
 
 # Order is important. The last matching pattern has the most precedence.
 # So if a pull request only touches javascript files, only these owners

doc/HelloWorld/CMakeLists.txt

@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
 PROJECT(HelloWorld)
 ENABLE_LANGUAGE(Fortran)
 
@@ -166,7 +166,8 @@ list(APPEND DTABLE_CMD "${CCPP_CAP_FILES}/datatable.xml")
 list(APPEND DTABLE_CMD "--ccpp-files")
 list(APPEND DTABLE_CMD "--separator=\\;")
 string(REPLACE ";" " " DTABLE_STRING "${DTABLE_CMD}")
-MESSAGE(STATUS "Running: ${DTABLE_STRING}")
+string(STRIP ${DTABLE_STRING} DTABLE_STRING)
+MESSAGE(STATUS "Running: ${DTABLE_STRING};")
 EXECUTE_PROCESS(COMMAND ${DTABLE_CMD} OUTPUT_VARIABLE CCPP_CAPS
                 RESULT_VARIABLE RES
                 OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)

scripts/ccpp_capgen.py

@@ -15,6 +15,7 @@
 import logging
 import re
 # CCPP framework imports
+from ccpp_database_obj import CCPPDatabaseObj
 from ccpp_datafile import generate_ccpp_datatable
 from ccpp_suite import API
 from file_utils import check_for_writeable_file, remove_dir, replace_paths
@@ -26,11 +27,13 @@
 from host_model import HostModel
 from metadata_table import parse_metadata_file, SCHEME_HEADER_TYPE
 from parse_tools import init_log, set_log_level, context_string
+from parse_tools import register_fortran_ddt_name
 from parse_tools import CCPPError, ParseInternalError
 
 ## Capture the Framework root
-__SCRIPT_PATH = os.path.dirname(__file__)
-__FRAMEWORK_ROOT = os.path.abspath(os.path.join(__SCRIPT_PATH, os.pardir))
+_SCRIPT_PATH = os.path.dirname(__file__)
+_FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir))
+_SRC_ROOT = os.path.join(_FRAMEWORK_ROOT, "src")
 ## Init this now so that all Exceptions can be trapped
 _LOGGER = init_log(os.path.basename(__file__))
 
@@ -43,6 +46,11 @@
 ## Metadata table types where order is significant
 _ORDERED_TABLE_TYPES = [SCHEME_HEADER_TYPE]
 
+## CCPP Framework supported DDT types
+_CCPP_FRAMEWORK_DDT_TYPES = ["ccpp_hash_table_t",
+                             "ccpp_hashable_t",
+                             "ccpp_hashable_char_t"]
+
 ###############################################################################
 def delete_pathnames_from_file(capfile, logger):
 ###############################################################################
@@ -306,6 +314,17 @@ def compare_fheader_to_mheader(meta_header, fort_header, logger):
             lname = mvar.get_prop_value('local_name')
             arrayref = is_arrayspec(lname)
             fvar, find = find_var_in_list(lname, flist)
+            # Check for consistency between optional variables in metadata and
+            # optional variables in fortran. Error if optional attribute is
+            # missing from fortran declaration.
+            mopt  = mvar.get_prop_value('optional')
+            if find and mopt:
+                fopt = fvar.get_prop_value('optional')
+                if (not fopt):
+                    errmsg = 'Missing optional attribute in fortran declaration for variable {}, in file {}'
+                    errors_found = add_error(errors_found, errmsg.format(mname,title))
+                # end if
+            # end if
             if mind >= flen:
                 if arrayref:
                     # Array reference, variable not in Fortran table
@@ -559,7 +578,7 @@ def clean_capgen(cap_output_file, logger):
     set_log_level(logger, log_level)
 
 ###############################################################################
-def capgen(run_env):
+def capgen(run_env, return_db=False):
 ###############################################################################
     """Parse indicated host, scheme, and suite files.
     Generate code to allow host model to run indicated CCPP suites."""
@@ -578,12 +597,22 @@ def capgen(run_env):
         # Try to create output_dir (let it crash if it fails)
         os.makedirs(run_env.output_dir)
     # end if
+    # Pre-register base CCPP DDT types:
+    for ddt_name in _CCPP_FRAMEWORK_DDT_TYPES:
+        register_fortran_ddt_name(ddt_name)
+    # end for
+    src_dir = os.path.join(_FRAMEWORK_ROOT, "src")
     host_files = run_env.host_files
     host_name = run_env.host_name
     scheme_files = run_env.scheme_files
     # We need to create three lists of files, hosts, schemes, and SDFs
     host_files = create_file_list(run_env.host_files, ['meta'], 'Host',
                                   run_env.logger)
+    # The host model needs to know about the constituents module
+    const_mod = os.path.join(_SRC_ROOT, "ccpp_constituent_prop_mod.meta")
+    if const_mod not in host_files:
+        host_files.append(const_mod)
+    # end if
     scheme_files = create_file_list(run_env.scheme_files, ['meta'],
                                     'Scheme', run_env.logger)
     sdfs = create_file_list(run_env.suites, ['xml'], 'Suite', run_env.logger)
@@ -594,14 +623,21 @@ def capgen(run_env):
     # end if
     # First up, handle the host files
     host_model = parse_host_model_files(host_files, host_name, run_env)
+    # We always need to parse the ccpp_constituent_prop_ptr_t DDT
+    const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta")
+    if const_prop_mod not in scheme_files:
+        scheme_files = [const_prop_mod] + scheme_files
+    # end if
     # Next, parse the scheme files
     scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env)
-    ddts = host_model.ddt_lib.keys()
-    if ddts and run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
-        run_env.logger.debug("DDT definitions = {}".format(ddts))
+    if run_env.verbose:
+        ddts = host_model.ddt_lib.keys()
+        if ddts:
+            run_env.logger.debug("DDT definitions = {}".format(ddts))
+        # end if
     # end if
     plist = host_model.prop_list('local_name')
-    if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
+    if run_env.verbose:
         run_env.logger.debug("{} variables = {}".format(host_model.name, plist))
         run_env.logger.debug("schemes = {}".format([x.title
                                                     for x in scheme_headers]))
@@ -628,7 +664,8 @@ def capgen(run_env):
     cap_filenames = ccpp_api.write(outtemp_dir, run_env)
     if run_env.generate_host_cap:
         # Create a cap file
-        host_files = [write_host_cap(host_model, ccpp_api,
+        cap_module = host_model.ccpp_cap_name()
+        host_files = [write_host_cap(host_model, ccpp_api, cap_module,
                                      outtemp_dir, run_env)]
     else:
         host_files = list()
@@ -646,10 +683,13 @@ def capgen(run_env):
     # end if
     # Finally, create the database of generated files and caps
     # This can be directly in output_dir because it will not affect dependencies
-    src_dir = os.path.join(__FRAMEWORK_ROOT, "src")
     generate_ccpp_datatable(run_env, host_model, ccpp_api,
                             scheme_headers, scheme_tdict, host_files,
                             cap_filenames, kinds_file, src_dir)
+    if return_db:
+        return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api)
+    # end if
+    return None
 
 ###############################################################################
 def _main_func():
@@ -665,7 +705,7 @@ def _main_func():
     if framework_env.clean:
         clean_capgen(framework_env.datatable_file, framework_env.logger)
     else:
-        capgen(framework_env)
+        _ = capgen(framework_env)
     # end if (clean)
 
 ###############################################################################

scripts/ccpp_database_obj.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+"""
+Define the CCPPDatabaseObj object
+Object definition and methods to provide information from a run of capgen.
+"""
+
+from host_model import HostModel
+from ccpp_suite import API
+
+class CCPPDatabaseObjError(ValueError):
+    """Error class specific to CCPPDatabaseObj.
+    All uses of this error should be internal (i.e., programmer error,
+       not user error)."""
+
+    def __init__(self, message):
+        """Initialize this exception"""
+        super().__init__(message)
+
+class CCPPDatabaseObj:
+    """Object with data and methods to provide information from a run of capgen.
+    """
+
+    def __init__(self, run_env, host_model=None, api=None, database_file=None):
+        """Initialize this CCPPDatabaseObj.
+        If <database_file> is not None, all other inputs MUST be None and
+           the object is created from the database table created by capgen.
+        To initialize the object from an in-memory capgen run, ALL other
+           inputs MUST be passed (i.e., not None) and it is an error to pass
+           a value for <database_file>.
+        """
+
+        runtime_obj = all([host_model is not None, api is not None])
+        self.__host_model = None
+        self.__api = None
+        self.__database_file = None
+        if runtime_obj and database_file:
+            emsg = "Cannot provide both runtime arguments and database_file."
+        elif (not runtime_obj) and (not database_file):
+            emsg = "Must provide either database_file or all runtime arguments."
+        else:
+            emsg = ""
+        # end if
+        if emsg:
+            raise CCPPDatabaseObjError(f"ERROR: {emsg}")
+        # end if
+        if runtime_obj:
+            self.__host_model = host_model
+            self.__api = api
+        else:
+            self.db_from_file(run_env, database_file)
+        # end if
+
+    def db_from_file(self, run_env, database_file):
+        """Create the necessary internal data structures from a CCPP
+              datatable.xml file created by capgen.
+        """
+        metadata_tables = {}
+        host_name = "host"
+        self.__host_model = HostModel(metadata_tables, host_name, run_env)
+        self.__api = API(sdfs, host_model, scheme_headers, run_env)
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def host_model_dict(self):
+        """Return the host model dictionary for this CCPP DB object"""
+        if self.__host_model is not None:
+            return self.__host_model
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def suite_list(self):
+        """Return a list of suites built into the API"""
+        if self.__api is not None:
+            return list(self.__api.suites)
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def constituent_dictionary(self, suite):
+        """Return the constituent dictionary for <suite>"""
+        return suite.constituent_dictionary()
+
+    def call_list(self, phase):
+        """Return the API call list for <phase>"""
+        if self.__api is not None:
+            return self.__api.call_list(phase)
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")

scripts/ccpp_datafile.py

@@ -27,10 +27,6 @@
 from parse_tools import read_xml_file, PrettyElementTree
 from suite_objects import VerticalLoop, Subcycle
 
-# Find python version
-PY3 = sys.version_info[0] > 2
-PYSUBVER = sys.version_info[1]
-
 # Global data
 _INDENT_STR = "  "
 
@@ -652,12 +648,13 @@ def _new_var_entry(parent, var, full_entry=True):
     """Create a variable sub-element of <parent> with information from <var>.
     If <full_entry> is False, only include standard name and intent.
     """
-    prop_list = ["intent"]
+    prop_list = ["intent", "local_name"]
     if full_entry:
         prop_list.extend(["allocatable", "active", "default_value",
                           "diagnostic_name", "diagnostic_name_fixed",
                           "kind", "persistence", "polymorphic", "protected",
-                          "state_variable", "type", "units"])
+                          "state_variable", "type", "units", "molar_mass",
+                          "advected", "top_at_one", "optional"])
         prop_list.extend(Var.constituent_property_names())
     # end if
     ventry = ET.SubElement(parent, "var")
@@ -671,9 +668,13 @@ def _new_var_entry(parent, var, full_entry=True):
     if full_entry:
         dims = var.get_dimensions()
         if dims:
-            dim_entry = ET.SubElement(ventry, "dimensions")
-            dim_entry.text = " ".join(dims)
+            v_entry = ET.SubElement(ventry, "dimensions")
+            v_entry.text = " ".join(dims)
         # end if
+        v_entry = ET.SubElement(ventry, "source_type")
+        v_entry.text = var.source.ptype.lower()
+        v_entry = ET.SubElement(ventry, "source_name")
+        v_entry.text = var.source.name.lower()
     # end if
 
 ###############################################################################