Fix issues related to database and setting migration

This is a big one, sorry.
 - Adds an !include tag to the YAML parser, allowing blueprints to refer to
   external files when useful. These are resolved early on, and the fully-formed
   document can be re-serialized, or piped into the database for completeness.
 - Removes the `latticeFile` attribute from the grid blueprints. This was done
   because it makes the set of input files boundless, which leads to issues in
   non-file-based approaches to dealing with inputs (e.g., streams, database
   storage, test fixtures, etc.)
 - Re-arranges some of the migrations from the geometry.py-based methods of
   expressing core layout. SystemLayoutInput objects now have a method to
   render themselves as gridBlueprints.
 - In support of the above, the radial and axial sub-meshing in the geom files
   had to be moved to the assembly blueprint, alongside the axialMeshPoints.
   These should all actually be whipped up from settings, since they are
   ultimately physics-related (nodal neutronics), but at least for now they are
   in the same place. The migration capability to move these over is a bit
   simplistic right now, as to be able to migrate a full-complexity geom file to
   the blueprints with meshing parameters would require complex modifications to
   the assembly blueprints.

MANIFEST.in

@@ -42,7 +42,7 @@ include armi/tests/refOneBlockReactor.yaml
 include armi/tests/refSmallCartesian.yaml
 include armi/tests/refSmallReactor.yaml
 include armi/tests/refTestCartesian.yaml
-include armi/tests/sfpGeom.xml
+include armi/tests/sfpGeom.yaml
 include armi/tests/testEq.yaml
 include armi/tests/testEqMulti-geom.xml
 include armi/tests/testEqSingle-geom.xml

armi/__init__.py

@@ -338,10 +338,16 @@ def _cleanupOnCancel(signum, _frame):
     sys.exit(1)  # since we're handling the signal we have to cancel
 
 
-def configure(app: apps.App):
+def configure(app: Optional[apps.App]=None):
     """
     Set the plugin manager for the Framework and configure internals to those plugins.
 
+    Parameters
+    ----------
+    app :
+        An :py:class:`armi.apps.App` instance with which the framework is to be
+        configured. If it is not provided, then the default ARMI App will be used.
+
     Important
     ---------
     Since this affects the behavior of several modules at their import time, it is
@@ -363,6 +369,8 @@ def configure(app: apps.App):
         )
     )
 
+    app = app or apps.App()
+
     global _app
     _app = app
 

armi/bookkeeping/db/database3.py

@@ -38,6 +38,7 @@
 import io
 import itertools
 import os
+import pathlib
 import re
 import sys
 import time
@@ -73,6 +74,7 @@
 from armi.bookkeeping.db.types import History, Histories
 from armi.bookkeeping.db import database
 from armi.reactor import geometry
+from armi.utils.textProcessors import resolveMarkupInclusions
 
 ORDER = interfaces.STACK_ORDER.BOOKKEEPING
 DB_VERSION = "3.1"
@@ -155,15 +157,12 @@ def initDB(self):
 
         # Grab geomString here because the DB-level has no access to the reactor or
         # blueprints or anything.
-        # There's not always a geomFile; sometimes geom is brought in via systems.
-        # Eventually, we'll need to store multiple of these (one for each system).
-        # Just do core for now.
-        geomFileName = self.r.blueprints.gridDesigns["core"].latticeFile
-        if geomFileName:
-            with open(
-                os.path.join(os.path.dirname(self.cs.path), geomFileName), "r"
-            ) as fileStream:
-                geomString = fileStream.read()
+        # There's not always a geomFile; we are moving towards the core grid definition
+        # living in the blueprints themselves. In this case, the db doesnt need to store
+        # a geomFile at all.
+        if self.cs["geomFile"]:
+            with open(os.path.join(self.cs.inputDirectory, self.cs["geomFile"])) as f:
+                geomString = f.read()
         else:
             geomString = ""
         self._db.writeInputsToDB(self.cs, geomString=geomString)
@@ -192,7 +191,7 @@ def interactEOC(self, cycle=None):
     def interactEOL(self):
         """DB's should be closed at run's end. """
         # minutesSinceStarts should include as much of the ARMI run as possible so EOL
-        # is necessary  too.
+        # is necessary, too.
         self.r.core.p.minutesSinceStart = (time.time() - self.r.core.timeOfStart) / 60.0
         self._db.writeToDB(self.r)
         self._db.close(True)
@@ -604,10 +603,10 @@ def writeInputsToDB(self, cs, csString=None, geomString=None, bpString=None):
             csString = stream.read()
 
         if bpString is None:
-            with open(
-                os.path.join(os.path.dirname(cs.path), cs["loadingFile"]), "r"
-            ) as fileStream:
-                bpString = fileStream.read()
+            # Ensure that the input as stored in the DB is complete
+            bpString = resolveMarkupInclusions(
+                pathlib.Path(cs.inputDirectory) / cs["loadingFile"]
+            ).read()
 
         self.h5db["inputs/settings"] = csString
         self.h5db["inputs/geomFile"] = geomString

armi/bookkeeping/db/xtviewDB.py

@@ -1012,7 +1012,8 @@ def getAuxiliaryDataPath(self, ts: Tuple[int, int], name: str) -> str:
 
     def _getParamNames(self, objName):
         # TODO: add a set of names as attributes somewhere so it's reliably exhaustive?
-        # using the last TS currently to get all parameters defined, iterating across each entry is way too slow
+        # using the last TS currently to get all parameters defined, iterating across
+        # each entry is way too slow
         # TODO: should allow a specific TS lookup in the case of loadState
         last_ts = self.getAllTimesteps()[-1]
         return list(self._hdf_file["{}/{}".format(last_ts, objName)].keys())
@@ -1040,6 +1041,7 @@ def readReactorParam(self, param, ts=None):
                     "Reactor/Core parameter `{}` was unrecognized and is being "
                     "ignored.".format(param)
                 )
+
         all_vals = []
         for timestep in timesteps:
             value = self._get_1d_dataset("{}/reactors/{}".format(timestep, param))

armi/bookkeeping/mainInterface.py

@@ -97,9 +97,10 @@ def _activateDB(self):
                 dbi.prepRestartRun(dbCycle, dbNode)
             except:
                 runLog.error(
-                    "Could not load state at BOL as requested. DB {0} does "
-                    "not exist or does not have enough time steps to load this time"
-                    "".format(self.cs["reloadDBName"])
+                    "Could not load the initial state as requested. DB `{0}` does "
+                    "not exist or does not have enough time steps to load this time "
+                    "(cycle={}, tn={})"
+                    "".format(self.cs["reloadDBName"], dbCycle, dbNode)
                 )
                 raise
             self.r.p.cycle = self.cs["startCycle"]

armi/cases/case.py

@@ -25,6 +25,7 @@
 """
 import cProfile
 import os
+import pathlib
 import pstats
 import re
 import sys
@@ -52,6 +53,7 @@
 from armi.utils import pathTools
 from armi.utils.directoryChangers import DirectoryChanger
 from armi.utils.directoryChangers import ForcedCreationDirectoryChanger
+from armi.utils import textProcessors
 from armi.nucDirectory import nuclideBases
 
 # change from default .coverage to help with Windows dotfile issues.
@@ -498,9 +500,8 @@ def clone(self, additionalFiles=None, title=None, modifiedSettings=None):
         """
         Clone existing ARMI inputs to current directory with optional settings modifications.
 
-        Since each case depends on multiple inputs, this is a safer
-        way to move cases around without having to wonder if you
-        copied all the files appropriately.
+        Since each case depends on multiple inputs, this is a safer way to move cases
+        around without having to wonder if you copied all the files appropriately.
 
         Parameters
         ----------
@@ -559,11 +560,21 @@ def clone(self, additionalFiles=None, title=None, modifiedSettings=None):
 
         copyInterfaceInputs(self.cs, clone.cs.inputDirectory)
 
-        for grid in self.bp.gridDesigns or []:
-            if grid.latticeFile:
+        with open(self.cs["loadingFile"], "r") as f:
+            for includePath, mark in textProcessors.findYamlInclusions(
+                f, root=pathlib.Path(self.cs.inputDirectory)
+            ):
+                if not includePath.exists():
+                    raise OSError(
+                        "The input file file `{}` referenced at {} does not exist.".format(
+                            includePath, mark
+                        )
+                    )
                 pathTools.copyOrWarn(
-                    "system lattice for {}".format(grid.name),
-                    fromPath(grid.latticeFile),
+                    "auxiliary input file `{}` referenced at {}".format(
+                        includePath, mark
+                    ),
+                    fromPath(includePath),
                     clone.cs.inputDirectory,
                 )
 

armi/reactor/blueprints/__init__.py

@@ -66,14 +66,14 @@
 import collections
 from collections import OrderedDict
 import os
+import pathlib
 import traceback
 import typing
 
+import ruamel
 import tabulate
-import six
 import yamlize
 import yamlize.objects
-import ruamel.yaml
 import ordered_set
 
 import armi
@@ -85,15 +85,17 @@
 from armi.nucDirectory import nuclideBases
 from armi.nucDirectory import elements
 from armi.scripts import migration
+from armi.utils import textProcessors
+from armi.reactor import geometry
 
 # NOTE: using non-ARMI-standard imports because these are all a part of this package,
 # and using the module imports would make the attribute definitions extremely long
 # without adding detail
-from armi.reactor.blueprints.reactorBlueprint import Systems
+from armi.reactor.blueprints.reactorBlueprint import Systems, SystemBlueprint
 from armi.reactor.blueprints.assemblyBlueprint import AssemblyKeyedList
 from armi.reactor.blueprints.blockBlueprint import BlockKeyedList
 from armi.reactor.blueprints import isotopicOptions
-from armi.reactor.blueprints.gridBlueprint import Grids
+from armi.reactor.blueprints.gridBlueprint import Grids, Triplet
 
 context.BLUEPRINTS_IMPORTED = True
 context.BLUEPRINTS_IMPORT_CONTEXT = "".join(traceback.format_stack())
@@ -106,8 +108,12 @@ def loadFromCs(cs):
     # pylint: disable=import-outside-toplevel; circular import protection
     from armi.utils import directoryChangers
 
+    textProcessors.registerYamlIncludeConstructor()
+
     with directoryChangers.DirectoryChanger(cs.inputDirectory):
         with open(cs["loadingFile"], "r") as bpYaml:
+            # Make sure that the !include constructor is registered
+            bpYaml = textProcessors.resolveMarkupInclusions(bpYaml)
             try:
                 bp = Blueprints.load(bpYaml)
             except yamlize.yamlizing_error.YamlizingError as err:
@@ -153,11 +159,12 @@ def __new__(mcs, name, bases, attrs):
                     attrs[attrName] = section
                     attrs["_resolveFunctions"].append(resolver)
 
-        return yamlize.objects.ObjectType.__new__(mcs, name, bases, attrs)
+        newType = yamlize.objects.ObjectType.__new__(mcs, name, bases, attrs)
+
+        return newType
 
 
-@six.add_metaclass(_BlueprintsPluginCollector)
-class Blueprints(yamlize.Object):
+class Blueprints(yamlize.Object, metaclass=_BlueprintsPluginCollector):
     """Base Blueprintsobject representing all the subsections in the input file."""
 
     nuclideFlags = yamlize.Attribute(
@@ -193,10 +200,10 @@ def __new__(cls):
         return self
 
     def __init__(self):
-        # again, yamlize does not call __init__, instead we use Blueprints.load which creates and
-        # instance of a Blueprints object and initializes it with values using setattr. Since the
-        # method is never called, it serves the purpose of preventing pylint from issuing warnings
-        # about attributes not existing.
+        # again, yamlize does not call __init__, instead we use Blueprints.load which
+        # creates and instance of a Blueprints object and initializes it with values
+        # using setattr. Since the method is never called, it serves the purpose of
+        # preventing pylint from issuing warnings about attributes not existing.
         self._assembliesBySpecifier = {}
         self._prepped = False
         self.systemDesigns = Systems()
@@ -505,3 +512,67 @@ def migrate(cls, inp: typing.TextIO):
                 mig = migI(stream=inp)
                 inp = mig.apply()
         return inp
+
+def migrate(bp: Blueprints, cs):
+    """
+    Apply migrations to the input structure.
+
+    This is a good place to perform migrations that address changes to the system design
+    description (settings, blueprints, geom file). We have access to all three here, so
+    we can even move stuff between files. Namely, this:
+     - creates a grid blueprint to represent the core layout from the old ``geomFile``
+       setting, and applies that grid to a ``core`` system.
+     - moves the radial and azimuthal submesh values from the ``geomFile`` to the
+       assembly designs, but only if they are uniform (this is limiting, but could be
+       made more sophisticated in the future, if there is need)
+
+    This allows settings-driven core map to still be used for backwards compatibility.
+    At some point once the input stabilizes, we may wish to move this out to the
+    dedicated migration portion of the code, and not perform the migration so lazily.
+    """
+    from armi.reactor.blueprints import gridBlueprint
+
+    if bp.systemDesigns is None:
+        bp.systemDesigns = Systems()
+    if bp.gridDesigns is None:
+        bp.gridDesigns = gridBlueprint.Grids()
+
+    if "core" in [rd.name for rd in bp.gridDesigns]:
+        raise ValueError("Cannot auto-create a 2nd `core` grid. Adjust input.")
+
+    geom = geometry.SystemLayoutInput()
+    geom.readGeomFromFile(os.path.join(cs.inputDirectory, cs["geomFile"]))
+    gridDesign = geom.toGridBlueprint("core")
+    bp.gridDesigns["core"] = gridDesign
+
+    if "core" in [rd.name for rd in bp.systemDesigns]:
+        raise ValueError(
+            "Core map is defined in both the ``geometry`` setting and in "
+            "the blueprints file. Only one definition may exist. "
+            "Update inputs."
+        )
+    bp.systemDesigns["core"] = SystemBlueprint("core", "core", Triplet())
+
+    if geom.geomType in (geometry.RZT, geometry.RZ):
+        aziMeshes = {indices[4] for indices, _ in geom.assemTypeByIndices.items()}
+        radMeshes = {indices[5] for indices, _ in geom.assemTypeByIndices.items()}
+
+        if len(aziMeshes) > 1 or len(radMeshes) > 1:
+            raise ValueError("The system layout described in {} has non-uniform "
+                    "azimuthal and/or radial submeshing. This migration is currently "
+                    "only smart enough to handle a single radial and single azimuthal "
+                    "submesh for all assemblies.".format(cs["geomFile"]))
+        radMesh = next(iter(radMeshes))
+        aziMesh = next(iter(aziMeshes))
+
+        for _, aDesign in bp.assemDesigns.items():
+            aDesign.radialMeshPoints = radMesh
+            aDesign.azimuthalMeshPoints = aziMesh
+
+    # Someday: write out the migrated file. At the moment this messes up the case
+    # title and doesn't yet have the other systems in place so this isn't the right place.
+
+
+#     cs.writeToXMLFile(cs.caseTitle + '.migrated.xml')
+#     with open(os.path.split(cs['loadingFile'])[0] + '.migrated.' + '.yaml', 'w') as loadingFile:
+#         blueprints.Blueprints.dump(bp, loadingFile)

armi/reactor/blueprints/assemblyBlueprint.py

@@ -60,6 +60,12 @@ class AssemblyBlueprint(yamlize.Object):
     blocks = yamlize.Attribute(type=blockBlueprint.BlockList)
     height = yamlize.Attribute(type=yamlize.FloatList)
     axialMeshPoints = yamlize.Attribute(key="axial mesh points", type=yamlize.IntList)
+    radialMeshPoints = yamlize.Attribute(
+        key="radial mesh points", type=int, default=None
+    )
+    azimuthalMeshPoints = yamlize.Attribute(
+        key="azimuthal mesh points", type=int, default=None
+    )
     materialModifications = yamlize.Attribute(
         key="material modifications", type=MaterialModifications, default=None
     )
@@ -116,7 +122,15 @@ def _constructAssembly(self, cs, blueprint):
         a.spatialGrid = grids.axialUnitGrid(len(blocks))
         a.spatialGrid.armiObject = a
 
-        # loop a second time because we needed all the blocks before choosing the assembly class.
+        # TODO: Remove mesh points from blueprints entirely. Submeshing should be
+        # handled by specific physics interfaces
+        radMeshPoints = self.radialMeshPoints or 1
+        a.p.RadMesh = radMeshPoints
+        aziMeshPoints = self.azimuthalMeshPoints or 1
+        a.p.AziMesh = aziMeshPoints
+
+        # loop a second time because we needed all the blocks before choosing the
+        # assembly class.
         for axialIndex, block in enumerate(blocks):
             b.p.assemNum = a.p.assemNum
             b.name = b.makeName(a.p.assemNum, axialIndex)