Baikal*: Added new macros and PreSync BC registration

I changed the BSSN codegen library to use the new macros with function names appended. I also added the specify_Driver_BoundaryConditions_Baikal* for registering boundary conditions with PreSync.
zachetienne · Sep 29, 2022 · a738f47 · a738f47
1 parent 0379c32
commit a738f47
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 22 deletions.
diff --git a/BSSN/BSSN_Ccodegen_library.py b/BSSN/BSSN_Ccodegen_library.py
@@ -164,11 +164,11 @@ def EinsteinToolkit_SIMD_declare_C_params(ThornName):
     return SIMD_declare_C_params_str
 
 
-def set_ETK_func_params_preloop(func_name_suffix, enable_SIMD=True):
+def set_ETK_func_params_preloop(func_name, enable_SIMD=True):
     params = "CCTK_ARGUMENTS"
-    preloop = """  DECLARE_CCTK_ARGUMENTS;"""
+    preloop = """  DECLARE_CCTK_ARGUMENTS_"""+func_name+""";"""
     ThornName = "Baikal"
-    if "BaikalVacuum" in func_name_suffix:
+    if "BaikalVacuum" in func_name:
         ThornName = "BaikalVacuum"
     if enable_SIMD:
         preloop += EinsteinToolkit_SIMD_declare_C_params(ThornName)
@@ -314,7 +314,7 @@ def add_rhs_eval_to_Cfunction_dict(includes=None, rel_path_to_Cparams=os.path.jo
     enableCparameters=True
     # Set up preloop in case we're outputting code for the Einstein Toolkit (ETK)
     if par.parval_from_str("grid::GridFuncMemAccess") == "ETK":
-        params, preloop = set_ETK_func_params_preloop(func_name_suffix)
+        params, preloop = set_ETK_func_params_preloop(name)
         enableCparameters=False
 
     FD_outCparams = "outCverbose=False,enable_SIMD=" + str(enable_SIMD)
@@ -432,7 +432,7 @@ def add_Ricci_eval_to_Cfunction_dict(includes=None, rel_path_to_Cparams=os.path.
     enableCparameters=True
     # Set up preloop in case we're outputting code for the Einstein Toolkit (ETK)
     if par.parval_from_str("grid::GridFuncMemAccess") == "ETK":
-        params, preloop = set_ETK_func_params_preloop(func_name_suffix)
+        params, preloop = set_ETK_func_params_preloop(name)
         enableCparameters=False
 
     if enable_split_for_optimizations_doesnt_help and FDorder >= 8:
@@ -538,7 +538,7 @@ def add_BSSN_constraints_to_Cfunction_dict(includes=None, rel_path_to_Cparams=os
     enableCparameters=True
     # Set up preloop in case we're outputting code for the Einstein Toolkit (ETK)
     if par.parval_from_str("grid::GridFuncMemAccess") == "ETK":
-        params, preloop = set_ETK_func_params_preloop(func_name_suffix)
+        params, preloop = set_ETK_func_params_preloop(name)
         enableCparameters=False
 
     FD_outCparams = "outCverbose=False,enable_SIMD=" + str(enable_SIMD)
@@ -589,7 +589,7 @@ def add_enforce_detgammahat_constraint_to_Cfunction_dict(includes=None, rel_path
     enableCparameters=True
     # Set up preloop in case we're outputting code for the Einstein Toolkit (ETK)
     if par.parval_from_str("grid::GridFuncMemAccess") == "ETK":
-        params, preloop = set_ETK_func_params_preloop(func_name_suffix, enable_SIMD=False)
+        params, preloop = set_ETK_func_params_preloop(name, enable_SIMD=False)
         enableCparameters=False
 
     FD_outCparams = "outCverbose=False,enable_SIMD=False"

diff --git a/Tutorial-ETK_thorn-BaikalETK.ipynb b/Tutorial-ETK_thorn-BaikalETK.ipynb
@@ -763,6 +763,9 @@
     "CCTK_INT FUNCTION Boundary_SelectVarForBC(CCTK_POINTER_TO_CONST IN GH, CCTK_INT IN faces, CCTK_INT IN boundary_width, CCTK_INT IN table_handle, CCTK_STRING IN var_name, CCTK_STRING IN bc_name)\n",
     "USES FUNCTION Boundary_SelectVarForBC\n",
     "\n",
+    "CCTK_INT FUNCTION Driver_SelectVarForBC(CCTK_POINTER_TO_CONST IN GH, CCTK_INT IN faces, CCTK_INT IN boundary_width, CCTK_INT IN table_handle, CCTK_STRING IN group_name, CCTK_STRING IN bc_name)\n",
+    "USES FUNCTION Driver_SelectVarForBC\n",
+    "\n",
     "# Needed for EinsteinEvolve/NewRad outer boundary condition driver:\n",
     "CCTK_INT FUNCTION                         \\\n",
     "    NewRad_Apply                          \\\n",
@@ -816,10 +819,10 @@
     "    outstr += \"CCTK_REAL evol_variables_rhs type = GF Timelevels=1 TAGS=\\'InterpNumTimelevels=1 prolongation=\\\"none\\\"\\'\\n{\\n\"\n",
     "    outstr += output_list_of_gfs(rhs_list, \"right-hand-side storage for BSSN evolved gridfunctions\")\n",
     "    # Then AUX type:\n",
-    "    outstr += \"CCTK_REAL aux_variables type = GF Timelevels=3\\n{\\n\"\n",
+    "    outstr += \"CCTK_REAL aux_variables type = GF Timelevels=3 TAGS=\\'checkpoint=\\\"no\\\"\\'\\n{\\n\"\n",
     "    outstr += output_list_of_gfs(aux_gfs_list, \"Auxiliary gridfunctions for BSSN diagnostics\")\n",
     "    # Finally, AUXEVOL type:\n",
-    "    outstr += \"CCTK_REAL auxevol_variables type = GF Timelevels=1 TAGS=\\'InterpNumTimelevels=1 prolongation=\\\"none\\\"\\'\\n{\\n\"\n",
+    "    outstr += \"CCTK_REAL auxevol_variables type = GF Timelevels=1 TAGS=\\'InterpNumTimelevels=1 prolongation=\\\"none\\\"\\ checkpoint=\\\"no\\\"'\\n{\\n\"\n",
     "    outstr += output_list_of_gfs(auxevol_gfs_list, \"Auxiliary gridfunctions needed for evaluating the BSSN RHSs\")\n",
     "\n",
     "    with open(os.path.join(ThornName, \"interface.ccl\"), \"w\") as file:\n",
@@ -885,6 +888,13 @@
     "  OPTIONS: meta\n",
     "} \"Register 3+1 slicing condition\"\n",
     "\n",
+    "# This function registers the boundary conditions with PreSync.\n",
+    "schedule specify_Driver_BoundaryConditions_BaikalETK in Driver_BoundarySelect\n",
+    "{\n",
+    "  LANG: C\n",
+    "  OPTIONS: LEVEL\n",
+    "} \"Register boundary conditions with PreSync\"\n",
+    "\n",
     "schedule Symmetry_registration_oldCartGrid3D_BaikalETK at BASEGRID\n",
     "{\n",
     "  LANG: C\n",
@@ -1032,9 +1042,9 @@
     "    READS: cfGF(everywhere), lambdaU0GF, lambdaU1GF, lambdaU2GF\"\"\"\n",
     "        if enable_stress_energy_source_terms == True:\n",
     "            outstr += \"\"\"\n",
-    "    READS: alphaGF(everywhere), vetU0GF, vetU1GF, vetU2GF\"\"\"\n",
-    "            outstr += \"\"\"\n",
-    "    READS: T4UU00GF(everywhere), T4UU01GF, T4UU02GF, T4UU03GF\n",
+    "    READS: alphaGF(everywhere), vetU0GF, vetU1GF, vetU2GF\n",
+    "    READS: T4UU00GF(everywhere), T4UU01GF, T4UU02GF, T4UU03GF\"\"\"\n",
+    "        outstr += \"\"\"\n",
     "    WRITES: aux_variables(interior)\n",
     "  } \"Compute BSSN (Hamiltonian and momentum) constraints, at finite-differencing order: \"\"\"+str(order)+\"\"\"\"\n",
     "}\n",
@@ -1367,6 +1377,56 @@
   {
    "cell_type": "code",
    "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def add_to_Cfunction_dict_specify_Driver_BoundaryConditions(ThornName):\n",
+    "    includes = [\"stdio.h\", \"cctk.h\", \"cctk_Arguments.h\", \"cctk_Parameters.h\",\n",
+    "                \"cctk_Faces.h\", \"util_Table.h\"]\n",
+    "    desc = \"\"\"\"\"\"\n",
+    "\n",
+    "    desc = \"\"\"Set all boundary conditions for PreSync.\n",
+    "BSSN constraints are set to use `flat` boundary conditions, similar to what Lean does.\n",
+    "\n",
+    "BSSN evolved variables are set to use `none` boundary conditions, as these are set via NewRad.\n",
+    "\n",
+    "Since we choose NewRad boundary conditions, we must register all\n",
+    "evolved gridfunctions to have boundary type \"none\". This is because\n",
+    "NewRad directly modifies the RHSs.\n",
+    "\n",
+    "This code is based on Kranc's McLachlan/ML_BSSN/src/Boundaries.cc code.\n",
+    "\"\"\"\n",
+    "    c_type = \"void\"\n",
+    "    name = \"specify_Driver_BoundaryConditions_\" + ThornName\n",
+    "    params = \"CCTK_ARGUMENTS\"\n",
+    "    body = r\"\"\"  DECLARE_CCTK_ARGUMENTS_\"\"\"+name+r\"\"\";\n",
+    "  DECLARE_CCTK_PARAMETERS;\n",
+    "  CCTK_INT ierr CCTK_ATTRIBUTE_UNUSED = 0;\n",
+    "  const CCTK_INT bndsize = FD_order / 2 + 1;  // <- bndsize = number of ghostzones\n",
+    "\"\"\"\n",
+    "    for gf in aux_gfs_list:\n",
+    "        body += \"\"\"\n",
+    "  ierr = Driver_SelectVarForBC(cctkGH, CCTK_ALL_FACES, bndsize, -1, \"BaikalETK::\"\"\"+gf+\"\"\"\", \"flat\");\n",
+    "  if (ierr < 0) CCTK_ERROR(\"Failed to register BC with Driver for BaikalETK::\"\"\"+gf+\"\"\"!\");\n",
+    "\"\"\"\n",
+    "    for gf in evol_gfs_list:\n",
+    "        body += \"\"\"\n",
+    "  ierr = Driver_SelectVarForBC(cctkGH, CCTK_ALL_FACES, 1, -1, \"BaikalETK::\"\"\"+gf+\"\"\"\", \"none\");\n",
+    "  if (ierr < 0) CCTK_ERROR(\"Failed to register BC with Driver for BaikalETK::\"\"\"+gf+\"\"\"!\");\n",
+    "\"\"\"\n",
+    "    add_to_Cfunction_dict(\n",
+    "        includes=includes,\n",
+    "        desc=desc,\n",
+    "        c_type=c_type, name=name, params=params,\n",
+    "        body=body.replace(\"BaikalETK\", ThornName),\n",
+    "        enableCparameters=False)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.738265Z",
@@ -1409,7 +1469,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 18,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.742773Z",
@@ -1449,7 +1509,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 19,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.747393Z",
@@ -1520,7 +1580,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 20,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.752649Z",
@@ -1682,7 +1742,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 21,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.766419Z",
@@ -1756,7 +1816,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 22,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.772647Z",
@@ -1862,7 +1922,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 23,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:26.779476Z",
@@ -1881,6 +1941,7 @@
     "    add_to_Cfunction_dict_floor_the_lapse(thorn)\n",
     "    add_to_Cfunction_dict_MoL_registration(thorn)\n",
     "    add_to_Cfunction_dict_RegisterSlicing(thorn)\n",
+    "    add_to_Cfunction_dict_specify_Driver_BoundaryConditions(thorn)\n",
     "    add_to_Cfunction_dict_specify_BoundaryConditions_aux_gfs(thorn)\n",
     "    add_to_Cfunction_dict_specify_BoundaryConditions_evolved_gfs(thorn)\n",
     "    add_to_Cfunction_dict_specify_NewRad_BoundaryConditions_parameters(thorn)\n",
@@ -1890,7 +1951,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 24,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:30.371534Z",
@@ -1914,7 +1975,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 25,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:30.375642Z",
@@ -1946,7 +2007,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 26,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:30.379466Z",
@@ -1992,7 +2053,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 27,
    "metadata": {
     "execution": {
      "iopub.execute_input": "2022-09-22T15:59:30.392953Z",