Closes #1572: Update modular build process to initialize modules on c…

…ommand line (#1606) * Update modular build process to initialize modules on command line This PR changes from the current approach of generating a `ServerRegistration.chpl` file that calls the `registerMe()` function required in all modules to instead move the contents of the `registerMe()` to module-level scope that will add the functions to the command map upon module initialization. All modules that are to be modularly included in the server are now directly specified on the command line. This change is enabled by the work from chapel-lang/chapel#18724, which initializes all modules specified on the command line of a `chpl` call, even if the modules aren't explicitly `use`d. This means that Arkouda will require at least Chapel 1.26 after this change. * Convert rest of modules * Update python script to handle non-src dir files * Remove 1.25.1 compat check * Add space caught by Pierce * Update modular building docs * Update MODULAR.md fixed a typo `pat` -> `path` Co-authored-by: pierce314159 <48131946+pierce314159@users.noreply.github.com>
Bears-R-Us · Jul 21, 2022 · bf3ed39 · bf3ed39
1 parent 00bc0fb
commit bf3ed39
Show file tree

Hide file tree

Showing 30 changed files with 128 additions and 190 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        chpl-version: ['1.25.1', '1.26.0', '1.27.0']
+        chpl-version: ['1.26.0', '1.27.0']
     container:
       image: chapel/chapel:${{matrix.chpl-version}}
     steps:

diff --git a/MODULAR.md b/MODULAR.md
@@ -12,17 +12,13 @@ New config files must follow the same format, listing one module name per line w
 
 ### Adding new modules into the build process
 
-Adding a module from outside of the Arkouda `src/` directory requires adding the module name in the `ServerModules.cfg` file, but also requires setting the `ARKOUDA_SERVER_USER_MODULES` environment variable to a string of the absolute path to the Chapel module. If multiple user modules are included, the paths must be separated by a space. For example, Arkouda could be built with a custom module `TestMsg.chpl` by adding `TestMsg` to a line in the `ServerModules.cfg` file and then by running the command `ARKOUDA_SERVER_USER_MODULES='/Users/path/to/TestMsg.chpl' make` in the Arkouda home directory. Note that for this to work, the absolute path to the `.chpl` file must be specified in the `ARKOUDA_SERVER_USER_MODULES` environment variable.
+Adding a module from outside of the Arkouda `src/` directory can be done in one of two ways: (1) adding the absolute path of the module as a new line to the `ServerModules.cfg` file or (2) setting the `ARKOUDA_SERVER_USER_MODULES` environment variable to a string of the absolute path to the Chapel module. If multiple user modules are included, the paths must be separated by a space. For example, Arkouda could be built with a custom module `/path/toTestMsg.chpl` by adding `/path/to/TestMsg` to the `ServerModules.cfg` file and running `make` or by running the command `ARKOUDA_SERVER_USER_MODULES='/path/to/TestMsg.chpl' make` in the Arkouda home directory. Note that for this to work, the absolute path to the `.chpl` file must be specified in the `ARKOUDA_SERVER_USER_MODULES` environment variable.
 
-Additionally, a `registerMe()` function is required in the module in order to make the new functionality visible to the Arkouda server. This function must have the `CommandMap` module in scope and must call `registerFunction()` with the server message string and function name to be called.
-
-Here is an example `registerFunction()` call taken from [src/KEXtremeMsg.chpl](src/KExtremeMsg.chpl):
+Additionally, code to add the functions contained in the new module must be in module-level scope. Here is an example of what that might look like in practice (taken from [src/KEXtremeMsg.chpl](src/KExtremeMsg.chpl)):
 ```
-proc registerMe() {
-  use CommandMap;
-  registerFunction("mink", minkMsg);
-  registerFunction("maxk", maxkMsg);
-}
+use CommandMap;
+registerFunction("mink", minkMsg, getModuleName());
+registerFunction("maxk", maxkMsg, getModuleName());
 ```
 
 The last step on adding a new function is to add a function for the client side of the server. This can be accomplished in a number of ways, but a simple approach is to create a script of this form:
@@ -48,4 +44,4 @@ To use this flag:
 
 Upon server shutdown, a `UsedModules.cfg` file is created that includes the list of modules that were used in that particular Arkouda server instance. This can then either be inspected to assist in modification of the `ServerModules.cfg` file, or set to be used explicitly by the server by setting the `ARKOUDA_CONFIG_FILE` environment variable to this new file.
 
-If you wish to determine what modules are used in a particular benchmark run, that can be done by running an Arkouda benchmark with the `--server-args` argument set: `./benchmarks/run-benchmarks.py BENCHMARK-NAME --server-args='--saveUsedModules'`.
+If you wish to determine what modules are used in a particular benchmark run, that can be done by running an Arkouda benchmark with the `--server-args` argument set: `./benchmarks/run-benchmarks.py BENCHMARK-NAME --server-args='--saveUsedModules'`.
diff --git a/Makefile b/Makefile
@@ -285,15 +285,14 @@ endif
 MODULE_GENERATION_SCRIPT=$(ARKOUDA_SOURCE_DIR)/serverModuleGen.py
 # This is the main compilation statement section
 $(ARKOUDA_MAIN_MODULE): check-deps $(ARROW_O) $(ARKOUDA_SOURCES) $(ARKOUDA_MAKEFILES)
-	$(eval MOD_GEN_OUT=$(shell python3 $(MODULE_GENERATION_SCRIPT) $(ARKOUDA_CONFIG_FILE)))
-	@echo $(MOD_GEN_OUT);
+	$(eval MOD_GEN_OUT=$(shell python3 $(MODULE_GENERATION_SCRIPT) $(ARKOUDA_CONFIG_FILE) $(ARKOUDA_SOURCE_DIR)))
 
-	$(CHPL) $(CHPL_DEBUG_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) $(ARKOUDA_SERVER_USER_MODULES) -o $@
+	$(CHPL) $(CHPL_DEBUG_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) $(ARKOUDA_SERVER_USER_MODULES) $(MOD_GEN_OUT) -o $@
 
 CLEAN_TARGETS += arkouda-clean
 .PHONY: arkouda-clean
 arkouda-clean:
-	$(RM) $(ARKOUDA_MAIN_MODULE) $(ARKOUDA_MAIN_MODULE)_real $(ARKOUDA_SOURCE_DIR)/ServerRegistration.chpl $(ARROW_O)
+	$(RM) $(ARKOUDA_MAIN_MODULE) $(ARKOUDA_MAIN_MODULE)_real $(ARROW_O)
 
 .PHONY: tags
 tags:
@@ -447,7 +446,6 @@ $(TEST_BINARY_DIR):
 
 .PHONY: $(TEST_TARGETS) # Force tests to always rebuild.
 $(TEST_TARGETS): $(TEST_BINARY_DIR)/$(TEST_BINARY_SIGIL)%: $(TEST_SOURCE_DIR)/%.chpl | $(TEST_BINARY_DIR)
-	python3 $(MODULE_GENERATION_SCRIPT) $(ARKOUDA_CONFIG_FILE)
 	$(CHPL) $(TEST_CHPL_FLAGS) -M $(ARKOUDA_SOURCE_DIR) $(ARKOUDA_COMPAT_MODULES) $< -o $@
 
 print-%:

diff --git a/src/ArgSortMsg.chpl b/src/ArgSortMsg.chpl
@@ -396,9 +396,7 @@ module ArgSortMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("argsort", argsortMsg, getModuleName());
-      registerFunction("coargsort", coargsortMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("argsort", argsortMsg, getModuleName());
+    registerFunction("coargsort", coargsortMsg, getModuleName());
 }
diff --git a/src/ArraySetopsMsg.chpl b/src/ArraySetopsMsg.chpl
@@ -262,11 +262,9 @@ module ArraySetopsMsg
             errorClass="ErrorWithContext");
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("intersect1d", intersect1dMsg, getModuleName());
-      registerFunction("setdiff1d", setdiff1dMsg, getModuleName());
-      registerFunction("setxor1d", setxor1dMsg, getModuleName());
-      registerFunction("union1d", union1dMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("intersect1d", intersect1dMsg, getModuleName());
+    registerFunction("setdiff1d", setdiff1dMsg, getModuleName());
+    registerFunction("setxor1d", setxor1dMsg, getModuleName());
+    registerFunction("union1d", union1dMsg, getModuleName());
 }
diff --git a/src/BroadcastMsg.chpl b/src/BroadcastMsg.chpl
@@ -121,8 +121,6 @@ module BroadcastMsg {
     return new MsgTuple(repMsg, MsgType.NORMAL);    
   }
 
-  proc registerMe() {
-    use CommandMap;
-    registerFunction("broadcast", broadcastMsg, getModuleName());
-  }
+  use CommandMap;
+  registerFunction("broadcast", broadcastMsg, getModuleName());
 }
diff --git a/src/CastMsg.chpl b/src/CastMsg.chpl
@@ -156,8 +156,6 @@ module CastMsg {
     }
   }
 
-  proc registerMe() {
-    use CommandMap;
-    registerFunction("cast", castMsg, getModuleName());
-  }
+  use CommandMap;
+  registerFunction("cast", castMsg, getModuleName());
 }
diff --git a/src/ConcatenateMsg.chpl b/src/ConcatenateMsg.chpl
@@ -352,8 +352,6 @@ module ConcatenateMsg
         }
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("concatenate", concatenateMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("concatenate", concatenateMsg, getModuleName());
 }
diff --git a/src/DataFrameIndexingMsg.chpl b/src/DataFrameIndexingMsg.chpl
@@ -228,9 +228,6 @@
         return new MsgTuple(repMsg, MsgType.NORMAL); 
     }
 
-
-    proc registerMe() {
-        use CommandMap;
-        registerFunction("dataframe_idx", dataframeBatchIndexingMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("dataframe_idx", dataframeBatchIndexingMsg, getModuleName());
 }
diff --git a/src/EfuncMsg.chpl b/src/EfuncMsg.chpl
@@ -832,12 +832,10 @@ module EfuncMsg
       return C;
     }    
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("efunc", efuncMsg, getModuleName());
-      registerFunction("efunc3vv", efunc3vvMsg, getModuleName());
-      registerFunction("efunc3vs", efunc3vsMsg, getModuleName());
-      registerFunction("efunc3sv", efunc3svMsg, getModuleName());
-      registerFunction("efunc3ss", efunc3ssMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("efunc", efuncMsg, getModuleName());
+    registerFunction("efunc3vv", efunc3vvMsg, getModuleName());
+    registerFunction("efunc3vs", efunc3vsMsg, getModuleName());
+    registerFunction("efunc3sv", efunc3svMsg, getModuleName());
+    registerFunction("efunc3ss", efunc3ssMsg, getModuleName());
 }
diff --git a/src/FindSegmentsMsg.chpl b/src/FindSegmentsMsg.chpl
@@ -216,8 +216,6 @@ module FindSegmentsMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("findSegments", findSegmentsMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("findSegments", findSegmentsMsg, getModuleName());
 }
diff --git a/src/FlattenMsg.chpl b/src/FlattenMsg.chpl
@@ -92,10 +92,8 @@ module FlattenMsg {
     fmLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
     return new MsgTuple(repMsg, MsgType.NORMAL);
   }
-
-  proc registerMe() {
-    use CommandMap;
-    registerFunction("segmentedFlatten", segFlattenMsg, getModuleName());
-    registerFunction("segmentedSplit", segmentedSplitMsg, getModuleName());
-  }
+
+  use CommandMap;
+  registerFunction("segmentedFlatten", segFlattenMsg, getModuleName());
+  registerFunction("segmentedSplit", segmentedSplitMsg, getModuleName());
 }
diff --git a/src/HDF5Msg.chpl b/src/HDF5Msg.chpl
@@ -2160,11 +2160,8 @@ module HDF5Msg {
         }
     }
 
-    proc registerMe() {
-        use CommandMap;
-        registerFunction("lshdf", lshdfMsg, getModuleName());
-        registerFunction("readAllHdf", readAllHdfMsg, getModuleName());
-        registerFunction("tohdf", tohdfMsg, getModuleName());
-    }
-
+    use CommandMap;
+    registerFunction("lshdf", lshdfMsg, getModuleName());
+    registerFunction("readAllHdf", readAllHdfMsg, getModuleName());
+    registerFunction("tohdf", tohdfMsg, getModuleName());
 }
diff --git a/src/HDF5MultiDim.chpl b/src/HDF5MultiDim.chpl
@@ -389,10 +389,7 @@ module HDF5MultiDim {
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-
-    proc registerMe() {
-        use CommandMap;
-        registerFunction("readhdf_multi", read_hdf_multi_msg, getModuleName());
-        registerFunction("writehdf_multi", write_hdf_multi_msg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("readhdf_multi", read_hdf_multi_msg, getModuleName());
+    registerFunction("writehdf_multi", write_hdf_multi_msg, getModuleName());
 }
diff --git a/src/HistogramMsg.chpl b/src/HistogramMsg.chpl
@@ -78,8 +78,6 @@ module HistogramMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("histogram", histogramMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("histogram", histogramMsg, getModuleName());
 }
diff --git a/src/In1dMsg.chpl b/src/In1dMsg.chpl
@@ -75,8 +75,6 @@ module In1dMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("in1d", in1dMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("in1d", in1dMsg, getModuleName());
 }
diff --git a/src/IndexingMsg.chpl b/src/IndexingMsg.chpl
@@ -1149,19 +1149,17 @@ module IndexingMsg
         imLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL); 
     }
-
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("arrayViewIntIndex", arrayViewIntIndexMsg, getModuleName());
-      registerFunction("arrayViewMixedIndex", arrayViewMixedIndexMsg, getModuleName());
-      registerFunction("arrayViewIntIndexAssign", arrayViewIntIndexAssignMsg, getModuleName());
-      registerFunction("[int]", intIndexMsg, getModuleName());
-      registerFunction("[slice]", sliceIndexMsg, getModuleName());
-      registerFunction("[pdarray]", pdarrayIndexMsg, getModuleName());
-      registerFunction("[int]=val", setIntIndexToValueMsg, getModuleName());
-      registerFunction("[pdarray]=val", setPdarrayIndexToValueMsg, getModuleName());
-      registerFunction("[pdarray]=pdarray", setPdarrayIndexToPdarrayMsg, getModuleName());
-      registerFunction("[slice]=val", setSliceIndexToValueMsg, getModuleName());
-      registerFunction("[slice]=pdarray", setSliceIndexToPdarrayMsg, getModuleName());
-    }
+
+    use CommandMap;
+    registerFunction("arrayViewIntIndex", arrayViewIntIndexMsg, getModuleName());
+    registerFunction("arrayViewMixedIndex", arrayViewMixedIndexMsg, getModuleName());
+    registerFunction("arrayViewIntIndexAssign", arrayViewIntIndexAssignMsg, getModuleName());
+    registerFunction("[int]", intIndexMsg, getModuleName());
+    registerFunction("[slice]", sliceIndexMsg, getModuleName());
+    registerFunction("[pdarray]", pdarrayIndexMsg, getModuleName());
+    registerFunction("[int]=val", setIntIndexToValueMsg, getModuleName());
+    registerFunction("[pdarray]=val", setPdarrayIndexToValueMsg, getModuleName());
+    registerFunction("[pdarray]=pdarray", setPdarrayIndexToPdarrayMsg, getModuleName());
+    registerFunction("[slice]=val", setSliceIndexToValueMsg, getModuleName());
+    registerFunction("[slice]=pdarray", setSliceIndexToPdarrayMsg, getModuleName());
 }
diff --git a/src/JoinEqWithDTMsg.chpl b/src/JoinEqWithDTMsg.chpl
@@ -370,8 +370,6 @@ module JoinEqWithDTMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }// end joinEqWithDTMsg()
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("joinEqWithDT", joinEqWithDTMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("joinEqWithDT", joinEqWithDTMsg, getModuleName());
 }// end module JoinEqWithDTMsg
diff --git a/src/KExtremeMsg.chpl b/src/KExtremeMsg.chpl
@@ -152,9 +152,7 @@ module KExtremeMsg
         }
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("mink", minkMsg, getModuleName());
-      registerFunction("maxk", maxkMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("mink", minkMsg, getModuleName());
+    registerFunction("maxk", maxkMsg, getModuleName());
 }
diff --git a/src/OperatorMsg.chpl b/src/OperatorMsg.chpl
@@ -1007,12 +1007,10 @@ module OperatorMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("binopvv", binopvvMsg, getModuleName());
-      registerFunction("binopvs", binopvsMsg, getModuleName());
-      registerFunction("binopsv", binopsvMsg, getModuleName());
-      registerFunction("opeqvv", opeqvvMsg, getModuleName());
-      registerFunction("opeqvs", opeqvsMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("binopvv", binopvvMsg, getModuleName());
+    registerFunction("binopvs", binopvsMsg, getModuleName());
+    registerFunction("binopsv", binopsvMsg, getModuleName());
+    registerFunction("opeqvv", opeqvvMsg, getModuleName());
+    registerFunction("opeqvs", opeqvsMsg, getModuleName());
 }
diff --git a/src/ParquetMsg.chpl b/src/ParquetMsg.chpl
@@ -976,13 +976,10 @@ module ParquetMsg {
     return new MsgTuple(repMsg,MsgType.NORMAL);
   }
 
-  proc registerMe() {
-    use CommandMap;
-    registerFunction("readAllParquet", readAllParquetMsg, getModuleName());
-    registerFunction("writeParquet", toparquetMsg, getModuleName());
-    registerFunction("lspq", lspqMsg, getModuleName());
-    registerFunction("getnullparquet", nullIndicesMsg, getModuleName());
-    ServerConfig.appendToConfigStr("ARROW_VERSION", getVersionInfo());
-  }
-
+  use CommandMap;
+  registerFunction("readAllParquet", readAllParquetMsg, getModuleName());
+  registerFunction("writeParquet", toparquetMsg, getModuleName());
+  registerFunction("lspq", lspqMsg, getModuleName());
+  registerFunction("getnullparquet", nullIndicesMsg, getModuleName());
+  ServerConfig.appendToConfigStr("ARROW_VERSION", getVersionInfo());
 }
diff --git a/src/RandMsg.chpl b/src/RandMsg.chpl
@@ -135,9 +135,7 @@ module RandMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("randint", randintMsg, getModuleName());
-      registerFunction("randomNormal", randomNormalMsg, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("randint", randintMsg, getModuleName());
+    registerFunction("randomNormal", randomNormalMsg, getModuleName());
 }
diff --git a/src/ReductionMsg.chpl b/src/ReductionMsg.chpl
@@ -1145,11 +1145,8 @@ module ReductionMsg
       throw new owned ErrorWithMsg("message: skipNan must be of type bool");
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("segmentedReduction", segmentedReductionMsg, getModuleName());
-      registerFunction("reduction", reductionMsg, getModuleName());
-      registerFunction("countReduction", countReductionMsg, getModuleName());
-    }
-
+    use CommandMap;
+    registerFunction("segmentedReduction", segmentedReductionMsg, getModuleName());
+    registerFunction("reduction", reductionMsg, getModuleName());
+    registerFunction("countReduction", countReductionMsg, getModuleName());
 }
diff --git a/src/RegistrationMsg.chpl b/src/RegistrationMsg.chpl
@@ -519,12 +519,10 @@ module RegistrationMsg
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
 
-    proc registerMe() {
-      use CommandMap;
-      registerFunction("register", registerMsg, getModuleName());
-      registerFunction("attach", attachMsg, getModuleName());
-      registerFunction("genericAttach", genAttachMsg, getModuleName());
-      registerFunction("unregister", unregisterMsg, getModuleName());
-      registerFunction("genericUnregisterByName", unregisterByName, getModuleName());
-    }
+    use CommandMap;
+    registerFunction("register", registerMsg, getModuleName());
+    registerFunction("attach", attachMsg, getModuleName());
+    registerFunction("genericAttach", genAttachMsg, getModuleName());
+    registerFunction("unregister", unregisterMsg, getModuleName());
+    registerFunction("genericUnregisterByName", unregisterByName, getModuleName());
 }