Merge pull request #2252 from stfc/1990_handle_imported_variable_in_k…

…ernel_extraction

1990 handle imported variable in kernel extraction

changelog

@@ -79,6 +79,9 @@
 	29) PR #2556 towards #1351. Documents the new OPERATES_ON=dof
 	LFRic kernel type.
 
+	30) PR #2252 for #1990. Support extraction of kernels that import
+	variables from other modules.
+
 release 2.5.0 14th of February 2024
 
 	1) PR #2199 for #2189. Fix bugs with missing maps in enter data

config/psyclone.cfg

@@ -55,7 +55,7 @@ VALID_PSY_DATA_PREFIXES = profile, extract, read_only_verify, nan_test
 OCL_DEVICES_PER_NODE = 1
 
 # Symbols imported from the following modules will be ignored when parsing
-# and will not produce a warning message if they cannot be found
+# and will not produce a warning message if they cannot be found.
 IGNORE_MODULES = netcdf, mpi
 
 # Settings specific to the LFRic (Dynamo 0.3) API

doc/user_guide/psy_data.rst

@@ -170,7 +170,7 @@ linking the verification library. The application which uses the
 read-only-verification library needs to link in the infrastructure
 library anyway.
 
-.. note:
+.. note::
     It is the responsibility of the user to make sure that the infrastructure
     files used during compilation of the read-only-verification library are
     also used when linking the application. Otherwise strange and
@@ -255,3 +255,44 @@ An executable example for using the LFRic read-only-verification library is
 included in ``tutorial/practicals/LFRic/building_code/4_psydata`` directory,
 see `this link for more information
 <https://github.com/stfc/PSyclone/tree/master/tutorial/practicals/LFRic/building_code/4_psydata>`_.
+
+
+.. _integrating_psy_data_lfric:
+
+Integrating PSyData Libraries into the LFRic Build Environment
+--------------------------------------------------------------
+The easiest way of integrating any PSyData-based library into the LFRic
+build environment is:
+
+- In the LFRic source tree create a new directory under ``infrastructure/source``,
+  e.g. ``infrastructure/source/psydata``.
+- Build the PSyData wrapper stand-alone in ``lib/extract/netcdf/lfric`` (which
+  will use NetCDF as output format) or ``lib/extract/standalone/lfric`` (which
+  uses standard Fortran binary output format) by executing ``make``. The compiled
+  files will actually not be used, but this step will create all source
+  files (some of which are created by jinja). Do not copy
+  the compiled files into your LFRic build tree, since these files might be
+  compiled with an outdated version of the infrastructure files and be
+  incompatible with files in a current LFRic version.
+- Copy all processed source files (``extract_netcdf_base.f90``,
+  ``kernel_data_netcdf.f90``, ``psy_data_base.f90``,
+  ``read_kernel_data_mod.f90``) into ``infrastructure/source/psydata``
+- Start the LFRic build process as normal. The LFRic build environment will
+  copy the PSyData source files into the working directory and compile
+  them.
+- If the PSyData library needs additional include paths (e.g. when using an
+  external profiling tool), add the required paths to ``$FFLAGS``.
+- If additional libraries are required at link time, add the paths
+  and libraries to ``$LDFLAGS``. Alternatively, when a compiler wrapper
+  script is provided by a third-party tool (e.g. the profiling tool
+  TAU provides a script ``tau_f90.sh``), either set the environment variable
+  ``$FC``, or if this is only required at link time, the variable ``$LDMPI``
+  to this compiler wrapper.
+
+.. warning::
+    Only one PSyData library can be integrated at a time. Otherwise there
+    will be potentially several modules with the same name (e.g.
+    ``psy_data_base``), resulting in errors at compile time.
+
+.. note::
+    With the new build system FAB this process might change.

doc/user_guide/psyke.rst

@@ -457,13 +457,8 @@ code, it will create an output file for each instrumented code region.
 The same logic for naming variables (using ``_post`` for output variables)
 used in :ref:`extraction_for_gocean` is used here.
 
-As in the case of e.g. :ref:`read-only verification
-<psydata_read_verification>`, this library uses the pared-down LFRic
-infrastructure located in a clone of PSyclone repository,
-``<PSYCLONEHOME>/src/psyclone/tests/test_files/dynamo0p3/infrastructure``.
-However, this needs to be changed for any user (for instance with
-PSyclone installation). Please refer to the relevant ``README.md``
-documentation on how to build and link this library.
+Check :ref:`integrating_psy_data_lfric` for the recommended way of linking
+an extraction library to LFRic.
 
 The output file contains the values of all variables used in the
 subroutine. The ``LFRicExtractTrans`` transformation can automatically
@@ -495,14 +490,6 @@ optimisation of a stand-alone kernel.
     stores the variable names and will not be able to find a variable
     if its name has changed.
 
-.. note:: If the kernel, or any function called from an extracted kernel
-    should use a variable from a module directly (as opposed to supplying
-    this as parameter in the kernel call), this variable will not be
-    written to the extract data file, and the driver will also not try to
-    read in the value. As a result, the kernel will not be able to
-    run stand-alone. As a work-around, these values can be added manually
-    to the driver program. Issue #1990 tracks improvement of this situation.
-
 The LFRic kernel driver will inline all required external modules into the
 driver. It uses a ``ModuleManager`` to find the required modules, based on the
 assumption that a file ``my_special_mod.f90`` will define exactly one module
@@ -531,6 +518,21 @@ paths (infrastructure files and extraction library) for the compiler, but
 these flags are actually only required for compiling the example program, not
 for the driver.
 
+Restrictions of Kernel Extraction and Driver Creation
+#####################################################
+A few restrictions still apply to the current implementation of the driver
+creation code:
+
+- Distributed memory is not yet supported. See #1992.
+- The extraction code will now write variables that are used from other
+  modules to the kernel data file, and the driver will read these values in.
+  Unfortunately, if a variable is used that is defined as private,
+  the value cannot be written to the file, and compilation will abort.
+  The only solution is to modify this file and make all variables public.
+  This mostly affects ``log_mod.F90``, but a few other modules as well.
+- The new build system FAB will be able to remove ``private`` and
+  ``protected`` declarations in any source files, meaning no manual
+  modification of files is required anymore (TODO #2536).
 
 Extraction for NEMO
 ++++++++++++++++++++

examples/gocean/eg5/extract/Makefile

@@ -61,19 +61,16 @@ INF_INC = $(INF_DIR)/src
 INF_LIB = $(INF_DIR)/src/lib_fd.a
 ifeq ($(TYPE), netcdf)
 	EXTRACT_DIR ?= $(PSYROOT)/lib/extract/netcdf/dl_esm_inf
-	READ_DIR ?= $(PSYROOT)/lib/extract/netcdf
 	F90FLAGS += $$(nf-config --fflags)
 	LDFLAGS += $$(nf-config --flibs) $$(nc-config --libs)
 	GENERATED_FILES += main-init.nc main-update.nc
 else
 	EXTRACT_DIR ?= $(PSYROOT)/lib/extract/standalone/dl_esm_inf
-	READ_DIR ?= $(PSYROOT)/lib/extract/standalone
 	GENERATED_FILES += main-init.binary main-update.binary
 
 endif
 
 LIB_NAME = lib_extract.a
-READ_KERNEL_DATA = $(READ_DIR)/read_kernel_data_mod.o \
 
 
 # The two kernels used in the application.
@@ -91,16 +88,16 @@ DRIVER_UPDATE = driver-main-update
 .PHONY: transform compile run
 
 run: compile
-	./extract_test
-	./driver-main-init
-	./driver-main-update
+	./$(NAME)
+	./driver-main-init.$(TYPE)
+	./driver-main-update.$(TYPE)
 
 compile: transform $(NAME) $(DRIVER_INIT).$(TYPE) $(DRIVER_UPDATE).$(TYPE)
 
 transform: psy.f90
 
 
-F90FLAGS += -I$(INF_INC) -I$(EXTRACT_DIR) -I$(READ_DIR)
+F90FLAGS += -I$(INF_INC) -I$(EXTRACT_DIR)
 
 alg.f90 psy.f90: test.x90 extract_transform.py
 	$(PSYCLONE) -nodm -api "gocean1.0" -s ./extract_transform.py\
@@ -112,15 +109,15 @@ $(NAME): $(INF_LIB) $(EXTRACT_DIR)/$(LIB_NAME) $(KERNELS) alg.o psy.o
 
 #TODO #1757: $(INF_LIB) is required because of the meta-data in the
 #            kernel - once this is fixed, $(INF_LIB) can be removed.
-$(DRIVER_INIT).$(TYPE):	$(KERNELS) $(DRIVER_INIT).o $(READ_KERNEL_DATA)
+$(DRIVER_INIT).$(TYPE):	$(KERNELS) $(DRIVER_INIT).o
 	$(F90) $(KERNELS) $(DRIVER_INIT).o -o $(DRIVER_INIT).$(TYPE) \
-		$(INF_LIB) $(READ_KERNEL_DATA) $(LDFLAGS)
+		$(INF_LIB) $(EXTRACT_DIR)/$(LIB_NAME) $(LDFLAGS)
 
 #TODO #1757: $(INF_LIB) is required because of the meta-data in the
 #            kernel - once this is fixed, $(INF_LIB) can be removed.
-$(DRIVER_UPDATE).$(TYPE): $(KERNELS) $(DRIVER_UPDATE).o $(READ_KERNEL_DATA)
+$(DRIVER_UPDATE).$(TYPE): $(KERNELS) $(DRIVER_UPDATE).o
 	$(F90) $(KERNELS) $(DRIVER_UPDATE).o -o $(DRIVER_UPDATE).$(TYPE) \
-		$(INF_LIB) $(READ_KERNEL_DATA) $(LDFLAGS)
+		$(INF_LIB) $(EXTRACT_DIR)/$(LIB_NAME) $(LDFLAGS)
 
 # The dl_esm_inf library
 $(INF_LIB):
@@ -138,8 +135,6 @@ psy.o:	$(KERNELS)
 # directory will fail.
 $(DRIVER_INIT).f90: psy.f90
 $(DRIVER_UPDATE).f90: psy.f90
-$(DRIVER_INIT).o: $(READ_KERNEL_DATA)
-$(DRIVER_UPDATE).o: $(READ_KERNEL_DATA)
 
 # Dependency to INF_LIB to make sure the mod file are available
 $(KERNELS):  $(INF_LIB)
@@ -152,10 +147,6 @@ $(KERNELS):  $(INF_LIB)
 $(EXTRACT_DIR)/lib_kernel_data_netcdf.a:
 	make -C $(EXTRACT_DIR)
 
-$(READ_KERNEL_DATA):
-		make -C $(READ_DIR)
-
 allclean: clean
 	make -C $(INF_DIR) clean
 	make -C $(EXTRACT_DIR) clean
-	make -C $(READ_DIR) clean

examples/lfric/eg17/full_example_extract/Makefile

@@ -58,7 +58,7 @@ GENERATED_FILES += driver-main-init driver-main-init.F90       \
 F90 ?= gfortran
 F90FLAGS ?= -Wall -g -ffree-line-length-none
 
-OBJ = main_psy.o main_alg.o testkern_w0_kernel_mod.o
+OBJ = main_psy.o main_alg.o testkern_w0_kernel_mod.o dummy_mod.o
 
 ifeq ($(TYPE), netcdf)
 	EXTRACT_DIR ?= $(PSYROOT)/lib/extract/netcdf/lfric
@@ -71,7 +71,6 @@ endif
 EXEC = extract.$(TYPE)
 EXTRACT_NAME ?= _extract
 EXTRACT_LIB = $(EXTRACT_DIR)/lib$(EXTRACT_NAME).a
-READ_KERNEL_DATA_OBJ = $(EXTRACT_DIR)/../read_kernel_data_mod.o
 LFRIC_PATH ?= $(PSYROOT)/src/psyclone/tests/test_files/dynamo0p3/infrastructure
 LFRIC_NAME=lfric
 LFRIC_LIB=$(LFRIC_PATH)/lib$(LFRIC_NAME).a
@@ -101,19 +100,16 @@ $(EXTRACT_LIB): $(LFRIC_LIB)
 # Dependencies
 main_psy.o:	testkern_w0_kernel_mod.o $(EXTRACT_LIB) $(LFRIC_LIB)
 main_alg.o:	main_psy.o
-testkern_w0_kernel_mod.o: $(LFRIC_LIB)
+testkern_w0_kernel_mod.o: dummy_mod.o $(LFRIC_LIB)
 
-driver-main-update:	LFRIC_INCLUDE_FLAGS += -I $(EXTRACT_DIR)/..
-driver-main-update:	driver-main-update.o $(READ_KERNEL_DATA_OBJ)
+driver-main-update:	driver-main-update.o
 	$(F90) $(F90FLAGS) $(LFRIC_INCLUDE_FLAGS)  driver-main-update.o \
-		   $(LDFLAGS) -o driver-main-update
+		$(LDFLAGS) -o driver-main-update
 
 driver-main-init:	LFRIC_INCLUDE_FLAGS += -I $(EXTRACT_DIR)/..
-driver-main-init:	driver-main-init.o $(READ_KERNEL_DATA_OBJ)
+driver-main-init:	driver-main-init.o
 	$(F90) $(F90FLAGS) $(LFRIC_INCLUDE_FLAGS)  driver-main-init.o \
-		   $(LDFLAGS) -o driver-main-init
-
-driver-main-update.o driver-main-init.o:  $(READ_KERNEL_DATA_OBJ)
+		$(LDFLAGS) -o driver-main-init
 
 %.o: %.F90
 	$(F90) $(F90FLAGS) $(LFRIC_INCLUDE_FLAGS) -c $<
@@ -124,14 +120,14 @@ driver-main-update.o driver-main-init.o:  $(READ_KERNEL_DATA_OBJ)
 # Keep the generated psy and alg files
 .precious: main_psy.f90 main_alg.f90
 
+# This dependency will make sure that read_kernel_data_mod was created
+# (which will be inlined in the driver).
+main_psy.f90: $(EXTRACT_LIB)
 main_alg.f90: main_psy.f90
 
-$(READ_KERNEL_DATA_OBJ):
-	$(MAKE) -C $(EXTRACT_DIR)/..
-
 %_psy.f90:	%.x90
 	${PSYCLONE} -s ./extract_transform.py       \
-	-d . -d $(EXTRACT_DIR) -d $(EXTRACT_DIR)/.. \
+	-d . -d $(EXTRACT_DIR)                      \
 	-d $(LFRIC_PATH)                            \
 	-nodm -opsy $*_psy.f90 -oalg $*_alg.f90 $<
 

examples/lfric/eg17/full_example_extract/dummy_mod.f90

@@ -0,0 +1,67 @@
+! BSD 3-Clause License
+!
+! Copyright (c) 2023-2024, Science and Technology Facilities Council
+! All rights reserved.
+!
+! Redistribution and use in source and binary forms, with or without
+! modification, are permitted provided that the following conditions are met:
+!
+! * Redistributions of source code must retain the above copyright notice, this
+!   list of conditions and the following disclaimer.
+!
+! * Redistributions in binary form must reproduce the above copyright notice,
+!   this list of conditions and the following disclaimer in the documentation
+!   and/or other materials provided with the distribution.
+!
+! * Neither the name of the copyright holder nor the names of its
+!   contributors may be used to endorse or promote products derived from
+!   this software without specific prior written permission.
+!
+! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+! AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+! IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+! DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+! FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+! DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+! SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+! CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+! OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+! OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+! -----------------------------------------------------------------------------
+! Author J. Henrichs, Bureau of Meteorology
+
+! This simple module is used to showcase and test the extraction of non-local
+! module variables with the driver extraction.
+
+module dummy_mod
+  integer :: dummy_var1
+  real :: dummy_var2
+  real :: dummy_var3 = 3
+
+  public :: dummy_code
+
+  interface dummy_code
+   module procedure dummy_code_1, dummy_code_2
+  end interface
+
+  contains
+
+  subroutine dummy_code_1(a)
+    implicit none
+    integer :: a
+    dummy_var1 = dummy_var1 + 1
+  end subroutine dummy_code_1
+
+  subroutine dummy_code_2(a)
+    implicit none
+    real :: a
+    dummy_var1 = dummy_var1 + 1
+  end subroutine dummy_code_2
+
+  integer function dummy_func(a)
+    implicit none
+    integer :: a
+    dummy_func = a+1 + dummy_var2
+  end function dummy_func
+
+end module dummy_mod

examples/lfric/eg17/full_example_extract/testkern_w0_kernel_mod.f90

@@ -32,6 +32,7 @@
 ! Modified by J. Henrichs, Bureau of Meteorology
 ! Modified by I. Kavcic, Met Office
 
+
 module testkern_w0_kernel_mod
 
   use argument_mod
@@ -40,8 +41,15 @@ module testkern_w0_kernel_mod
 
   use constants_mod
 
+  ! This is used to showcase the ability of the kernel extraction
+  ! to write and for the driver creation to read non-local module variables
+  ! when importing them in the module scope
+  use dummy_mod, only: dummy_var1, dummy_code
+
   implicit none
 
+  integer, public :: some_other_var
+  integer, parameter :: some_other_const = 123
   private
 
   type, public, extends(kernel_type) :: testkern_w0_kernel_type
@@ -64,6 +72,10 @@ module testkern_w0_kernel_mod
   subroutine testkern_w0_code(nlayers, fld1, fld2, chi1, chi2, chi3, &
                               some_logical, ndf_w0, undf_w0, map_w0)
 
+    ! This is used to showcase the ability of the kernel extraction
+    ! to write and for the driver creation to read non-local module
+    ! variables when importing them in the kernel itself.
+    use dummy_mod, only: dummy_var2, dummy_var3, dummy_func, dummy_code
     implicit none
 
     integer(kind=i_def), intent(in)                     :: nlayers
@@ -75,12 +87,19 @@ subroutine testkern_w0_code(nlayers, fld1, fld2, chi1, chi2, chi3, &
     integer(kind=i_def), dimension(ndf_w0)              :: map_w0
 
     integer(kind=i_def)                                 :: i, k
+    real(kind=r_def) :: some_r
 
+    call dummy_code(1)
+    some_r = 0
     do k=0, nlayers-1
       do i=1, ndf_w0
-        fld1(map_w0(i)+k) = fld1(map_w0(i)+k) + fld2(map_w0(i)+k)
+        some_r = some_r + 1
+        fld1(map_w0(i)+k) = fld1(map_w0(i)+k) + fld2(map_w0(i)+k)             &
+                          + dummy_func(i)
         if (some_logical) then
-          fld1(map_w0(i)+k) = fld1(map_w0(i)+k) + 1
+          fld1(map_w0(i)+k) = fld1(map_w0(i)+k) + 1 + dummy_var1 + dummy_var2 &
+                            + some_other_var + some_r + dummy_var3            &
+                            + some_other_const
         endif
       end do
     end do

lib/extract/netcdf/dl_esm_inf/.gitignore

@@ -0,0 +1 @@
+read_kernel_data_mod.f90