Makefile refactoring to factor out common build code (#2672)

* Don't use double brackets in source script

Match the same style as used in the rest of the script. I observed a
syntax error when running in a Docker container locally as well, so fix
this up quickly.

* Move arch and uarch specific code out of main Makefile

Following on from pull request
#2614, we want to make it
easier to distinguish common and platform specific code when building.
We take common code from the main makefile, and split this out into arch
and uarch specific files. We try and share as much code as possible
between uarch files.

In this first commit, we make sure that the x86 build works on linux.
Next commits will make sure that the Mac, Windows and Arm builds also
work. For now these are broken.

* Update Mac, Windows and Arm makefile configuration

Put Mac and Windows makefile code into their own files, and correct some
copy-paste errors in the Arm build. This should build on all platforms
now, but there are some more commits to go on top before the refactor is
done.

* Refactor compiler configuration in makefiles

We want to have as much common code shared between makefiles for
different compilers, as well as for the different backends. We shuffle
things around a little. This now passes on x86 Linux as well as aarch64.
Needs some internal tests to be run.

* Remove recursive definition

* Code review copyright notice and new line changes

* Remove year from contributors copyright notice

* Don't use year in copyright

* review comments

deploy/local/vars_lnx.sh

@@ -226,9 +226,9 @@ fi
 
 ARCH_ONEDAL=$(uname -m)
 
-if [[ "${ARCH_ONEDAL}" == "x86_64" ]]; then
+if [ "${ARCH_ONEDAL}" = "x86_64" ]; then
     ARCH_DIR_ONEDAL="intel64"
-elif [[ "${ARCH_ONEDAL}" == "aarch64" ]]; then
+elif [ "${ARCH_ONEDAL}" = "aarch64" ]; then
     ARCH_DIR_ONEDAL="arm"
 else
     echo "Unsupported CPU architecture '${ARCH_ONEDAL}'"

dev/make/common.mk

@@ -78,6 +78,7 @@ secure.opts.link.mac =
 
 RC.COMPILE = rc.exe $(RCOPT) -fo$@ $<
 
+# Used as $(eval $(call set_c_compile,$(COMPILER),$(_OS),$(gcc_toolchain))
 C.COMPILE = $(if $(COMPILER.$(_OS).$(COMPILER)),$(COMPILER.$(_OS).$(COMPILER)),$(error COMPILER.$(_OS).$(COMPILER) must be defined)) \
             $(if $(C.COMPILE.gcc_toolchain),--gcc-toolchain=$(C.COMPILE.gcc_toolchain)) \
             -c $(secure.opts.icc.$(_OS)) $(COPT) $(INCLUDES) $1 $(-Fo)$@ $<

dev/make/cmplr.clang.mkl.32e.mk → dev/make/compiler_definitions/clang.32e.mk

@@ -1,4 +1,4 @@
-# file: cmplt.clang.mk
+# file: clang.32e.mk
 #===============================================================================
 # Copyright 2012 Intel Corporation
 #
@@ -16,17 +16,15 @@
 #===============================================================================
 
 #++
-#  Clang definitions for makefile
+#  Clang definitions for makefile.
+#  This file contains definitions common to clang on a 32e (intel64) platform.
+#  It should only be included from files which have more specializations (e.g.
+#  clang.mkl.32e.mk)
 #--
 
-PLATs.clang = lnx32e mac32e
-
-CMPLRDIRSUFF.clang = _clang
-
-CORE.SERV.COMPILER.clang = generic
+include dev/make/compiler_definitions/clang.mk
 
--Zl.clang =
--DEBC.clang = -g
+PLATs.clang = lnx32e mac32e
 
 COMPILER.mac.clang = clang++ -m64 -fgnu-runtime -stdlib=libc++ -mmacosx-version-min=10.15 -fwrapv \
                      -Werror -Wreturn-type
@@ -36,11 +34,6 @@ COMPILER.lnx.clang = clang++ -m64 \
 link.dynamic.mac.clang = clang++ -m64
 link.dynamic.lnx.clang = clang++ -m64
 
-pedantic.opts.clang = -pedantic \
-                      -Wall \
-                      -Wextra \
-                      -Wno-unused-parameter
-
 pedantic.opts.mac.clang = $(pedantic.opts.clang)
 pedantic.opts.lnx.clang = $(pedantic.opts.clang)
 

dev/make/cmplr.clang.ref.32e.mk → dev/make/compiler_definitions/clang.mk

@@ -1,6 +1,6 @@
-# file: cmplt.clang.mk
+# file: clang.mk
 #===============================================================================
-# Copyright 2023 Intel Corporation
+# Copyright contributors to the oneDAL project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,34 +17,19 @@
 
 #++
 #  Clang definitions for makefile
+#  This file contains definitions common to clang on all platforms.
+#  It should only be included from files which have more specializations (e.g.
+#  clang.32e.mk)
 #--
 
-PLATs.clang = lnx32e mac32e
-
 CMPLRDIRSUFF.clang = _clang
 
 CORE.SERV.COMPILER.clang = generic
 
 -Zl.clang =
 -DEBC.clang = -g
 
-COMPILER.mac.clang = clang++ -m64 -fgnu-runtime -stdlib=libc++ -mmacosx-version-min=10.15 -fwrapv \
-                     -DDAAL_REF -DONEDAL_REF -Werror -Wreturn-type
-COMPILER.lnx.clang = clang++ -m64 \
-                     -DDAAL_REF -DONEDAL_REF -Werror -Wreturn-type
-
-link.dynamic.mac.clang = clang++ -m64
-link.dynamic.lnx.clang = clang++ -m64
-
 pedantic.opts.clang = -pedantic \
                       -Wall \
                       -Wextra \
                       -Wno-unused-parameter
-
-pedantic.opts.mac.clang = $(pedantic.opts.clang)
-pedantic.opts.lnx.clang = $(pedantic.opts.clang)
-
-p4_OPT.clang   = $(-Q)march=nocona
-mc3_OPT.clang  = $(-Q)$(if $(OS_is_mac),march=nocona,march=nehalem) $(if $(OS_is_mac),$(-Q)mtune=nehalem)
-avx2_OPT.clang = $(-Q)march=haswell
-skx_OPT.clang  = $(-Q)march=skx

dev/make/compiler_definitions/clang.mkl.32e.mk

@@ -0,0 +1,22 @@
+# file: clang.mkl.32e.mk
+#===============================================================================
+# Copyright 2012 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+#++
+#  Clang definitions for makefile
+#--
+
+include dev/make/compiler_definitions/clang.32e.mk

dev/make/compiler_definitions/clang.ref.32e.mk

@@ -0,0 +1,25 @@
+# file: clang.ref.32e.mk
+#===============================================================================
+# Copyright contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+#++
+#  Clang definitions for makefile
+#--
+
+include dev/make/compiler_definitions/clang.32e.mk
+
+COMPILER.mac.clang = $(COMPILER.mac.clang) -DDAAL_REF -DONEDAL_REF
+COMPILER.lnx.clang = $(COMPILER.lnx.clang) -DDAAL_REF -DONEDAL_REF

dev/make/cmplr.clang.ref.arm.mk → dev/make/compiler_definitions/clang.ref.arm.mk

@@ -1,4 +1,4 @@
-# file: cmplr.clang.ref.arm.mk
+# file: clang.ref.arm.mk
 #===============================================================================
 # Copyright contributors to the oneDAL project
 #
@@ -19,26 +19,15 @@
 #  Clang definitions for makefile
 #--
 
-PLATs.clang = lnxarm
-
-CMPLRDIRSUFF.clang = _clang
-
-CORE.SERV.COMPILER.clang = generic
+include dev/make/compiler_definitions/clang.mk
 
--Zl.clang =
--DEBC.clang = -g
+PLATs.clang = lnxarm
 
 COMPILER.lnx.clang= clang++ -march=armv8-a+sve \
                      -DDAAL_REF -DONEDAL_REF -DDAAL_CPU=sve -Werror -Wreturn-type
 # Linker flags
 link.dynamic.lnx.clang = clang++ -march=armv8-a+sve
 
-pedantic.opts.clang = -pedantic \
-                      -Wall \
-                      -Wextra \
-                      -Wno-unused-parameter
-
-pedantic.opts.mac.clang = $(pedantic.opts.clang)
 pedantic.opts.lnx.clang = $(pedantic.opts.clang)
 
 # For SVE

dev/make/cmplr.gnu.mkl.32e.mk → dev/make/compiler_definitions/gnu.32e.mk

@@ -1,5 +1,5 @@
 #===============================================================================
-# Copyright 2023 Intel Corporation
+# Copyright contributors to the oneDAL project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,34 +16,25 @@
 
 #++
 #  g++ definitions for makefile
+#  This file contains definitions common to gnu on a 32e (intel64) platform. It
+#  should only be included from files which have more specializations (e.g.
+#  gnu.mkl.32e.mk)
 #--
 
-PLATs.gnu = lnx32e mac32e
-
-CMPLRDIRSUFF.gnu = _gnu
-
-CORE.SERV.COMPILER.gnu = generic
+include dev/make/compiler_definitions/gnu.mk
 
--Zl.gnu =
--DEBC.gnu = -g
+PLATs.gnu = lnx32e mac32e
 
 COMPILER.all.gnu =  ${CXX} -m64 -fwrapv -fno-strict-overflow -fno-delete-null-pointer-checks \
                     -Werror -Wreturn-type
 
 link.dynamic.all.gnu = ${CXX} -m64
 
-pedantic.opts.all.gnu = -pedantic \
-                        -Wall \
-                        -Wextra \
-                        -Wno-unused-parameter
-
-COMPILER.lnx.gnu = $(COMPILER.all.gnu)
-link.dynamic.lnx.gnu = $(link.dynamic.all.gnu)
 pedantic.opts.lnx.gnu = $(pedantic.opts.all.gnu)
+pedantic.opts.mac.gnu = $(pedantic.opts.all.gnu)
 
-COMPILER.mac.gnu = $(COMPILER.all.gnu)
+link.dynamic.lnx.gnu = $(link.dynamic.all.gnu)
 link.dynamic.mac.gnu = $(link.dynamic.all.gnu)
-pedantic.opts.mac.gnu = $(pedantic.opts.all.gnu)
 
 p4_OPT.gnu   = $(-Q)march=nocona
 mc3_OPT.gnu  = $(-Q)march=corei7

dev/make/cmplr.gnu.ref.32e.mk → dev/make/compiler_definitions/gnu.mk

@@ -1,5 +1,5 @@
 #===============================================================================
-# Copyright 2023 Intel Corporation
+# Copyright contributors to the oneDAL project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,36 +16,19 @@
 
 #++
 #  g++ definitions for makefile
+#  This file contains definitions common to gnu on all platforms. It
+#  should only be included from files which have more specializations (e.g.
+#  gnu.32e.mk)
 #--
 
-PLATs.gnu = lnx32e mac32e
-
 CMPLRDIRSUFF.gnu = _gnu
 
 CORE.SERV.COMPILER.gnu = generic
 
 -Zl.gnu =
 -DEBC.gnu = -g
 
-COMPILER.all.gnu =  ${CXX} -m64 -fwrapv -fno-strict-overflow -fno-delete-null-pointer-checks \
-                    -DDAAL_REF -DONEDAL_REF -Werror -Wreturn-type
-
-link.dynamic.all.gnu = ${CXX} -m64
-
 pedantic.opts.all.gnu = -pedantic \
                         -Wall \
                         -Wextra \
                         -Wno-unused-parameter
-
-COMPILER.lnx.gnu = $(COMPILER.all.gnu)
-link.dynamic.lnx.gnu = $(link.dynamic.all.gnu)
-pedantic.opts.lnx.gnu = $(pedantic.opts.all.gnu)
-
-COMPILER.mac.gnu = $(COMPILER.all.gnu)
-link.dynamic.mac.gnu = $(link.dynamic.all.gnu)
-pedantic.opts.mac.gnu = $(pedantic.opts.all.gnu)
-
-p4_OPT.gnu   = $(-Q)march=nocona
-mc3_OPT.gnu  = $(-Q)march=corei7
-avx2_OPT.gnu = $(-Q)march=haswell
-skx_OPT.gnu  = $(-Q)march=skylake

dev/make/compiler_definitions/gnu.mkl.32e.mk

@@ -0,0 +1,24 @@
+#===============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+#++
+#  g++ definitions for makefile
+#--
+
+include dev/make/compiler_definitions/gnu.32e.mk
+
+COMPILER.lnx.gnu = $(COMPILER.all.gnu)
+COMPILER.mac.gnu = $(COMPILER.all.gnu)

dev/make/compiler_definitions/gnu.ref.32e.mk

@@ -0,0 +1,24 @@
+#===============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+#++
+#  g++ definitions for makefile
+#--
+
+include dev/make/compiler_definitions/gnu.32e.mk
+
+COMPILER.lnx.gnu = $(COMPILER.all.gnu) -DDAAL_REF -DONEDAL_REF
+COMPILER.mac.gnu = $(COMPILER.all.gnu) -DDAAL_REF -DONEDAL_REF

dev/make/cmplr.gnu.ref.arm.mk → dev/make/compiler_definitions/gnu.ref.arm.mk

@@ -18,31 +18,17 @@
 #  g++ definitions for makefile
 #--
 
-PLATs.gnu = lnxarm
-
-CMPLRDIRSUFF.gnu = _gnu
-
-CORE.SERV.COMPILER.gnu = generic
+include dev/make/compiler_definitions/gnu.mk
 
--Zl.gnu =
--DEBC.gnu = -g
+PLATs.gnu = lnxarm
 
 COMPILER.all.gnu =  ${CXX} -march=armv8-a+sve -fwrapv -fno-strict-overflow -fno-delete-null-pointer-checks \
                     -DDAAL_REF -DONEDAL_REF -DDAAL_CPU=sve -Werror -Wreturn-type
 
 link.dynamic.all.gnu = ${CXX} -march=native
 
-pedantic.opts.all.gnu = -pedantic \
-                        -Wall \
-                        -Wextra \
-                        -Wno-unused-parameter
-
 COMPILER.lnx.gnu = $(COMPILER.all.gnu)
 link.dynamic.lnx.gnu = $(link.dynamic.all.gnu)
 pedantic.opts.lnx.gnu = $(pedantic.opts.all.gnu)
 
-COMPILER.mac.gnu = $(COMPILER.all.gnu)
-link.dynamic.mac.gnu = $(link.dynamic.all.gnu)
-pedantic.opts.mac.gnu = $(pedantic.opts.all.gnu)
-
 a8sve_OPT.gnu = $(-Q)march=armv8-a+sve