From 93a3ce5276b72c7917b136065fff640014feab13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Tue, 4 Jun 2019 17:17:28 +0200 Subject: [PATCH] Add tests for alternative OS/compilers --- .CI/Jenkinsfile.more-compilers | 242 ++++++++++++++++++++ .CI/cache-32/Dockerfile | 6 + .CI/cache-disco/Dockerfile | 6 + .CI/cache-jessie/Dockerfile | 6 + .CI/cache-stretch/Dockerfile | 6 + .CI/common.groovy | 18 +- OMCompiler/Compiler/boot/Makefile.common | 2 +- OMCompiler/Compiler/runtime/Makefile.common | 2 +- OMCompiler/configure.ac | 35 +-- 9 files changed, 304 insertions(+), 19 deletions(-) create mode 100644 .CI/Jenkinsfile.more-compilers create mode 100644 .CI/cache-32/Dockerfile create mode 100644 .CI/cache-disco/Dockerfile create mode 100644 .CI/cache-jessie/Dockerfile create mode 100644 .CI/cache-stretch/Dockerfile diff --git a/.CI/Jenkinsfile.more-compilers b/.CI/Jenkinsfile.more-compilers new file mode 100644 index 00000000000..1c0c92a3019 --- /dev/null +++ b/.CI/Jenkinsfile.more-compilers @@ -0,0 +1,242 @@ +def common +pipeline { + agent none + options { + newContainerPerStage() + } + environment { + LC_ALL = 'C.UTF-8' + } + // stages are ordered according to execution time; highest time first + // nodes are selected based on a priority (in Jenkins config) + stages { + stage('Environment') { + agent { + label 'linux' + } + steps { + script { + if (changeRequest()) { + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) + } + common = load("${env.workspace}/.CI/common.groovy") + } + } + } + stage('builds') { + parallel { + stage('clang-32bit') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-32' + label 'linux' + args "--mount type=volume,source=runtest-clang-32-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary " + + "--privileged" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + } + steps { + script { + common.buildOMC('clang', 'clang++', '--without-hwloc') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + // gcc-4.9 jessie + stage('gcc-4.9 (Debian Jessie)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-jessie' + label 'linux' + args "--mount type=volume,source=runtest-gcc-4.9-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + HOME = "/cache/runtest" + } + steps { + script { + // Otherwise old git fails to clone... + sh 'git config --global user.name OpenModelica' + sh 'git config --global user.email openmodelica@ida.liu.se' + common.buildOMC('gcc-4.9', 'g++-4.9', '--without-hwloc') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + stage('clang-3.5 (Debian Jessie)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-jessie' + label 'linux' + args "--mount type=volume,source=runtest-clang-3.5-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + HOME = "/cache/runtest" + } + steps { + script { + // Otherwise old git fails to clone... + sh 'git config --global user.name OpenModelica' + sh 'git config --global user.email openmodelica@ida.liu.se' + common.buildOMC('clang-3.5', 'clang++-3.5', '--without-hwloc') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + // gcc-5 Ubuntu Xenial + stage('gcc-5') { + steps { + print "Ubuntu Xenial GCC 5 tested on every pull request" + } + } + stage('clang-3.8 (Ubuntu Xenial)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-xenial' + label 'linux' + args "--mount type=volume,source=runtest-clang-3.8-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + } + steps { + script { + common.buildOMC('clang-3.8', 'clang++-3.8', '--without-hwloc') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + // gcc-6 Debian stretch + stage('gcc-6 (Debian Stretch)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-stretch' + label 'linux' + args "--mount type=volume,source=runtest-gcc-6-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + } + steps { + script { + common.buildOMC('gcc-6', 'g++-6', '--without-hwloc') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + // clang 3.8 also in Debian stretch... + // gcc-7 Ubuntu Bionic + stage('gcc-7 (Ubuntu Bionic)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache' + label 'linux' + args "--mount type=volume,source=runtest-gcc-7-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + } + steps { + script { + common.buildOMC('gcc-7', 'g++-7', '--without-hwloc') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + stage('clang-6.0') { + steps { + print "Ubuntu Bionic clang 6.0 tested on every pull request" + } + } + // gcc-8 in Ubuntu Disco + stage('gcc-8 (Ubuntu Disco)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-disco' + label 'linux' + args "--mount type=volume,source=runtest-gcc-8-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + } + steps { + script { + common.buildOMC('gcc-8', 'g++-8', '') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + stage('clang-8 (Ubuntu Disco)') { + agent { + dockerfile { + additionalBuildArgs '--pull' + dir '.CI/cache-disco' + label 'linux' + args "--mount type=volume,source=runtest-clang-8-cache,target=/cache/runtest " + + "--mount type=volume,source=omlibrary-cache,target=/cache/omlibrary" + } + } + environment { + RUNTESTDB = "/cache/runtest/" + LIBRARIES = "/cache/omlibrary" + } + steps { + script { + common.buildOMC('clang-8', 'clang++-8', '') + common.makeLibsAndCache() + common.buildGUI('') + common.partest() + } + } + } + } + } + } +} diff --git a/.CI/cache-32/Dockerfile b/.CI/cache-32/Dockerfile new file mode 100644 index 00000000000..ab15f4a56d8 --- /dev/null +++ b/.CI/cache-32/Dockerfile @@ -0,0 +1,6 @@ +# Cannot be parametrized in Jenkins... +FROM docker.openmodelica.org/build-deps:v1.13-i386 + +# We don't know the uid of the user who will build, so make the /cache directories world writable + +RUN mkdir -p /cache/runtest/ /cache/omlibrary/ && chmod ugo+rwx /cache/runtest/ /cache/omlibrary/ diff --git a/.CI/cache-disco/Dockerfile b/.CI/cache-disco/Dockerfile new file mode 100644 index 00000000000..630c0d9894f --- /dev/null +++ b/.CI/cache-disco/Dockerfile @@ -0,0 +1,6 @@ +# Cannot be parametrized in Jenkins... +FROM docker.openmodelica.org/build-deps:v1.13-disco + +# We don't know the uid of the user who will build, so make the /cache directories world writable + +RUN mkdir -p /cache/runtest/ /cache/omlibrary/ && chmod ugo+rwx /cache/runtest/ /cache/omlibrary/ diff --git a/.CI/cache-jessie/Dockerfile b/.CI/cache-jessie/Dockerfile new file mode 100644 index 00000000000..2e6a067a072 --- /dev/null +++ b/.CI/cache-jessie/Dockerfile @@ -0,0 +1,6 @@ +# Cannot be parametrized in Jenkins... +FROM docker.openmodelica.org/build-deps:v1.13-jessie + +# We don't know the uid of the user who will build, so make the /cache directories world writable + +RUN mkdir -p /cache/runtest/ /cache/omlibrary/ && chmod ugo+rwx /cache/runtest/ /cache/omlibrary/ diff --git a/.CI/cache-stretch/Dockerfile b/.CI/cache-stretch/Dockerfile new file mode 100644 index 00000000000..57878943712 --- /dev/null +++ b/.CI/cache-stretch/Dockerfile @@ -0,0 +1,6 @@ +# Cannot be parametrized in Jenkins... +FROM docker.openmodelica.org/build-deps:v1.13-stretch + +# We don't know the uid of the user who will build, so make the /cache directories world writable + +RUN mkdir -p /cache/runtest/ /cache/omlibrary/ && chmod ugo+rwx /cache/runtest/ /cache/omlibrary/ diff --git a/.CI/common.groovy b/.CI/common.groovy index 0c0cc47a3e7..f83c38a28bc 100644 --- a/.CI/common.groovy +++ b/.CI/common.groovy @@ -5,36 +5,46 @@ void standardSetup() { } def numPhysicalCPU() { + if (env.JENKINS_NUM_PHYSICAL_CPU) { + return env.JENKINS_NUM_PHYSICAL_CPU + } def uname = sh script: 'uname', returnStdout: true if (uname.startsWith("Darwin")) { - return sh ( + env.JENKINS_NUM_PHYSICAL_CPU = sh ( script: 'sysctl hw.physicalcpu_max | cut -d" " -f2', returnStdout: true ).trim().toInteger() ?: 1 } else { - return sh ( + env.JENKINS_NUM_PHYSICAL_CPU = sh ( script: 'lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l', returnStdout: true ).trim().toInteger() ?: 1 } + return env.JENKINS_NUM_PHYSICAL_CPU } def numLogicalCPU() { + if (env.JENKINS_NUM_LOGICAL_CPU) { + return env.JENKINS_NUM_LOGICAL_CPU + } def uname = sh script: 'uname', returnStdout: true if (uname.startsWith("Darwin")) { - return sh ( + env.JENKINS_NUM_LOGICAL_CPU = sh ( script: 'sysctl hw.logicalcpu_max | cut -d" " -f2', returnStdout: true ).trim().toInteger() ?: 1 } else { - return sh ( + env.JENKINS_NUM_LOGICAL_CPU = sh ( script: 'lscpu -p | egrep -v "^#" | wc -l', returnStdout: true ).trim().toInteger() ?: 1 } + return env.JENKINS_NUM_LOGICAL_CPU } void partest(cache=true, extraArgs='') { + sh "rm -f omc-diff.skip && make -C testsuite/difftool clean && make --output-sync -C testsuite/difftool" + sh 'build/bin/omc-diff -v1.4' sh ("""#!/bin/bash -x ulimit -t 1500 ulimit -v 6291456 # Max 6GB per process diff --git a/OMCompiler/Compiler/boot/Makefile.common b/OMCompiler/Compiler/boot/Makefile.common index 6f62d40f31e..e4758ac46f7 100644 --- a/OMCompiler/Compiler/boot/Makefile.common +++ b/OMCompiler/Compiler/boot/Makefile.common @@ -26,7 +26,7 @@ ifeq ($(BOOTSTRAP_STAGE_1),1) OMCRUNTIMEAR=$(OMCRUNTIMEAR1) OMPARSEAR=../../Parser/libomparse-boot.a LOMPARSE=$(OMPARSEAR) -LCOMPILERRUNTIME=$(TOP_DIR)/Compiler/runtime/libomcgraphstream.a $(TOP_DIR)/Compiler/runtime/libomcbackendruntime.a $(abspath $(OMCRUNTIMEAR)) +LCOMPILERRUNTIME=$(TOP_DIR)/Compiler/runtime/libomcgraphstream.a $(TOP_DIR)/Compiler/runtime/libomcbackendruntime.a $(abspath $(OMCRUNTIMEAR)) $(RPATH_BOOT_STAGE_1) FMILIB_OR_BOOT= else OMCRUNTIMEAR=$(OMCRUNTIMEAR2) diff --git a/OMCompiler/Compiler/runtime/Makefile.common b/OMCompiler/Compiler/runtime/Makefile.common index b70d269460f..32401030b49 100644 --- a/OMCompiler/Compiler/runtime/Makefile.common +++ b/OMCompiler/Compiler/runtime/Makefile.common @@ -85,7 +85,7 @@ libomcruntime-boot.dylib : $(OMC_OBJ_BOOT) libomcruntime.so : $(OMC_OBJ) rm -f $@ - $(CXX) -shared $(RPATH) -o $@ $(OMC_OBJ) $(LDFLAGS) + $(CXX) -shared $(RPATH) $(RPATH_BOOT_STAGE_1) -o $@ $(OMC_OBJ) $(LDFLAGS) libomcruntime-boot.so : $(OMC_OBJ_BOOT) rm -f $@ diff --git a/OMCompiler/configure.ac b/OMCompiler/configure.ac index 3d2ce53cce9..dbd52b10420 100644 --- a/OMCompiler/configure.ac +++ b/OMCompiler/configure.ac @@ -210,11 +210,7 @@ if echo $CC | grep -v ccc-analyzer; then # ccc-analyzer calls gcc first and then # GCC 7.x issues: # - RUNPATH instead of RPATH, disable that via -Wl,--disable-new-dtags # - sqrt from libcolamd is not found in -lm if -Wl,--no-as-needed is not specified - if $CC --version | grep -q gcc && $CC -dumpversion | grep -q "^7"; then - TRY_FLAGS="$TRY_FLAGS -Wl,--no-as-needed" - RPATH_EXTRA="-Wl,--disable-new-dtags" - fi - + TRY_FLAGS_LD="$TRY_FLAGS_LD -Wl,--no-as-needed -Wl,--disable-new-dtags" fi AC_MSG_CHECKING([for flags to append to CFLAGS ($TRY_FLAGS)]) @@ -224,6 +220,15 @@ for flag in $TRY_FLAGS; do AC_TRY_LINK([], [return 0;], [CFLAGS="$OLD_CFLAGS $flag"],[CFLAGS="$OLD_CFLAGS"]) done AC_MSG_RESULT([$CFLAGS]) + +AC_MSG_CHECKING([for flags to append to LDFLAGS ($TRY_FLAGS_LD)]) +for flag in $TRY_FLAGS_LD; do + OLD_LDFLAGS="$LDFLAGS" + LDFLAGS="$flag -Werror" + AC_TRY_LINK([], [return 0;], [LDFLAGS="$OLD_LDFLAGS $flag"],[LDFLAGS="$OLD_LDFLAGS"]) +done +AC_MSG_RESULT([$LDFLAGS]) + OLD_CFLAGS="$CFLAGS" for flag in -Wno-parentheses-equality -Wno-unused-variable; do CFLAGS="$RUNTIMECFLAGS $flag -Werror" @@ -610,12 +615,16 @@ fi AC_ARG_WITH(hwloc, [ --without-hwloc (default to looking for it)],[ LIBS="" - AC_CHECK_HEADERS(hwloc.h, - [AC_CHECK_LIB(hwloc,hwloc_get_nbobjs_by_depth, - [WITH_HWLOC="1";OMC_LIBS="$OMC_LIBS -lhwloc"], - [WITH_HWLOC="0"])], - [WITH_HWLOC="0"] - ) + if ! test "$withval" = "no"; then + AC_CHECK_HEADERS(hwloc.h, + [AC_CHECK_LIB(hwloc,hwloc_get_nbobjs_by_depth, + [WITH_HWLOC="1";OMC_LIBS="$OMC_LIBS -lhwloc"], + [WITH_HWLOC="0"])], + [WITH_HWLOC="0"] + ) + else + WITH_HWLOC="0" + fi ],[ WITH_HWLOC="0" ]) @@ -778,8 +787,8 @@ else LDFLAGS="$LDFLAGS -Wl,-rpath-link,$OMBUILDDIR/lib/$host_short/omc" RT_LDFLAGS_SHARED="-Wl,-rpath-link,$OMBUILDDIR/lib/$host_short/omc" OMCRUNTIME_SHARED_LDFLAGS="$RT_LDFLAGS $LIBLPSOLVE55 -lzmq $LIBUUID" - RPATH="$RPATH_EXTRA -Wl,-z,origin -Wl,-rpath,'\$\$ORIGIN/../lib/$host_short/omc' -Wl,-rpath,'\$\$ORIGIN'" - RPATH_QMAKE="$RPATH_EXTRA -Wl,-z,origin -Wl,-rpath,\\'\\\$\$ORIGIN/../lib/$host_short/omc\\' -Wl,-rpath,\\'\\\$\$ORIGIN\\'" + RPATH="-Wl,-z,origin -Wl,-rpath,'\$\$ORIGIN/../lib/$host_short/omc' -Wl,-rpath,'\$\$ORIGIN'" + RPATH_QMAKE="-Wl,-z,origin -Wl,-rpath,\\'\\\$\$ORIGIN/../lib/$host_short/omc\\' -Wl,-rpath,\\'\\\$\$ORIGIN\\'" OS_TARGET="linux" AR_SH="$AR -ru" BOOTSTRAP_STATIC=""