Skip to content

Commit 716cbd9

Browse files
committed
Add Jenkins pipelines for slower jobs
1 parent 75252ef commit 716cbd9

File tree

5 files changed

+292
-183
lines changed

5 files changed

+292
-183
lines changed

.CI/Jenkinsfile.osx

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
def common
2+
pipeline {
3+
agent none
4+
environment {
5+
LC_ALL = 'C.UTF-8'
6+
}
7+
stages {
8+
stage('MacOS') {
9+
agent {
10+
node {
11+
label 'osx'
12+
}
13+
}
14+
environment {
15+
RUNTESTDB = "/Users/hudson/jenkins-cache/runtest/"
16+
LIBRARIES = "/Users/hudson/jenkins-cache/omlibrary"
17+
}
18+
steps {
19+
script {
20+
common = load("${env.workspace}/.CI/common.groovy")
21+
}
22+
common.buildOMC('cc', 'c++', '')
23+
common.buildGUI('')
24+
common.makeLibsAndCache()
25+
common.partest()
26+
}
27+
}
28+
}
29+
}

.CI/Jenkinsfile.static_analysis

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
def common
2+
3+
pipeline {
4+
agent none
5+
environment {
6+
LC_ALL = 'C.UTF-8'
7+
}
8+
stages {
9+
stage('abort running builds') {
10+
/* If this is a change request, cancel previous builds if you push a new commit */
11+
steps {
12+
script {
13+
def buildNumber = env.BUILD_NUMBER as int
14+
if (buildNumber > 1) milestone(buildNumber - 1)
15+
milestone(buildNumber)
16+
}
17+
}
18+
}
19+
stage('clang-analyzer') {
20+
agent {
21+
dockerfile {
22+
additionalBuildArgs '--pull'
23+
dir '.CI/clang-tools'
24+
label 'linux'
25+
}
26+
}
27+
steps {
28+
script {
29+
common = load("${env.workspace}/.CI/common.groovy")
30+
common.buildOMC('clang', 'clang++', '')
31+
def numCPU = common.numLogicalCPU()
32+
def tagName = common.tagName()
33+
sh "test ! -z ${tagName}"
34+
sh """
35+
mv build .build.save
36+
# Save&restore 3rdParty so we do not analyze these sources
37+
rm -rf OMCompiler/3rdParty.save
38+
cp -a OMCompiler/3rdParty OMCompiler/3rdParty.save
39+
make -j${numCPU} clean
40+
rm -rf OMCompiler/3rdParty
41+
mv OMCompiler/3rdParty.save OMCompiler/3rdParty
42+
# Skips bootstrapping
43+
cp -a .build.save build
44+
export OMC="`pwd`/.build.save/bin/omc"
45+
test -f "\$OMC"
46+
scan-build ./configure --with-omniORB --disable-modelica3d CFLAGS='-O2 -march=native' --with-omc="\$OMC" "CXXFLAGS=-std=c++11" --without-omlibrary
47+
mkdir -p html
48+
"""
49+
sh "scan-build -o html make -j${numCPU}"
50+
sshPublisher(publishers: [sshPublisherDesc(configName: 'OMC-scan-build', transfers: [
51+
sshTransfer(remoteDirectory: ".tmp/${tagName}", removePrefix: "html", sourceFiles: 'html/**'),
52+
sshTransfer(execCommand: "rm -rf '/var/www/scan-build/${tagName}' && mv '/var/www/scan-build/.tmp/${tagName}' '/var/www/scan-build/${tagName}'")
53+
])])
54+
}
55+
}
56+
}
57+
}
58+
}

.CI/clang-tools/Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Cannot be parametrized in Jenkins...
2+
FROM docker.openmodelica.org/build-deps:v1.13
3+
4+
RUN apt-get update && apt-get install -qyy clang-tools && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.CI/common.groovy

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
void standardSetup() {
2+
echo "${env.NODE_NAME}"
3+
// Jenkins cleans with -fdx; --ffdx is needed to remove git repositories
4+
sh "git clean -ffdx && git submodule foreach --recursive git clean -ffdx"
5+
}
6+
7+
def numPhysicalCPU() {
8+
def uname = sh script: 'uname', returnStdout: true
9+
if (uname.startsWith("Darwin")) {
10+
return sh (
11+
script: 'sysctl hw.physicalcpu_max | cut -d" " -f2',
12+
returnStdout: true
13+
).trim().toInteger() ?: 1
14+
} else {
15+
return sh (
16+
script: 'lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l',
17+
returnStdout: true
18+
).trim().toInteger() ?: 1
19+
}
20+
}
21+
22+
def numLogicalCPU() {
23+
def uname = sh script: 'uname', returnStdout: true
24+
if (uname.startsWith("Darwin")) {
25+
return sh (
26+
script: 'sysctl hw.logicalcpu_max | cut -d" " -f2',
27+
returnStdout: true
28+
).trim().toInteger() ?: 1
29+
} else {
30+
return sh (
31+
script: 'lscpu -p | egrep -v "^#" | wc -l',
32+
returnStdout: true
33+
).trim().toInteger() ?: 1
34+
}
35+
}
36+
37+
void partest(cache=true, extraArgs='') {
38+
sh ("""#!/bin/bash -x
39+
ulimit -t 1500
40+
ulimit -v 6291456 # Max 6GB per process
41+
42+
cd testsuite/partest
43+
./runtests.pl -j${numPhysicalCPU()} -nocolour -with-xml ${extraArgs}
44+
CODE=\$?
45+
test \$CODE = 0 -o \$CODE = 7 || exit 1
46+
"""
47+
+ (cache ?
48+
"""
49+
if test \$CODE = 0; then
50+
mkdir -p "${env.RUNTESTDB}/"
51+
cp ../runtest.db.* "${env.RUNTESTDB}/"
52+
fi
53+
""" : ''))
54+
junit 'testsuite/partest/result.xml'
55+
}
56+
57+
void patchConfigStatus() {
58+
// Running on nodes with different paths for the workspace
59+
sh 'sed -i "s,--with-ombuilddir=[A-Za-z0-9/_-]*,--with-ombuilddir=`pwd`/build," config.status OMCompiler/config.status'
60+
}
61+
62+
void makeLibsAndCache(libs='core') {
63+
// If we don't have any result, copy to the master to get a somewhat decent cache
64+
sh "cp -f ${env.RUNTESTDB}/${cacheBranch()}/runtest.db.* testsuite/ || " +
65+
"cp -f ${env.RUNTESTDB}/master/runtest.db.* testsuite/ || true"
66+
// env.WORKSPACE is null in the docker agent, so link the svn/git cache afterwards
67+
sh "mkdir -p '${env.LIBRARIES}/svn' '${env.LIBRARIES}/git'"
68+
sh "find libraries"
69+
sh "ln -s '${env.LIBRARIES}/svn' '${env.LIBRARIES}/git' libraries/"
70+
generateTemplates()
71+
def cmd = "make -j${numLogicalCPU()} --output-sync omlibrary-${libs} ReferenceFiles"
72+
if (env.SHARED_LOCK) {
73+
lock(env.SHARED_LOCK) {
74+
sh cmd
75+
}
76+
} else {
77+
sh cmd
78+
}
79+
}
80+
81+
void buildOMC(CC, CXX, extraFlags) {
82+
standardSetup()
83+
sh 'autoconf'
84+
// Note: Do not use -march=native since we might use an incompatible machine in later stages
85+
sh "./configure CC='${CC}' CXX='${CXX}' FC=gfortran CFLAGS=-Os --with-cppruntime --without-omc --without-omlibrary --with-omniORB --enable-modelica3d ${extraFlags}"
86+
// OMSimulator requires HOME to be set and writeable
87+
sh "HOME='${env.WORKSPACE}' make -j${numPhysicalCPU()} --output-sync omc omc-diff omsimulator"
88+
sh 'find build/lib/*/omc/ -name "*.so" -exec strip {} ";"'
89+
}
90+
91+
void buildGUI(stash) {
92+
standardSetup()
93+
if (stash) {
94+
unstash stash
95+
}
96+
sh 'autoconf'
97+
patchConfigStatus()
98+
sh 'CONFIG=`./config.status --config` && ./configure `eval $CONFIG`'
99+
sh 'touch omc.skip omc-diff.skip ReferenceFiles.skip omsimulator.skip && make -q omc omc-diff ReferenceFiles omsimulator' // Pretend we already built omc since we already did so
100+
sh "make -j${numPhysicalCPU()} --output-sync" // Builds the GUI files
101+
}
102+
103+
void generateTemplates() {
104+
patchConfigStatus()
105+
// Runs Susan again, for bootstrapping tests, etc
106+
sh 'make -C OMCompiler/Compiler/Template/ -f Makefile.in OMC=$PWD/build/bin/omc'
107+
sh 'cd OMCompiler && ./config.status'
108+
sh './config.status'
109+
}
110+
111+
def getVersion() {
112+
return (sh (script: 'build/bin/omc --version | grep -o "v[0-9]\\+[.][0-9]\\+[.][0-9]\\+[^ ]*"', returnStdout: true)).replaceAll("\\s","")
113+
}
114+
115+
void compliance() {
116+
standardSetup()
117+
unstash 'omc-clang'
118+
makeLibsAndCache('all')
119+
sh 'build/bin/omc -g=MetaModelica build/share/doc/omc/testmodels/ComplianceSuite.mos'
120+
sh "mv ${env.COMPLIANCEPREFIX}.html ${env.COMPLIANCEPREFIX}-current.html"
121+
sh "test -f ${env.COMPLIANCEPREFIX}.xml"
122+
// Only publish openmodelica-current.html if we are running master
123+
sh "cp -p ${env.COMPLIANCEPREFIX}-current.html ${env.COMPLIANCEPREFIX}${cacheBranch()=='master' ? '' : ('-' + cacheBranch()).replace('/','-')}-${getVersion()}.html"
124+
sh "test ! '${cacheBranch()}' = 'master' || rm -f ${env.COMPLIANCEPREFIX}-current.html"
125+
stash name: "${env.COMPLIANCEPREFIX}", includes: "${env.COMPLIANCEPREFIX}-*.html"
126+
archiveArtifacts "${env.COMPLIANCEPREFIX}*${getVersion()}.html, ${env.COMPLIANCEPREFIX}.failures"
127+
// get rid of freaking %
128+
sh "sed -i.bak 's/%/\\%/g' ${env.COMPLIANCEPREFIX}.ignore.xml && sed -i.bak 's/[^[:print:]]/ /g' ${env.COMPLIANCEPREFIX}.ignore.xml"
129+
junit "${env.COMPLIANCEPREFIX}.ignore.xml"
130+
}
131+
132+
def cacheBranch() {
133+
return "${env.CHANGE_TARGET ?: env.GIT_BRANCH}"
134+
}
135+
136+
def tagName() {
137+
def name = env.TAG_NAME ?: cacheBranch()
138+
return name == "master" ? "latest" : name
139+
}
140+
141+
return this

0 commit comments

Comments
 (0)