Update JVM args (#517)

- Use recommended Java 1.8 (full-speed) debug args, not 1.3 - Use random debug port to avoid conflicts - Extract server.jvmargs property - Limit metaspace size, remove obsolete PermGen args - Enable GC logging - Archive gc.log/heapdump in subdirs - Separate archives by appserver
zanata · Sep 26, 2017 · ccb4cfe · ccb4cfe
1 parent 2c3c8e8
commit ccb4cfe
Show file tree

Hide file tree

Showing 5 changed files with 99 additions and 89 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -376,37 +376,39 @@ void integrationTests(String appserver) {
     currentBuild.displayName = currentBuild.displayName + " {${env.NODE_NAME}}"
 
     echo "WORKSPACE=${env.WORKSPACE}"
-    checkout scm
-    // Clean the workspace
-    sh "git clean -fdx"
+    // we want parallel builds to use different directories so we can distinguish the archived files
+    dir(appserver) {
+      checkout scm
+      // Clean the workspace
+      sh "git clean -fdx"
 
-    unstash 'generated-files'
+      unstash 'generated-files'
 
-    /* touch all target */
-    //sh "find `pwd -P` -path '*/target/*' -print -exec touch '{}' \\;"
+      /* touch all target */
+      //sh "find `pwd -P` -path '*/target/*' -print -exec touch '{}' \\;"
 
-    xvfb {
-      withPorts {
-        echo "Running maven build for integration tests"
+      xvfb {
+        withPorts {
+          echo "Running maven build for integration tests"
 
-        // Run the maven build
-        echo "env.DISPLAY=${env.DISPLAY}"
-        echo "env.JBOSS_HTTP_PORT=${env.JBOSS_HTTP_PORT}"
-        echo "env.JBOSS_HTTPS_PORT=${env.JBOSS_HTTPS_PORT}"
+          // Run the maven build
+          echo "env.DISPLAY=${env.DISPLAY}"
+          echo "env.JBOSS_HTTP_PORT=${env.JBOSS_HTTP_PORT}"
+          echo "env.JBOSS_HTTPS_PORT=${env.JBOSS_HTTPS_PORT}"
 
-        // Build up Maven options for running functional tests:
+          // Build up Maven options for running functional tests:
 
-        // avoid port conflict for debugger:
-        def ftOpts = '-Dcargo.debug.jvm.args= '
+          // disable debugging:
+          def ftOpts = '-Dcargo.debug.jvm.args= '
 
-        // run *all* functional tests in these branches only:
-        if (env.BRANCH_NAME in mainlineBranches || resolveAllFuncTests()) {
-          ftOpts += '-DallFuncTests '
-        }
+          // run *all* functional tests in these branches only:
+          if (env.BRANCH_NAME in mainlineBranches || resolveAllFuncTests()) {
+            ftOpts += '-DallFuncTests '
+          }
 
-        // skip recompilation, unit tests, static analysis
-        // (done in Build stage):
-        ftOpts += """\
+          // skip recompilation, unit tests, static analysis
+          // (done in Build stage):
+          ftOpts += """\
               -Dgwt.compiler.skip \
               -Dmaven.main.skip \
               -Dskip.npminstall \
@@ -418,7 +420,7 @@ void integrationTests(String appserver) {
               $gwtOpts \
         """
 
-        def mvnResult = sh returnStatus: true, script: """\
+          def mvnResult = sh returnStatus: true, script: """\
             ./run-clean.sh ./mvnw -e -V -T 1 \
             -Dbuildtime.output.csv -Dbuildtime.output.csv.file=buildtime.csv \
             install \
@@ -430,41 +432,48 @@ void integrationTests(String appserver) {
             -Dwebdriver.chrome.driver=/opt/chromedriver \
             ${ftOpts}
         """
-        /* TODO
+          /* TODO
         -Dassembly.skipAssembly \
         -DskipAppassembler \
         -DskipShade \
          */
 
-        // retain traceability report and build time info
-        archive(includes: "server/functional-test/target/**/traceability.json,target/buildtime.csv")
-
-        if (mvnResult != 0) {
-          notify.testResults(appserver, 'UNSTABLE', 'Failed maven build for integration tests')
-          currentBuild.result = 'UNSTABLE'
-
-          // gather db/app logs and screenshots to help debugging
-          archive(
-                  includes: 'server/functional-test/target/**/*.log,server/functional-test/target/screenshots/**',
-                  excludes: '**/BACKUP-*.log')
-        } else {
-          notify.testResults(appserver, 'SUCCESS')
-        }
+          // retain traceability report and build time info
+          // work from parent directory so that $appserver will appear at the beginning of the archive paths
+          dir('..') {
+            archive('*/server/functional-test/target/**/traceability.json,*/target/buildtime.csv')
+          }
+          if (mvnResult != 0) {
+            notify.testResults(appserver, 'UNSTABLE',
+                    'Failed maven build for integration tests')
+            currentBuild.result = 'UNSTABLE'
+
+            // gather db/app/gc logs, heap dumps and screenshots to help debugging
+            // work from parent directory so that $appserver will appear at the beginning of the archive paths
+            dir('..') {
+              archive(
+                      includes: '*/server/functional-test/target/**/*.log,*/server/functional-test/target/screenshots/**,*/server/*/target/**/gc.log*,*/server/*/target/**/*.hprof',
+                      excludes: '**/BACKUP-*.log')
+            }
+          } else {
+            notify.testResults(appserver, 'SUCCESS')
+          }
 
-        echo "Capturing JUnit results"
-        if (setJUnitPrefix(appserver, failsafeTestReports)) {
-          junit(testResults: "**/${failsafeTestReports}",
-                  // NB: if this is enabled, make sure (a) max history in Jenkins
-                  // Configuration is small (eg 3) or
-                  // (b) https://issues.jenkins-ci.org/browse/JENKINS-33168 is fixed.
-                  testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
-          )
-          // Reduce workspace size
-          sh "git clean -fdx"
-        } else {
-          notify.error("No integration test result for $appserver")
-          currentBuild.result = 'FAILURE'
-          error "no integration test results for $appserver"
+          echo "Capturing JUnit results"
+          if (setJUnitPrefix(appserver, failsafeTestReports)) {
+            junit(testResults: "**/${failsafeTestReports}",
+                    // NB: if this is enabled, make sure (a) max history in Jenkins
+                    // Configuration is small (eg 3) or
+                    // (b) https://issues.jenkins-ci.org/browse/JENKINS-33168 is fixed.
+                    testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
+            )
+            // Reduce workspace size
+            sh "git clean -fdx"
+          } else {
+            notify.error("No integration test result for $appserver")
+            currentBuild.result = 'FAILURE'
+            error "no integration test results for $appserver"
+          }
         }
       }
     }

diff --git a/server/functional-test/pom.xml b/server/functional-test/pom.xml
@@ -71,10 +71,8 @@
     <webdriver.screenshot.dir>${project.build.directory}/screenshots</webdriver.screenshot.dir>
     <webdriver.wait>20</webdriver.wait>
     <googleopenid.credentials>zanata.user.1:</googleopenid.credentials>
-    <!-- on jenkins, this needs to be set to empty - so that cargo can shutdown. see http://stackoverflow.com/questions/1096642/tomcat-failed-to-shutdown -->
     <cargo.debug.jvm.args>
-      -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8787
-      -Xnoagent -Djava.compiler=NONE
+      -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0
     </cargo.debug.jvm.args>
     <!-- this property can be used to control what test needs to be run by failsafe -->
     <default.test.patterns>**/BasicAcceptanceTestSuite.java</default.test.patterns>
@@ -753,14 +751,7 @@
                 <configuration>
                   <configuration>
                     <properties>
-                      <cargo.jvmargs>
-                        <!-- NB: JDom parser doesn't like comments starting with '-' -->
-                        -XX:-OmitStackTraceInFastThrow
-                        -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target
-                        -XX:PermSize=512m -XX:MaxPermSize=1024
-                        -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled
-                        ${cargo.debug.jvm.args}
-                      </cargo.jvmargs>
+                      <cargo.jvmargs>${server.jvmargs}</cargo.jvmargs>
                     </properties>
                   </configuration>
                 </configuration>
@@ -770,14 +761,7 @@
                 <configuration>
                   <configuration>
                     <properties>
-                      <cargo.jvmargs>
-                        <!-- NB: JDom parser doesn't like comments starting with '-' -->
-                        -XX:-OmitStackTraceInFastThrow
-                        -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target
-                        -XX:PermSize=512m -XX:MaxPermSize=1024
-                        -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled
-                        ${cargo.debug.jvm.args}
-                      </cargo.jvmargs>
+                      <cargo.jvmargs>${server.jvmargs}</cargo.jvmargs>
                     </properties>
                   </configuration>
                 </configuration>
@@ -1023,7 +1007,7 @@
                 <echo />
                 <echo>-DallFuncTests to enable all functional tests (defaults to smoke tests)</echo>
                 <echo />
-                <echo>-Dcargo.debug.jvm.args : If not set by default will listen to port 8787. Need to set to empty on jenkins</echo>
+                <echo>-Dcargo.debug.jvm.args : If not set by default will listen to a random port.</echo>
                 <echo />
                 <echo>-Dzanata.target.version=version of zanata to deploy. Default is: ${project.parent.version}</echo>
                 <echo>-Dzanata.instance.url=http://${cargo.host}:${cargo.servlet.port}/${context.path}</echo>

diff --git a/server/pom.xml b/server/pom.xml
@@ -106,6 +106,28 @@
     <!-- Version of the management api used by Arquillian -->
     <wildfly.arquillian.version>2.0.0.Final</wildfly.arquillian.version>
     <jackson2.version>2.7.2</jackson2.version>
+
+    <!-- Ref: http://blog.sokolenko.me/2014/11/javavm-options-production.html -->
+    <server.jvmargs>
+      <!-- NB: JDom parser doesn't like comments starting with '-' -->
+      -server
+      -showversion
+      -XX:+PrintCommandLineFlags
+      -Xmx1536m
+      -XX:MaxMetaspaceSize=1g
+      -XX:-OmitStackTraceInFastThrow
+      -XX:+PrintGCDateStamps
+      -verbose:gc
+      -XX:+PrintGCDetails
+      -Xloggc:gc.log
+      -XX:+UseGCLogFileRotation
+      -XX:NumberOfGCLogFiles=10
+      -XX:GCLogFileSize=100M
+      -XX:+HeapDumpOnOutOfMemoryError
+      -XX:HeapDumpPath=.
+      -Djava.net.preferIPv4Stack=true
+      -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0
+    </server.jvmargs>
   </properties>
 
   <dependencyManagement>
@@ -2183,6 +2205,12 @@
               <goals>
                 <goal>integration-test</goal>
               </goals>
+              <configuration>
+                <systemPropertyVariables combine.children="append">
+                  <!-- Override gc.log and heapdump dir for Arquillian tests -->
+                  <server.jvmargs>${server.jvmargs} -Xloggc:target/gc.log -XX:HeapDumpPath=target</server.jvmargs>
+                </systemPropertyVariables>
+              </configuration>
             </execution>
             <execution>
               <id>verify</id>

diff --git a/server/services/src/test/resources/arquillian.xml b/server/services/src/test/resources/arquillian.xml
@@ -15,10 +15,7 @@
       <property name="jbossHome">${jboss.home}</property>
       <!-- We can use the JAVA_HOME env variable instead of this -->
       <!--             <property name="javaHome">/usr/lib/jvm/jre-1.6.0-openjdk.x86_64</property> -->
-      <!--
-                  <property name="javaVmArguments">-Xmx1536m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8787</property>
-      -->
-      <property name="javaVmArguments">-Xmx1536m -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target -Djava.net.preferIPv4Stack=true</property>
+      <property name="javaVmArguments">${server.jvmargs}</property>
       <!-- From JBOSS_HOME/standalone/configuration -->
       <property name="serverConfig">standalone-full.xml</property>
       <!--<property name="allowConnectingToRunningServer">true</property>-->
@@ -32,14 +29,10 @@
   <container qualifier="wildfly" default="true">
     <configuration>
       <property name="jbossHome">${jboss.home}</property>
-      <!--<property name="javaVmArguments">-Xmx1536m -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target -Djava.net.preferIPv4Stack=true -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8787</property>-->
-      <property name="javaVmArguments">-Xmx1536m -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target -Djava.net.preferIPv4Stack=true</property>
-       From JBOSS_HOME/standalone/configuration
+      <property name="javaVmArguments">${server.jvmargs}</property>
+      <!-- From JBOSS_HOME/standalone/configuration -->
       <property name="serverConfig">standalone-full.xml</property>
-      <property name="managementAddress">127.0.0.1</property>
       <!-- NB: don't put whitespace around the number -->
-      <!--<property name="managementPort">${jboss.management.http.port,env.JBOSS_MANAGEMENT_HTTP_PORT:56999}</property>-->
-      <!--<property name="managementPort">${jboss.management.http.port,env.JBOSS_MANAGEMENT_HTTP_PORT:9990}</property>-->
       <property name="managementPort">${jboss.management.http.port,env.JBOSS_MANAGEMENT_HTTP_PORT:10090}</property>
       <property name="startupTimeoutInSeconds">${startup.timeout:60}</property>
     </configuration>

diff --git a/server/zanata-war/src/test/resources/arquillian.xml b/server/zanata-war/src/test/resources/arquillian.xml
@@ -15,10 +15,7 @@
       <property name="jbossHome">${jboss.home}</property>
       <!-- We can use the JAVA_HOME env variable instead of this -->
       <!--             <property name="javaHome">/usr/lib/jvm/jre-1.6.0-openjdk.x86_64</property> -->
-      <!--
-                  <property name="javaVmArguments">-Xmx1536m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8787</property>
-      -->
-      <property name="javaVmArguments">-Xmx1536m -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target -Djava.net.preferIPv4Stack=true</property>
+      <property name="javaVmArguments">${server.jvmargs}</property>
       <!-- From JBOSS_HOME/standalone/configuration -->
       <property name="serverConfig">standalone-full.xml</property>
       <!--<property name="allowConnectingToRunningServer">true</property>-->
@@ -32,8 +29,7 @@
   <container qualifier="wildfly" default="true">
     <configuration>
       <property name="jbossHome">${jboss.home}</property>
-      <!--<property name="javaVmArguments">-Xmx1536m -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target -Djava.net.preferIPv4Stack=true -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8787</property>-->
-      <property name="javaVmArguments">-Xmx1536m -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=target -Djava.net.preferIPv4Stack=true</property>
+      <property name="javaVmArguments">${server.jvmargs}</property>
       <!-- From JBOSS_HOME/standalone/configuration -->
       <property name="serverConfig">standalone-full.xml</property>
       <!-- NB: don't put whitespace around the number -->