Merge PR #366: Use libgraal as standard compiler

smarr · Aug 6, 2020 · 583bd33 · 583bd33
2 parents 7898240 + c0cb3bf
commit 583bd33
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 37 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ kompos_and_dym_tests:
   stage: full-test
   tags: [benchmarks, infinity]
   script:
-    - ant jacoco-lib compile dynamic-metrics-tests superinstructions-tests
+    - ant jacoco-lib jar dynamic-metrics-tests superinstructions-tests
     - (cd tools/kompos && npm install . && npm -s run verify && npm test)
 
 replay_tests:
@@ -55,14 +55,6 @@ benchmark_savina_job:
     - ant compile
     - rebench --experiment="CI Benchmark Run Pipeline ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c codespeed.conf SOMns-Savina
 
-benchmark_savina_csp_job:
-  stage: benchmark
-  tags: [benchmarks, infinity]
-  allow_failure: true
-  script:
-    - ant compile
-    - rebench --experiment="CI Benchmark Run Pipeline ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c codespeed.conf SOMns-SavinaCSP
-
 benchmark_job:
   stage: benchmark
   tags: [benchmarks, infinity]
@@ -71,12 +63,13 @@ benchmark_job:
     - ant compile
     - rebench --experiment="CI Benchmark Run Pipeline ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c codespeed.conf SOMns
 
-benchmark_interp_job:
+benchmark_csp_and_interp_job:
   stage: benchmark
   tags: [benchmarks, infinity]
   allow_failure: true
   script:
     - ant compile
+    - rebench --experiment="CI Benchmark Run Pipeline ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c codespeed.conf SOMns-SavinaCSP
     - rebench --experiment="CI Benchmark Run Pipeline ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c codespeed.conf SOMns-interp
 
 benchmark_nightly_job:

diff --git a/.travis.yml b/.travis.yml
@@ -38,17 +38,17 @@ jobs:
     - name: Kompos Tests
       script:
         - run nvm install 8
-        - $ANT
+        - $ANT jar kompos
         - cd tools/kompos
         - run npm -s run verify
         - run npm test
     - name: Replay Tests 1
       script:
-        - $ANT compile
+        - $ANT jar
         - run ./tests/replay/test.sh 1
     - name: Replay Tests 2
       script:
-        - $ANT compile
+        - $ANT jar
         - run ./tests/replay/test.sh 2
 
     # Disabled due to breaking changes in the tracing infrastructure. Not fixed to make merge of Snapshotting PR(#293) easier.
@@ -65,16 +65,16 @@ jobs:
       dist: trusty
       jdk:  openjdk11
       script:
-        - $ANT compile
-        - run ./som core-lib/TestSuite/TestRunner.ns
+        - $ANT jar
+        - run ./som --no-libgraal core-lib/TestSuite/TestRunner.ns
 
     - name: macOS Unit Tests
       os: osx
       osx_image: xcode9.3
       language: generic
       script:
-        - $ANT compile
-        - run ./som core-lib/TestSuite/TestRunner.ns
+        - $ANT jar
+        - run ./som --no-libgraal core-lib/TestSuite/TestRunner.ns
 
 before_install:
   - |

diff --git a/build.xml b/build.xml
@@ -50,6 +50,7 @@
     <property name="tools.dir"   location="${lib.dir}/truffle/tools" />
     <property name="truffle.dir" location="${lib.dir}/truffle/truffle" />
     <property name="svm.dir"     location="${lib.dir}/truffle/substratevm" />
+    <property name="vm.dir"      location="${lib.dir}/truffle/vm" />
     <property name="compiler.build" location="${graal.dir}/mxbuild/dists/jdk1.8" />
     <property name="sdk.build"   location="${sdk.dir}/mxbuild/dists/jdk1.8" />
     <property name="tools.build" location="${tools.dir}/mxbuild/dists/jdk1.8" />
@@ -142,7 +143,7 @@
 
         <exec executable="${mx.cmd}" dir="${svm.dir}" failonerror="true" if:true="${truffle.and.jvmci.present}">
             <arg value="--dynamicimports"/>
-            <arg value="../truffle,../tools,../compiler,../sdk"/>
+            <arg value="../truffle,../tools,../compiler,../sdk,../vm"/>
             <arg value="fsckprojects"/>
             <env key="JAVA_HOME" value="${jvmci.home}" />
         </exec>
@@ -160,6 +161,7 @@
       </exec>
     </target>
 
+    <!-- libgraal-jdk also implicitly builds truffle-libs -->
     <target name="truffle-libs" unless="skip.truffle" depends="jvmci-libs,truffle-submodule">
         <travis target="truffle-libs" start="Build Truffle and Graal" />
         <exec executable="${mx.cmd}" dir="${svm.dir}" failonerror="true">
@@ -172,7 +174,20 @@
         <travis target="truffle-libs" />
     </target>
 
-    <target name="bd-libs"> <!-- implicit dependency on truffle-libs -->
+    <target name="libgraal-jdk" unless="skip.truffle" depends="jvmci-libs,truffle-submodule">
+        <travis target="libgraal-jdk" start="Build LibGraal-enabled JDK" />
+        <exec executable="${mx.cmd}" dir="${vm.dir}" failonerror="true">
+            <env key="JAVA_HOME" value="${jvmci.home}" />
+            <env key="DYNAMIC_IMPORTS" value="/substratevm,/tools,/truffle,/sdk,/compiler" />
+            <env key="FORCE_BASH_LAUNCHERS" value="true" />
+            <env key="DISABLE_LIBPOLYGLOT" value="true" />
+            <env key="EXCLUDE_COMPONENTS" value="svmag,nju,nic,ni,nil" />
+            <arg line="build"/>
+        </exec>
+        <travis target="libgraal-jdk" />
+    </target>
+
+    <target name="bd-libs"> <!-- implicit dependency on truffle-libs/libgraal-jdk -->
         <travis target="bd-libs" start="Build Black Diamonds" />
 
         <ant dir="${bd.dir}" useNativeBasedir="true" target="libs-junit" inheritAll="false">
@@ -240,7 +255,7 @@
         <travis target="som-libs" />
     </target>
 
-    <target name="libs" depends="truffle-libs,som-libs,bd-libs" unless="skip.libs" />
+    <target name="libs" depends="som-libs,bd-libs" unless="skip.libs" />
 
     <target name="source" description="Download Source Jars for development">
         <get src="${lib.url}/somns-deps-source-${somns-deps.version}.jar"
@@ -368,7 +383,10 @@
         <travis target="compile-som" />
     </target>
 
-    <target name="compile" depends="libs,compile-som,compile-test-extension" description="Compile SOMns and dependencies">
+    <target name="compile-core" depends="libs,compile-som,compile-test-extension" description="Compile libs and core">
+    </target>
+
+    <target name="compile" depends="libgraal-jdk,compile-core" description="Compile SOMns and dependencies">
     </target>
 
     <target name="compile-test-extension">
@@ -389,11 +407,11 @@
     <target name="compile-all" depends="compile,kompos" description="Build SOMns and Kompos">
     </target>
 
-    <target name="jar" depends="compile" description="Package as JAR">
+    <target name="jar" depends="truffle-libs,compile-core" description="Package as JAR">
         <jar destfile="${build.dir}/som.jar" basedir="${classes.dir}"></jar>
     </target>
 
-    <target name="unit-tests" depends="compile,jacoco-lib" description="Execute tests">
+    <target name="unit-tests" depends="truffle-libs,compile-core,jacoco-lib" description="Execute tests">
       <travis target="unit-tests" start="Execute Tests" />
       <jacoco:coverage>
         <junit haltonerror="false" haltonfailure="false" failureproperty="test.failed"
@@ -424,7 +442,7 @@
      <travis target="unit-tests" />
     </target>
 
-    <target name="som-tests" depends="compile">
+    <target name="som-tests" depends="truffle-libs,compile-core">
       <travis target="som-tests" start="Run SOMns Tests" />
       <!-- delete old coverage data, this should be the first task generating coverage data -->
       <delete file="all.gcov" />
@@ -445,7 +463,7 @@
       <travis target="som-tests" />
     </target>
 
-    <target name="serialization-tests" depends="compile">
+    <target name="serialization-tests" depends="truffle-libs,compile-core">
       <travis target="serialization-tests" start="Test Snapshot Serialization" />
       <exec executable="./som" failonerror="true">
         <arg value="-G" />
@@ -481,7 +499,7 @@
       </exec>
     </target>
 
-    <target name="replay-tests" depends="compile">
+    <target name="replay-tests" depends="truffle-libs,compile-core">
       <travis target="replay-tests" start="Test Trace Replay" />
       <exec executable="tests/replay/test.sh" failonerror="true">
           <arg value="1" />
@@ -492,7 +510,7 @@
       <travis target="replay-tests" />
     </target>
 
-    <target name="snapshot-tests" depends="compile">
+    <target name="snapshot-tests" depends="truffle-libs,compile-core">
       <travis target="snapshot-tests" start="Tests Snapshotting" />
       <exec executable="tests/snapshot/test.sh" failonerror="true">
       </exec>
@@ -563,7 +581,7 @@
     <target name="tests" depends="core-tests,replay-tests,coverage">
     </target>
 
-    <target name="native" depends="libs,jvmci-libs,compile-som">
+    <target name="native" depends="truffle-libs,libs,jvmci-libs,compile-som">
       <travis target="native" start="Build Native Image" />
 
       <exec executable="${mx.cmd}" dir="${svm.dir}" failonerror="true">

diff --git a/som b/som
@@ -124,6 +124,8 @@ parser.add_argument('-E', '--no-early-compilation', help='start compilation usin
                     dest='early_compilation', action='store_false', default=True)
 parser.add_argument('-EG', '--no-embedded-graal', help='run without the embedded Graal',
                     dest='embedded_graal', action='store_false', default=True)
+parser.add_argument('-LG', '--no-libgraal', help='run without using the embedded libgraal. Settings like JVMCI_BIN and GRAAL_HOME are ignored as long as libgraal is not disabled.',
+                    dest='use_libgraal', action='store_false', default=True)
 parser.add_argument('-X', '--java-interpreter', help='run without Graal, and only the Java interpreter',
                     dest='java_interpreter', action='store_true', default=False)
 parser.add_argument('-T', '--no-trace', help='do not print truffle compilation info',
@@ -160,13 +162,17 @@ if args.dynamic_metrics:
 if args.java_interpreter:
     args.interpreter = True
 
+if args.interpreter:
+    args.use_libgraal = False
+
 # Determine JVM to be used
 java_bin = None
 
-if JVMCI_BIN:
-  java_bin = JVMCI_BIN
-if not java_bin and GRAAL_HOME and os.path.isfile(GRAAL_HOME + '/bin/java'):
-  java_bin = GRAAL_HOME + '/bin/java'
+if not args.use_libgraal:
+  if JVMCI_BIN:
+    java_bin = JVMCI_BIN
+  if not java_bin and GRAAL_HOME and os.path.isfile(GRAAL_HOME + '/bin/java'):
+    java_bin = GRAAL_HOME + '/bin/java'
 
 if not java_bin:
     # use local JVMCI, which ant already needed
@@ -198,6 +204,16 @@ if not args.interpreter and not java_bin.startswith(BASE_DIR + '/libs/jvmci') an
   print "No compatible JDK found. Please set the GRAAL_HOME or JVMCI_BIN environment variables."
   sys.exit(1)
 
+if args.use_libgraal:
+  from subprocess import check_output, STDOUT, CalledProcessError
+  try:
+    libgraal_jdk_home = check_output([BASE_DIR + '/libs/mx/mx', '--primary-suite-path', BASE_DIR + '/libs/truffle/vm', '--env', 'libgraal', 'graalvm-home'], stderr=STDOUT, env = {'JAVA_HOME': java_bin.replace('/bin/java', '')})
+    java_bin = libgraal_jdk_home.strip() + '/bin/java'
+  except CalledProcessError as e:
+    print "Failed to determine location of libgraal"
+    print e.output
+    sys.exit(1)
+
 ##
 ## Defining Necessary Parameter Bits
 ##
@@ -216,8 +232,15 @@ BOOT_CLASSPATH = ('-Xbootclasspath/a:'
              + TRUFFLE_DIR + '/sdk/mxbuild/dists/jdk1.8/graal-sdk.jar:'
              + TRUFFLE_DIR + '/truffle/mxbuild/dists/jdk1.8/truffle-api.jar')
 
-GRAAL_JAVA_8_FLAGS = ['-Djvmci.Compiler=graal',
-  '-Djvmci.class.path.append=' + TRUFFLE_DIR + '/compiler/mxbuild/dists/jdk1.8/graal.jar']
+if args.use_libgraal:
+  GRAAL_JAVA_8_FLAGS = ['-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary',
+    '-XX:-UseJVMCIClassLoader', '-Dgraalvm.locatorDisabled=true']
+  GRAAL_JVMCI_FLAGS = []
+else:
+  GRAAL_JAVA_8_FLAGS = ['-Djvmci.Compiler=graal',
+    '-Djvmci.class.path.append=' + TRUFFLE_DIR + '/compiler/mxbuild/dists/jdk1.8/graal.jar']
+  GRAAL_JVMCI_FLAGS = ['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI', '-XX:-UseJVMCICompiler']
+
 
 GRAAL_JAVA_9_FLAGS = [
   '--module-path=' + TRUFFLE_DIR + '/sdk/mxbuild/modules/org.graalvm.graal_sdk.jar:' +
@@ -232,8 +255,6 @@ TWEAK_INLINING = ['-Dgraal.TruffleCompilationThreshold=191',
                   '-Dgraal.TruffleInliningMaxCallerSize=10000',
                   '-Dgraal.TruffleSplittingMaxCalleeSize=100000']
 
-GRAAL_JVMCI_FLAGS = ['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI', '-XX:-UseJVMCICompiler']
-
 JAVA_ARGS = ['-server', '-XX:+UseThreadPriorities']
 
 if JAVA_MAJOR_VERSION == 8:

diff --git a/tests/superinstructions/test.sh b/tests/superinstructions/test.sh
@@ -23,7 +23,7 @@ NEEDS_UPDATE=false
 
 function runBenchmark {
   BENCH=$1
-  HARNESS="${SOM_DIR}/som -si -Dsi.output=${SCRIPT_PATH}/results/${BENCH}.txt \
+  HARNESS="${SOM_DIR}/som --no-libgraal -si -Dsi.output=${SCRIPT_PATH}/results/${BENCH}.txt \
     -G $SOM_DIR/core-lib/Benchmarks/Harness.ns"
   echo $HARNESS $@
   $HARNESS $@