From d072fd45c6433c9ae66f6c120fc7648a592c4f1a Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 21:42:41 -0700 Subject: [PATCH 01/15] Attempt to remove ant for CI --- .travis.yml | 14 +++++++------- appveyor.yml | 16 +++------------- appveyor/install.ps1 | 2 +- appveyor/install.sh | 3 +-- project/ant/bin/ant.py | 19 +++++++++++++++++++ project/ant/pom.xml | 40 ++++++++++++++++++++++++++++++++++++++++ setupext/build_java.py | 3 +++ 7 files changed, 74 insertions(+), 23 deletions(-) create mode 100644 project/ant/bin/ant.py create mode 100644 project/ant/pom.xml diff --git a/.travis.yml b/.travis.yml index 4ec0e1ed9..be2c95280 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,13 +85,13 @@ matrix: - PYTHON=python3 - INST=--user - PIP=pip3 - addons: - homebrew: - packages: - - ant - # Remove the following after bug is resolved - # https://travis-ci.community/t/macos-build-fails-because-of-homebrew-bundle-unknown-command/7296/17 - update: true + install: + - echo $PYTHON $INST + - mvn -f project/ant package + - $PYTHON setup.py --ant="project/ant/bin/ant.py" sdist + - $PIP install $INST dist/* $NUMPY + - $PIP install $INST -r test-requirements.txt + - python project/ant/bin/ant.py -f test/build.xml install: diff --git a/appveyor.yml b/appveyor.yml index 561c3d4c3..70323599e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ environment: # /E:ON and /V:ON options are not enabled in the batch script intepreter # See: http://stackoverflow.com/a/13751649/163740 CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" - ANT_HOME: "C:\\ProgramData\\chocolatey\\lib\\ant\\apache-ant-1.10.5" + ANT: "project\\ant\\bin\\ant.py" NUMPY_: "numpy x.x" #JAVA_HOME: "C:\\jdk8" JAVA_HOME: "C:\\Program Files\\Java\\jdk1.8.0" @@ -51,21 +51,11 @@ environment: install: - # Force java to be installed where JAVA_HOME points - # We can install and test multiple versions of java here - # - We will have to move the JAVA_HOME and the build of the harness to the test section. - # - We will need to figure out how to run nose multiple times on the same run so that - # we can get the result from each of the java version runs. - # - #- cinst jdk8 -params 'installdir=C:\\jdk8' %CINST_OPTS% - - # - cinst jdk7 -params 'installdir=C:\\jdk7' - # - cinst jdk9 -version 9.0.4.11 -params 'installdir=C:\\jdk9' - - cinst ant %CINST_OPTS% --ignore-dependencies + - mvn -f project/ant package - refreshenv # Add thinges to path - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%ANT_HOME%\\bin;%PATH%" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" # If cygwin installed run install.sh, else run the install.ps1 - "IF DEFINED CYGWIN (%CYGSH% appveyor/install.sh) ELSE (powershell ./appveyor/install.ps1)" diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 index 31df66633..1c9794884 100644 --- a/appveyor/install.ps1 +++ b/appveyor/install.ps1 @@ -18,6 +18,6 @@ pip.exe install numpy ant.exe -f test\\build.xml # Build the compiled extension and run the project tests -python.exe setup.py --enable-build-jar bdist_wheel +python.exe setup.py --ant="$ANT" --enable-build-jar bdist_wheel dir .\dist Get-ChildItem -File -Path .\dist\*.whl | Foreach {pip install --upgrade $_.fullname} diff --git a/appveyor/install.sh b/appveyor/install.sh index bcd01d1d1..1f04322ab 100755 --- a/appveyor/install.sh +++ b/appveyor/install.sh @@ -52,7 +52,6 @@ rm -r ./pip ./wheel ./setuptools_scm # Check versions echo "==== Check versions" -"$ANT_HOME/bin/ant" -version $PYTHON --version "$JAVA_HOME/bin/java.exe" -version @@ -70,7 +69,7 @@ $PYTHON -c "import struct; print(struct.calcsize('P') * 8)" # Build the test harness echo "==== Build test" -"$ANT_HOME/bin/ant" -f test/build.xml +$PYTHON "$ANT" -f test/build.xml # Install the package echo "==== Build module" diff --git a/project/ant/bin/ant.py b/project/ant/bin/ant.py new file mode 100644 index 000000000..32d4d895d --- /dev/null +++ b/project/ant/bin/ant.py @@ -0,0 +1,19 @@ +import os +import sys +import glob +import subprocess + +if __name__ == "__main__": + sep = ':' + if sys.platform == "cygwin" or sys.platform == "win32": + sep = ';' + + home = os.path.dirname(__file__) + args = [] + args.append('java') + args.append('-cp') + l = sep.join(glob.glob(os.path.join(home,"..","lib","*.jar"))) + args.append(l) + args.append("org.apache.tools.ant.launch.Launcher") + args.extend(sys.argv[1:]) + subprocess.run(args) diff --git a/project/ant/pom.xml b/project/ant/pom.xml new file mode 100644 index 000000000..aca4ed2a2 --- /dev/null +++ b/project/ant/pom.xml @@ -0,0 +1,40 @@ + + 4.0.0 + jpype.reporting + coverage + 1 + + + + org.apache.ant + ant + 1.10.7 + + + org.apache.ant + ant-launcher + 1.10.7 + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + copy-dependencies + package + + copy-dependencies + + + lib + + + + + + + diff --git a/setupext/build_java.py b/setupext/build_java.py index da5a363ad..bdf50a12d 100644 --- a/setupext/build_java.py +++ b/setupext/build_java.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import sys import subprocess import distutils.cmd import distutils.log @@ -42,6 +43,8 @@ def run(self): buildXmlFile = os.path.join("native", "build.xml") command = [self.distribution.ant, '-Dbuild=%s' % buildDir, '-f', buildXmlFile] + if self.distribution.ant.endswith(".py"): + command.insert(0, sys.executable) cmdStr = ' '.join(command) self.announce(" %s" % cmdStr, level=distutils.log.INFO) try: From e4f5888a4762ed888c43c0ff2857dbfaf59058b0 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 21:49:25 -0700 Subject: [PATCH 02/15] Working on appveyor --- appveyor.yml | 1 - appveyor/install.ps1 | 5 ++--- appveyor/install.sh | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 70323599e..1cb844e45 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,6 @@ environment: # /E:ON and /V:ON options are not enabled in the batch script intepreter # See: http://stackoverflow.com/a/13751649/163740 CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" - ANT: "project\\ant\\bin\\ant.py" NUMPY_: "numpy x.x" #JAVA_HOME: "C:\\jdk8" JAVA_HOME: "C:\\Program Files\\Java\\jdk1.8.0" diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 index 1c9794884..9adc7922e 100644 --- a/appveyor/install.ps1 +++ b/appveyor/install.ps1 @@ -1,7 +1,6 @@ $env:Path += ";"+$env:PYTHON # Check that we have the expected version and architecture for Python -ant.exe -version python.exe --version python.exe -c "import struct; print(struct.calcsize('P') * 8)" @@ -15,9 +14,9 @@ pip.exe install numpy #pip.exe install pytest==5.2.0 #pip.exe install -r "test-requirements.txt" # -r dev-requirements.txt -ant.exe -f test\\build.xml +python project\\ant\\bin\\ant.py -f test\\build.xml # Build the compiled extension and run the project tests -python.exe setup.py --ant="$ANT" --enable-build-jar bdist_wheel +python.exe setup.py --ant="project/ant/bin/ant.py" --enable-build-jar bdist_wheel dir .\dist Get-ChildItem -File -Path .\dist\*.whl | Foreach {pip install --upgrade $_.fullname} diff --git a/appveyor/install.sh b/appveyor/install.sh index 1f04322ab..5086cc434 100755 --- a/appveyor/install.sh +++ b/appveyor/install.sh @@ -6,7 +6,6 @@ echo JAVA_HOME=$JAVA_HOME echo ARCH=$ARCH echo PATH=$PATH echo PYTHON=$PYTHON -echo ANT_HOME=$ANT_HOME # If we do not have Java installed we can't proceed if [ ! -d "$JAVA_HOME" ]; then @@ -69,7 +68,7 @@ $PYTHON -c "import struct; print(struct.calcsize('P') * 8)" # Build the test harness echo "==== Build test" -$PYTHON "$ANT" -f test/build.xml +$PYTHON "project/ant/bin/ant.py" -f test/build.xml # Install the package echo "==== Build module" From 51eb4091e592e222257be23025a3921c3e6d62b2 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 21:58:26 -0700 Subject: [PATCH 03/15] Working on osx pattern --- .travis.yml | 34 +++++++++++++++++----------------- appveyor.yml | 15 ++++++++------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index be2c95280..70e32266e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,23 @@ matrix: - python: pypy3 include: - - name: "Test document generation" + - name: "Python on macOS xcode 11.3" + os: osx + osx_image: xcode11.3 + language: java + env: + - PYTHON=python3 + - INST=--user + - PIP=pip3 + install: + - echo $PYTHON $INST + - mvn -f project/ant package + - $PYTHON setup.py --ant="project/ant/bin/ant.py" sdist + - $PIP install $INST dist/* $NUMPY + - $PIP install $INST -r test-requirements.txt + - $PYTHON project/ant/bin/ant.py -f test/build.xml + + - name: "Test document generation" python: 3.7 dist: xenial language: python @@ -77,22 +93,6 @@ matrix: - java -jar project/coverage/org.jacoco.cli-0.8.5-nodeps.jar report jacoco.exec --classfiles build/classes/ --xml coverage_java.xml - bash <(curl -s https://codecov.io/bash) -f coverage.xml -f coverage_py.xml -f coverage_java.xml - - name: "Python on macOS xcode 11.3" - os: osx - osx_image: xcode11.3 - language: java - env: - - PYTHON=python3 - - INST=--user - - PIP=pip3 - install: - - echo $PYTHON $INST - - mvn -f project/ant package - - $PYTHON setup.py --ant="project/ant/bin/ant.py" sdist - - $PIP install $INST dist/* $NUMPY - - $PIP install $INST -r test-requirements.txt - - python project/ant/bin/ant.py -f test/build.xml - install: - echo $PYTHON $INST diff --git a/appveyor.yml b/appveyor.yml index 1cb844e45..75351692f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,13 +24,14 @@ environment: # ARCH: x86_64 # CYGSH: C:\Cygwin64\bin\bash -c - - PYTHON: "C:\\Miniconda3" - ARCH: x86 - CONDA_PY: "3-latest" - - - PYTHON: "C:\\Miniconda3-x64" - CONDA_PY: "3-latest" - ARCH: x86_64 + # Disabling... stuck at 3.7 for a while + #- PYTHON: "C:\\Miniconda3" + #ARCH: x86 + #CONDA_PY: "3-latest" + + #- PYTHON: "C:\\Miniconda3-x64" + #CONDA_PY: "3-latest" + #ARCH: x86_64 - PYTHON: "C:\\Miniconda36-x64" CONDA_PY: "3.6.5" From 91a2cc0e7d2162f78947293ef2e2f0f70ddd71a0 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 22:03:40 -0700 Subject: [PATCH 04/15] Fix typo --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70e32266e..57ad2bb2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ matrix: - python: pypy3 include: - - name: "Python on macOS xcode 11.3" + - name: "Python on macOS xcode 11.3" os: osx osx_image: xcode11.3 language: java @@ -92,7 +92,6 @@ matrix: - gcovr -r . --xml -o coverage.xml -d --exclude-unreachable-branches --exclude-throw-branches - java -jar project/coverage/org.jacoco.cli-0.8.5-nodeps.jar report jacoco.exec --classfiles build/classes/ --xml coverage_java.xml - bash <(curl -s https://codecov.io/bash) -f coverage.xml -f coverage_py.xml -f coverage_java.xml - install: - echo $PYTHON $INST From d3034a75551f8a45145990a65fce624d6ae6da91 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 22:05:17 -0700 Subject: [PATCH 05/15] Fix typo --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57ad2bb2b..970de7e1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ matrix: - python: pypy3 include: - - name: "Python on macOS xcode 11.3" + - name: "Python on macOS xcode 11.3" os: osx osx_image: xcode11.3 language: java @@ -44,7 +44,7 @@ matrix: - $PIP install $INST -r test-requirements.txt - $PYTHON project/ant/bin/ant.py -f test/build.xml - - name: "Test document generation" + - name: "Test document generation" python: 3.7 dist: xenial language: python From 2fce6de3344897c52f67dfb53a92b7d8502ba3a1 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 22:09:40 -0700 Subject: [PATCH 06/15] Fix test_java --- setupext/test_java.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setupext/test_java.py b/setupext/test_java.py index 368568ed5..61e248504 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -27,6 +27,8 @@ def run(self): distutils.log.info("Building Java testbench") buildXmlFile = os.path.join("test", "build.xml") command = [self.distribution.ant, '-f', buildXmlFile] + if self.distribution.ant.endswith(".py"): + command.insert(0, sys.executable) cmdStr = ' '.join(command) self.announce(" %s" % cmdStr, level=distutils.log.INFO) subprocess.check_call(command) From cd50e46e652325adfe61e4c5e2b9969581cfeed1 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 19 Mar 2020 22:13:08 -0700 Subject: [PATCH 07/15] Missing sys --- setupext/test_java.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setupext/test_java.py b/setupext/test_java.py index 61e248504..69f8208d0 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import sys import os import subprocess import distutils.cmd From 43eae033964555de882b807d04c2f0b74465c374 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 07:20:14 -0700 Subject: [PATCH 08/15] Remove ant entirely --- .travis.yml | 14 ++++------- appveyor.yml | 1 - appveyor/install.ps1 | 4 ++-- appveyor/install.sh | 2 +- project/ant/bin/ant.py | 19 --------------- project/ant/pom.xml | 40 ------------------------------- setupext/build_java.py | 53 +++++++++++++++++++++++++++++++----------- setupext/dist.py | 2 -- setupext/sdist.py | 2 +- setupext/test_java.py | 40 ++++++++++++++++++++++++------- 10 files changed, 78 insertions(+), 99 deletions(-) delete mode 100644 project/ant/bin/ant.py delete mode 100644 project/ant/pom.xml diff --git a/.travis.yml b/.travis.yml index 970de7e1b..13eff3bfa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,6 @@ env: - PIP=pip - INST="" -addons: - apt: - packages: - - ant - matrix: allow_failures: - python: nightly @@ -38,11 +33,10 @@ matrix: - PIP=pip3 install: - echo $PYTHON $INST - - mvn -f project/ant package - - $PYTHON setup.py --ant="project/ant/bin/ant.py" sdist + - $PYTHON setup.py sdist - $PIP install $INST dist/* $NUMPY - $PIP install $INST -r test-requirements.txt - - $PYTHON project/ant/bin/ant.py -f test/build.xml + - $PYTHON setup.py test_java - name: "Test document generation" python: 3.7 @@ -86,7 +80,7 @@ matrix: - $PIP install -e . - $PIP install gcovr pytest-cov jedi - mvn -f project/coverage package - - ant -f test/build.xml + - $PYTHON setup.py test_java script: - $PYTHON -m pytest -v test/jpypetest --cov=jpype --cov-report=xml:coverage_py.xml --jar="native/org.jpype.jar" --jacoco --checkjni - gcovr -r . --xml -o coverage.xml -d --exclude-unreachable-branches --exclude-throw-branches @@ -98,7 +92,7 @@ install: - $PYTHON setup.py sdist - $PIP install $INST dist/* $NUMPY - $PIP install $INST -r test-requirements.txt - - ant -f test/build.xml + - $PYTHON setup.py test_java script: - $PYTHON -c "import jpype" diff --git a/appveyor.yml b/appveyor.yml index 75351692f..e21dcef9b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,7 +51,6 @@ environment: install: - - mvn -f project/ant package - refreshenv # Add thinges to path diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 index 9adc7922e..182bc9b16 100644 --- a/appveyor/install.ps1 +++ b/appveyor/install.ps1 @@ -14,9 +14,9 @@ pip.exe install numpy #pip.exe install pytest==5.2.0 #pip.exe install -r "test-requirements.txt" # -r dev-requirements.txt -python project\\ant\\bin\\ant.py -f test\\build.xml +python setup.py test_java # Build the compiled extension and run the project tests -python.exe setup.py --ant="project/ant/bin/ant.py" --enable-build-jar bdist_wheel +python.exe setup.py --enable-build-jar bdist_wheel dir .\dist Get-ChildItem -File -Path .\dist\*.whl | Foreach {pip install --upgrade $_.fullname} diff --git a/appveyor/install.sh b/appveyor/install.sh index 5086cc434..bb4c4e465 100755 --- a/appveyor/install.sh +++ b/appveyor/install.sh @@ -68,7 +68,7 @@ $PYTHON -c "import struct; print(struct.calcsize('P') * 8)" # Build the test harness echo "==== Build test" -$PYTHON "project/ant/bin/ant.py" -f test/build.xml +$PYTHON setup.py test_java # Install the package echo "==== Build module" diff --git a/project/ant/bin/ant.py b/project/ant/bin/ant.py deleted file mode 100644 index 32d4d895d..000000000 --- a/project/ant/bin/ant.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import sys -import glob -import subprocess - -if __name__ == "__main__": - sep = ':' - if sys.platform == "cygwin" or sys.platform == "win32": - sep = ';' - - home = os.path.dirname(__file__) - args = [] - args.append('java') - args.append('-cp') - l = sep.join(glob.glob(os.path.join(home,"..","lib","*.jar"))) - args.append(l) - args.append("org.apache.tools.ant.launch.Launcher") - args.extend(sys.argv[1:]) - subprocess.run(args) diff --git a/project/ant/pom.xml b/project/ant/pom.xml deleted file mode 100644 index aca4ed2a2..000000000 --- a/project/ant/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - 4.0.0 - jpype.reporting - coverage - 1 - - - - org.apache.ant - ant - 1.10.7 - - - org.apache.ant - ant-launcher - 1.10.7 - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.1.1 - - - copy-dependencies - package - - copy-dependencies - - - lib - - - - - - - diff --git a/setupext/build_java.py b/setupext/build_java.py index bdf50a12d..41ff241d3 100644 --- a/setupext/build_java.py +++ b/setupext/build_java.py @@ -6,12 +6,44 @@ import distutils.log from distutils.errors import DistutilsPlatformError from distutils.dir_util import copy_tree +import glob +import re +import shlex + + +def getJavaVersion(): + # Find Java version + out = subprocess.run(['javac', '-version'], capture_output=True) + version_str = out.stdout.decode('utf-8') + result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) + if int(result[1]) > 1: + return int(result[1]) + return int(result[2]) + + +def compileJava(self): + target_version = "1.7" + version = getJavaVersion() + srcs = glob.glob('native/java/**/*.java', recursive=True) + src1 = [ i for i in srcs if "JPypeClassLoader" in i] + src2 = [ i for i in srcs if not "JPypeClassLoader" in i] + cmd1 = shlex.split('javac -d build/lib -g:none -source %s -target %s'%(target_version, target_version)) + cmd1.extend(src1) + cmd2 = shlex.split('javac -d build/classes -g:none -source %s -target %s -cp build/lib'%(target_version, target_version)) + cmd2.extend(src2) + self.announce(" %s" % " ".join(cmd1), level=distutils.log.INFO) + subprocess.check_call(cmd1) + self.announce(" %s" % " ".join(cmd2), level=distutils.log.INFO) + subprocess.check_call(cmd2) + cmd3 = shlex.split('jar --create --file build/lib/org.jpype.jar -C build/classes/ .') + self.announce(" %s" % " ".join(cmd3), level=distutils.log.INFO) + subprocess.check_call(cmd3) class BuildJavaCommand(distutils.cmd.Command): """A custom command to create jar file during build.""" - description = 'run ant to make a jar' + description = 'build jpype jar' user_options = [] def initialize_options(self): @@ -39,22 +71,15 @@ def run(self): "Jar cache is missing, using --enable-build-jar to recreate it.") # build the jar - buildDir = os.path.join("..", "build") - buildXmlFile = os.path.join("native", "build.xml") - command = [self.distribution.ant, '-Dbuild=%s' % - buildDir, '-f', buildXmlFile] - if self.distribution.ant.endswith(".py"): - command.insert(0, sys.executable) - cmdStr = ' '.join(command) - self.announce(" %s" % cmdStr, level=distutils.log.INFO) try: - subprocess.check_call(command) + compileJava(self) except subprocess.CalledProcessError as exc: distutils.log.error(exc.output) raise DistutilsPlatformError("Error executing {}".format(exc.cmd)) + ## Disable for now. Java coverage tool needs work # Coverage tool requires special placement of the source - if self.distribution.enable_coverage: - import shutil - shutil.copyfile(os.path.join("build", "lib", "org.jpype.jar"), os.path.join( - "native", "org.jpype.jar")) + #if self.distribution.enable_coverage: + # import shutil + # shutil.copyfile(os.path.join("build", "lib", "org.jpype.jar"), os.path.join( + # "native", "org.jpype.jar")) diff --git a/setupext/dist.py b/setupext/dist.py index 91181f0e7..7e94d4d77 100644 --- a/setupext/dist.py +++ b/setupext/dist.py @@ -8,13 +8,11 @@ class Distribution(_Distribution): global_options = [ ('enable-build-jar', None, 'Build the java jar portion'), ('enable-tracing', None, 'Set for tracing for debugging'), - ('ant=', None, 'Set the ant executable (default ant)', 1), ('enable-coverage', None, 'Instrument c++ code for code coverage measuring'), ] + _Distribution.global_options def parse_command_line(self): - self.ant = "ant" self.enable_tracing = False self.enable_build_jar = False self.enable_coverage = False diff --git a/setupext/sdist.py b/setupext/sdist.py index adb7c9c85..abf53eb38 100644 --- a/setupext/sdist.py +++ b/setupext/sdist.py @@ -11,7 +11,7 @@ class BuildSourceDistribution(sdist): """ Override some behavior on sdist - Copy the build/lib to native/jars to remove ant/jdk dependency + Copy the build/lib to native/jars to remove javac/jdk dependency """ def run(self): diff --git a/setupext/test_java.py b/setupext/test_java.py index 69f8208d0..5b5416aed 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -4,12 +4,39 @@ import subprocess import distutils.cmd import distutils.log +import glob +import re +import shlex + +def getJavaVersion(): + # Find Java version + out = subprocess.run(['javac', '-version'], capture_output=True) + version_str = out.stdout.decode('utf-8') + result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) + if int(result[1]) > 1: + return int(result[1]) + return int(result[2]) + + +def compileJava(): + version = getJavaVersion() + srcs = glob.glob('test/harness/jpype/**/*.java', recursive=True) + exports = "" + if version > 7: + srcs.extend(glob.glob('test/harness/java8/**/*.java', recursive=True)) + if version > 8: + srcs.extend(glob.glob('test/harness/java9/**/*.java', recursive=True)) + exports = "--add-exports java.base/jdk.internal.reflect=ALL-UNNAMED" + cmd = shlex.split( + 'javac -d test/classes %s -g:lines,vars,source' % (exports)) + cmd.extend(srcs) + return cmd class TestJavaCommand(distutils.cmd.Command): """A custom command to create jar file during test.""" - description = 'run ant to make test harness' + description = 'run javac to make test harness' user_options = [] def initialize_options(self): @@ -25,11 +52,6 @@ def run(self): if os.path.exists(os.path.join("test", "classes")): distutils.log.info("Skip building Java testbench") return - distutils.log.info("Building Java testbench") - buildXmlFile = os.path.join("test", "build.xml") - command = [self.distribution.ant, '-f', buildXmlFile] - if self.distribution.ant.endswith(".py"): - command.insert(0, sys.executable) - cmdStr = ' '.join(command) - self.announce(" %s" % cmdStr, level=distutils.log.INFO) - subprocess.check_call(command) + cmdStr = compileJava() + self.announce(" %s" % " ".join(cmdStr), level=distutils.log.INFO) + subprocess.check_call(cmdStr) From 2df2d37ccf20d4d0eb700d0727f103c8a7fe1199 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 09:38:52 -0700 Subject: [PATCH 09/15] Debugging bootstrap failure --- native/common/jp_exception.cpp | 10 +++++++++- setupext/build_java.py | 28 ++++++++++------------------ setupext/test_java.py | 3 ++- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/native/common/jp_exception.cpp b/native/common/jp_exception.cpp index 61898a613..2fbc06d2e 100644 --- a/native/common/jp_exception.cpp +++ b/native/common/jp_exception.cpp @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004-2008 Steve Ménard + Copyright 2004-2008 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -150,6 +150,14 @@ void JPypeException::convertJavaToPython() jthrowable th = m_Throwable.get(); jvalue v; v.l = th; + // GCOVR_EXCL_START + // This is code is only hit if something fails during the initial boot + if (m_Context->getJavaContext() == NULL || m_Context->m_Context_GetExcClassID == NULL) + { + PyErr_SetString(PyExc_SystemError, frame.toString(th).c_str()); + return; + } + // GCOVR_EXCL_STOP jlong pycls = frame.CallLongMethodA(m_Context->getJavaContext(), m_Context->m_Context_GetExcClassID, &v); if (pycls != 0) { diff --git a/setupext/build_java.py b/setupext/build_java.py index 41ff241d3..b379af4ef 100644 --- a/setupext/build_java.py +++ b/setupext/build_java.py @@ -11,31 +11,23 @@ import shlex -def getJavaVersion(): - # Find Java version - out = subprocess.run(['javac', '-version'], capture_output=True) - version_str = out.stdout.decode('utf-8') - result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) - if int(result[1]) > 1: - return int(result[1]) - return int(result[2]) - - def compileJava(self): target_version = "1.7" - version = getJavaVersion() srcs = glob.glob('native/java/**/*.java', recursive=True) - src1 = [ i for i in srcs if "JPypeClassLoader" in i] - src2 = [ i for i in srcs if not "JPypeClassLoader" in i] - cmd1 = shlex.split('javac -d build/lib -g:none -source %s -target %s'%(target_version, target_version)) + src1 = [i for i in srcs if "JPypeClassLoader" in i] + src2 = [i for i in srcs if not "JPypeClassLoader" in i] + cmd1 = shlex.split('javac -d build/lib -g:none -source %s -target %s' % + (target_version, target_version)) cmd1.extend(src1) - cmd2 = shlex.split('javac -d build/classes -g:none -source %s -target %s -cp build/lib'%(target_version, target_version)) + cmd2 = shlex.split('javac -d build/classes -g:none -source %s -target %s -cp build/lib' % + (target_version, target_version)) cmd2.extend(src2) self.announce(" %s" % " ".join(cmd1), level=distutils.log.INFO) subprocess.check_call(cmd1) self.announce(" %s" % " ".join(cmd2), level=distutils.log.INFO) subprocess.check_call(cmd2) - cmd3 = shlex.split('jar --create --file build/lib/org.jpype.jar -C build/classes/ .') + cmd3 = shlex.split( + 'jar --create --file build/lib/org.jpype.jar -C build/classes/ .') self.announce(" %s" % " ".join(cmd3), level=distutils.log.INFO) subprocess.check_call(cmd3) @@ -77,9 +69,9 @@ def run(self): distutils.log.error(exc.output) raise DistutilsPlatformError("Error executing {}".format(exc.cmd)) - ## Disable for now. Java coverage tool needs work + # Disable for now. Java coverage tool needs work # Coverage tool requires special placement of the source - #if self.distribution.enable_coverage: + # if self.distribution.enable_coverage: # import shutil # shutil.copyfile(os.path.join("build", "lib", "org.jpype.jar"), os.path.join( # "native", "org.jpype.jar")) diff --git a/setupext/test_java.py b/setupext/test_java.py index 5b5416aed..44313fe18 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -8,9 +8,10 @@ import re import shlex + def getJavaVersion(): # Find Java version - out = subprocess.run(['javac', '-version'], capture_output=True) + out = subprocess.run(['javac', '-version'], stdout=subprocess.PIPE) version_str = out.stdout.decode('utf-8') result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) if int(result[1]) > 1: From 0b56b41d32210d594b40177bb5e1572c9c5b4bfc Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 10:48:54 -0700 Subject: [PATCH 10/15] Fix problem in thunk loader (jar without entry sizes) --- native/common/jp_exception.cpp | 2 +- .../jpype/classloader/JPypeClassLoader.java | 21 +++++++------------ setupext/build_makefile.py | 4 ++++ setupext/build_thunk.py | 1 + 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/native/common/jp_exception.cpp b/native/common/jp_exception.cpp index 2fbc06d2e..238e9342c 100644 --- a/native/common/jp_exception.cpp +++ b/native/common/jp_exception.cpp @@ -151,7 +151,7 @@ void JPypeException::convertJavaToPython() jvalue v; v.l = th; // GCOVR_EXCL_START - // This is code is only hit if something fails during the initial boot + // This is condition is only hit if something fails during the initial boot if (m_Context->getJavaContext() == NULL || m_Context->m_Context_GetExcClassID == NULL) { PyErr_SetString(PyExc_SystemError, frame.toString(th).c_str()); diff --git a/native/java/org/jpype/classloader/JPypeClassLoader.java b/native/java/org/jpype/classloader/JPypeClassLoader.java index 748dd02bf..758d3bb4a 100644 --- a/native/java/org/jpype/classloader/JPypeClassLoader.java +++ b/native/java/org/jpype/classloader/JPypeClassLoader.java @@ -17,6 +17,7 @@ package org.jpype.classloader; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.TreeMap; import java.util.jar.JarEntry; @@ -78,7 +79,7 @@ public void importClass(String name, byte[] code) */ public void importJar(byte[] bytes) { - try (JarInputStream is = new JarInputStream(new ByteArrayInputStream(bytes))) + try ( JarInputStream is = new JarInputStream(new ByteArrayInputStream(bytes))) { while (true) { @@ -88,20 +89,14 @@ public void importJar(byte[] bytes) // Skip directories and other non-class resources long size = nextEntry.getSize(); - if (size <= 0) + if (size == 0) continue; - byte[] data = new byte[(int) size]; - int total = 0; - - // Read the contents. - while (true) - { - int r = is.read(data, total, data.length - total); - total += r; - if (r == 0 || total == size) - break; - } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int q; + while ((q = is.read()) != -1) + baos.write(q); + byte[] data = baos.toByteArray(); // Store all classes we find String name = nextEntry.getName(); diff --git a/setupext/build_makefile.py b/setupext/build_makefile.py index 1de60eb49..a7cfba322 100644 --- a/setupext/build_makefile.py +++ b/setupext/build_makefile.py @@ -91,6 +91,10 @@ def write(self): print(""" all: $(LIB) +rwildcard=$(foreach d,$(wildcard $(1:=/*)),$(call rwildcard,$d,$2) $(filter $(subst *,%,$2),$d)) +build/src/jp_thunk.cpp: $(call rwildcard,native/java,*.java) + python setup.py build_thunk + DEPDIR = build/deps $(DEPDIR): ; @mkdir -p $@ diff --git a/setupext/build_thunk.py b/setupext/build_thunk.py index e68f26694..0882c859f 100644 --- a/setupext/build_thunk.py +++ b/setupext/build_thunk.py @@ -124,6 +124,7 @@ def finalize_options(self): def run(self): """Run command.""" + self.run_command("build_java") self.announce( 'Building thunks', level=distutils.log.INFO) From e21cb59194fb29d7728ed3710b0da8d8ea980c56 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 10:57:19 -0700 Subject: [PATCH 11/15] Fix for 3.5 re --- setupext/test_java.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setupext/test_java.py b/setupext/test_java.py index 44313fe18..9dcddc40e 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -14,9 +14,9 @@ def getJavaVersion(): out = subprocess.run(['javac', '-version'], stdout=subprocess.PIPE) version_str = out.stdout.decode('utf-8') result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) - if int(result[1]) > 1: - return int(result[1]) - return int(result[2]) + if int(result.group(1)) > 1: + return int(result.group(1)) + return int(result.group(2)) def compileJava(): From 24f5c2382b3233be34f6daa0ab6e3ef48465e5d3 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 11:13:29 -0700 Subject: [PATCH 12/15] Fix for windows --- setupext/build_java.py | 2 ++ setupext/test_java.py | 1 + 2 files changed, 3 insertions(+) diff --git a/setupext/build_java.py b/setupext/build_java.py index b379af4ef..c3eb3b20b 100644 --- a/setupext/build_java.py +++ b/setupext/build_java.py @@ -22,6 +22,8 @@ def compileJava(self): cmd2 = shlex.split('javac -d build/classes -g:none -source %s -target %s -cp build/lib' % (target_version, target_version)) cmd2.extend(src2) + os.makedirs("build/lib", exist_ok=True) + os.makedirs("build/classes", exist_ok=True) self.announce(" %s" % " ".join(cmd1), level=distutils.log.INFO) subprocess.check_call(cmd1) self.announce(" %s" % " ".join(cmd2), level=distutils.log.INFO) diff --git a/setupext/test_java.py b/setupext/test_java.py index 9dcddc40e..22f31e003 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -30,6 +30,7 @@ def compileJava(): exports = "--add-exports java.base/jdk.internal.reflect=ALL-UNNAMED" cmd = shlex.split( 'javac -d test/classes %s -g:lines,vars,source' % (exports)) + os.makedirs("test/classes", exist_ok=True) cmd.extend(srcs) return cmd From 567b5c908aae1c3e973874124886bdbc79c9dd3e Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 11:15:33 -0700 Subject: [PATCH 13/15] Fix for old java --- setupext/build_java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setupext/build_java.py b/setupext/build_java.py index c3eb3b20b..0076db2f5 100644 --- a/setupext/build_java.py +++ b/setupext/build_java.py @@ -29,7 +29,7 @@ def compileJava(self): self.announce(" %s" % " ".join(cmd2), level=distutils.log.INFO) subprocess.check_call(cmd2) cmd3 = shlex.split( - 'jar --create --file build/lib/org.jpype.jar -C build/classes/ .') + 'jar cvf build/lib/org.jpype.jar -C build/classes/ .') self.announce(" %s" % " ".join(cmd3), level=distutils.log.INFO) subprocess.check_call(cmd3) From 74b09aad6fa9251a5c6298e9a584d94225c99aa5 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 11:25:24 -0700 Subject: [PATCH 14/15] Stone clubs and wooden knives time. --- setupext/test_java.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setupext/test_java.py b/setupext/test_java.py index 22f31e003..f6a4a2b05 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -11,8 +11,7 @@ def getJavaVersion(): # Find Java version - out = subprocess.run(['javac', '-version'], stdout=subprocess.PIPE) - version_str = out.stdout.decode('utf-8') + version_str = os.popen('javac -version').read() result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) if int(result.group(1)) > 1: return int(result.group(1)) From e2e6953c733a64e4380d11b26514cefebeded217 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sun, 22 Mar 2020 11:32:01 -0700 Subject: [PATCH 15/15] Argggg! --- setupext/test_java.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setupext/test_java.py b/setupext/test_java.py index f6a4a2b05..455f48b27 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -13,6 +13,8 @@ def getJavaVersion(): # Find Java version version_str = os.popen('javac -version').read() result = re.match(r'javac ([0-9]+)\.([0-9]+)\..*', version_str) + if not result: + return 8 if int(result.group(1)) > 1: return int(result.group(1)) return int(result.group(2))