diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..be4a007f107 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,105 @@ +name: CI jobs +on: [push, pull_request] + +jobs: + linux-x86_64: + runs-on: ubuntu-latest + container: nvidia/cuda:10.1-cudnn7-devel-centos7 + strategy: + matrix: + ext: ["", -mkl, -gpu, -mkl-gpu] + steps: + - name: Install environment + run: | + yum -y update + yum -y install centos-release-scl-rh epel-release + yum -y install java-1.8.0-openjdk-devel devtoolset-7 rh-git218 rh-maven35 patch python36-devel python36-pip python36-six + echo Downloading Bazel + curl -L https://github.com/bazelbuild/bazel/releases/download/0.29.1/bazel-0.29.1-installer-linux-x86_64.sh -o bazel.sh --retry 10 + bash bazel.sh + - name: Checkout repository + uses: actions/checkout@v1 + - name: Build project + run: | + source scl_source enable devtoolset-7 rh-git218 rh-maven35 || true + git --version + gcc --version + mvn -version + bazel version + df -h + echo Executing Maven without Wagon HTTP pool that fails under Docker + mvn clean install -B -U -e -Dmaven.wagon.http.pool=false -Djavacpp.platform=linux-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} + df -h + macosx-x86_64: + runs-on: macos-latest + strategy: + matrix: + ext: ["", -mkl] + steps: + - name: Install environment + run: | + python3 -m pip install six + echo Downloading Bazel + curl -L https://github.com/bazelbuild/bazel/releases/download/0.29.1/bazel-0.29.1-installer-darwin-x86_64.sh -o bazel.sh --retry 10 + bash bazel.sh + - name: Checkout repository + uses: actions/checkout@v1 + - name: Build project + run: | + git --version + clang --version + mvn -version + bazel version + df -h + echo Executing Maven + mvn clean install -B -U -e -Djavacpp.platform=macosx-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} + df -h + windows-x86_64: + runs-on: windows-latest + strategy: + matrix: + ext: ["", -mkl, -gpu, -mkl-gpu] + steps: + - name: Install environment + shell: cmd + run: | + python -m pip install six + echo Removing some unused stuff to avoid running out of disk space + rm.exe -Rf "C:/Program Files (x86)/Android" "C:/Program Files/dotnet" "%CONDA%" "%GOROOT_1_10_X64%" "%GOROOT_1_11_X64%" "%GOROOT_1_12_X64%" "%GOROOT_1_13_X64%" "C:\hostedtoolcache\windows\Ruby" "C:\Rust" + echo Removing old versions of MSVC that interfere with Bazel + bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;" + echo Downloading Bazel + mkdir C:\bazel + curl.exe -L https://github.com/bazelbuild/bazel/releases/download/0.29.1/bazel-0.29.1-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10 + echo Installing CUDA + curl.exe -L http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_windows.exe -o cuda.exe + curl.exe -L https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.4/cudnn-10.1-windows7-x64-v7.6.4.38.zip -o cudnn.zip + cuda.exe -s + mkdir cuda + unzip.exe cudnn.zip + cp.exe -a cuda/include cuda/lib cuda/bin "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/" + - name: Checkout repository + uses: actions/checkout@v1 + - name: Build project + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 + set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v10.1" + set "CUDA_PATH_V10_1=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v10.1" + set "PATH=C:\bazel;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v10.1\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v10.1\libnvvp;%PATH%" + echo Shorten work paths to prevent Bazel from reaching MAX_PATH limit + set "TEST_TMPDIR=C:\tmp" + set "TMPDIR=C:\tmp" + set "TEMP=C:\tmp" + set "TMP=C:\tmp" + mkdir C:\tmp + git --version + cl + call mvn -version + bazel version + df -h + wmic pagefile list /format:list + echo Executing Maven + call mvn clean install -B -U -e -Djavacpp.platform=windows-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} + df -h + wmic pagefile list /format:list diff --git a/README.md b/README.md index da4232e9bab..8e49e8acef7 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,9 @@ artifact includes transitively all the artifacts above as a single dependency: Be aware though that the native library is quite large and including too many versions of it may significantly increase the size of your JAR. So it is good practice to limit your dependencies to -the platforms you are targeting. +the platforms you are targeting. For this purpose the `-platform` artifacts include profiles that follow +the conventions established on this page: +* [Reducing the Number of Dependencies](https://github.com/bytedeco/javacpp-presets/wiki/Reducing-the-Number-of-Dependencies) *Note: the `tensorflow-starters` artifact is not available at this moment* diff --git a/tensorflow-core/tensorflow-core-api/WORKSPACE b/tensorflow-core/tensorflow-core-api/WORKSPACE index 945b3e54943..8dfcdf6d57c 100644 --- a/tensorflow-core/tensorflow-core-api/WORKSPACE +++ b/tensorflow-core/tensorflow-core-api/WORKSPACE @@ -15,11 +15,11 @@ http_archive( ], patch_args = ["-p1"], urls = [ - "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/v2.0.0.tar.gz", - "https://github.com/tensorflow/tensorflow/archive/v2.0.0.tar.gz", + "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/v2.1.0-rc2.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/v2.1.0-rc2.tar.gz", ], - sha256 = "49b5f0495cd681cbcb5296a4476853d4aea19a43bdd9f179c928a977308a0617", - strip_prefix = "tensorflow-2.0.0" + sha256 = "8f0c227024fe2dffc691e71f4498c217f604d045079db254a300720db1eb4693", + strip_prefix = "tensorflow-2.1.0-rc2" ) # START: Upstream TensorFlow dependencies diff --git a/tensorflow-core/tensorflow-core-api/build.sh b/tensorflow-core/tensorflow-core-api/build.sh index 058715af6c0..5238676d46c 100755 --- a/tensorflow-core/tensorflow-core-api/build.sh +++ b/tensorflow-core/tensorflow-core-api/build.sh @@ -7,13 +7,13 @@ export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 export BAZEL_VC="${VCINSTALLDIR:-}" if [[ -d $BAZEL_VC ]]; then - # Work around compiler issues on Windows documented mainly in configure.py + # Work around compiler issues on Windows documented mainly in configure.py but also elsewhere export BUILD_FLAGS="--copt=//arch:AVX `#--copt=//arch:AVX2` --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN --copt=-DNOGDI --host_copt=-DNOGDI --define=override_eigen_strong_inline=true" # https://software.intel.com/en-us/articles/intel-optimization-for-tensorflow-installation-guide#wind_B_S export PATH=$PATH:$(pwd)/bazel-tensorflow-core-api/external/mkl_windows/lib/ export PYTHON_BIN_PATH=$(which python.exe) else - export BUILD_FLAGS="--copt=-msse4.1 --copt=-msse4.2 --copt=-mavx `#--copt=-mavx2 --copt=-mfma`" + export BUILD_FLAGS="--copt=-msse4.1 --copt=-msse4.2 --copt=-mavx `#--copt=-mavx2 --copt=-mfma` --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 --linkopt=-lstdc++ --host_linkopt=-lstdc++" export PYTHON_BIN_PATH=$(which python3) fi @@ -23,10 +23,15 @@ fi if [[ "${EXTENSION:-}" == *gpu* ]]; then export BUILD_FLAGS="$BUILD_FLAGS --config=cuda" + if [[ -z ${TF_CUDA_PATHS:-} ]] && [[ -d ${CUDA_PATH:-} ]]; then + # Work around some issue with Bazel preventing it from detecting CUDA on Windows + export TF_CUDA_PATHS="$CUDA_PATH" + fi fi # Build C API of TensorFlow itself including a target to generate ops for Java bazel build $BUILD_FLAGS --python_path="$PYTHON_BIN_PATH" --config=monolithic --output_filter=DONT_MATCH_ANYTHING --verbose_failures @org_tensorflow//tensorflow:tensorflow :java_op_gen_sources +ls -l bazel-bin/external/org_tensorflow/tensorflow/ # Normalize some paths with symbolic links TENSORFLOW_SO=(bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.so.?.?.?) @@ -39,10 +44,12 @@ if [[ -f $TENSORFLOW_DYLIB ]]; then ln -sf $(basename $TENSORFLOW_DYLIB) bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.dylib ln -sf $(basename $TENSORFLOW_DYLIB) bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.2.dylib fi -TENSORFLOW_LIB=(bazel-bin/external/org_tensorflow/tensorflow/tensorflow.dll.if.lib) -if [[ -f $TENSORFLOW_LIB ]]; then - ln -sf $(basename $TENSORFLOW_LIB) bazel-bin/external/org_tensorflow/tensorflow/tensorflow.lib -fi +TENSORFLOW_LIBS=(bazel-bin/external/org_tensorflow/tensorflow/tensorflow.dll.if.lib bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.dll.ifso) +for TENSORFLOW_LIB in ${TENSORFLOW_LIBS[@]}; do + if [[ -f $TENSORFLOW_LIB ]]; then + ln -sf $(basename $TENSORFLOW_LIB) bazel-bin/external/org_tensorflow/tensorflow/tensorflow.lib + fi +done # Copy only main generated Java source files for ops mkdir -p src/gen/java/ diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java index d1a58f84b66..143513ac188 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java @@ -140,7 +140,7 @@ public class tensorflow implements LoadEnabled, InfoMapper { case "linux-ppc64le": case "linux-x86_64": case "macosx-x86_64": - lib += lib.equals("cudnn") ? "@.7" : lib.equals("nccl") ? "@.2" : lib.equals("nvinfer") ? "@.5" : lib.equals("cudart") ? "@.10.1" : "@.10"; + lib += lib.equals("cudnn") ? "@.7" : lib.equals("nccl") ? "@.2" : lib.equals("nvinfer") ? "@.6" : lib.equals("cudart") ? "@.10.1" : "@.10"; break; case "windows-x86_64": lib += lib.equals("cudnn") ? "64_7" : lib.equals("cudart") ? "64_101" : "64_10";