Scripts for building dependency libraries of MXNet (apache#13282)

* openblas script * ps-lite dependencies * USE_S3 dependencies * image libraries * license
lanking520 · Apr 26, 2019 · 60f7737 · 60f7737
1 parent 1aee3d7
commit 60f7737
Show file tree

Hide file tree

Showing 14 changed files with 673 additions and 0 deletions.
diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md
@@ -0,0 +1,14 @@
+# Overview
+
+This folder contains scripts for building the dependencies from source. The static libraries from
+the build artifacts can be used to create self-contained shared object for mxnet through static
+linking.
+
+# Settings
+
+The scripts use the following environment variables for setting behavior:
+
+`DEPS_PATH`: the location in which the libraries are downloaded, built, and installed.
+`PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'.
+
+It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm
diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of cityhash that can be used as dependency of mxnet.
+CITYHASH_VERSION=1.1.1
+if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
+    # Download and build cityhash
+    >&2 echo "Building cityhash..."
+    git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION
+    cd $DEPS_PATH/cityhash-$CITYHASH_VERSION
+    git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db
+    ./configure -prefix=$DEPS_PATH --enable-sse4.2
+    make CXXFLAGS="-g -O3 -msse4.2"
+    make install
+    cd -
+fi
diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of libcurl that can be used as dependency of mxnet.
+LIBCURL_VERSION=7.61.0
+if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
+    # download and build libcurl
+    >&2 echo "Building libcurl..."
+    curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip
+    unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH
+    cd $DEPS_PATH/curl-$LIBCURL_VERSION
+    if [[ $PLATFORM == 'linux' ]]; then
+        CONFIG_FLAG=""
+    elif [[ $PLATFORM == 'darwin' ]]; then
+        CONFIG_FLAG="--with-darwinssl"
+    fi
+    ./configure $CONFIG_FLAG \
+                --with-zlib \
+                --with-nghttps2 \
+                --without-zsh-functions-dir \
+                --without-librtmp \
+                --without-libssh2 \
+                --disable-debug \
+                --disable-curldebug \
+                --enable-symbol-hiding=yes \
+                --enable-optimize=yes \
+                --enable-shared=no \
+                --enable-http=yes \
+                --enable-ipv6=yes \
+                --disable-ftp \
+                --disable-ldap \
+                --disable-ldaps \
+                --disable-rtsp \
+                --disable-proxy \
+                --disable-dict \
+                --disable-telnet \
+                --disable-tftp \
+                --disable-pop3 \
+                --disable-imap \
+                --disable-smb \
+                --disable-smtp \
+                --disable-gopher \
+                --disable-manual \
+                --prefix=$DEPS_PATH
+    make
+    make install
+    cd -
+fi
diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script imports the headers from eigen3 that can be used to in opencv.
+EIGEN_VERSION=3.3.4
+if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
+    # download eigen
+    >&2 echo "Loading eigen..."
+    curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip
+    unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH
+    mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
+    cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
+    cmake \
+          -D CMAKE_BUILD_TYPE=RELEASE \
+          -D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
+    make install
+    cd -
+fi
diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of libpng that can be used as dependency of mxnet/opencv.
+PNG_VERSION=1.6.34
+if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
+    # download and build libpng
+    >&2 echo "Building libpng..."
+    curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip
+    unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH
+    mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build
+    cd $DEPS_PATH/libpng-$PNG_VERSION/build
+    cmake \
+          -D PNG_SHARED=OFF \
+          -D PNG_STATIC=ON \
+          -D CMAKE_BUILD_TYPE=RELEASE \
+          -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
+          -D CMAKE_C_FLAGS=-fPIC ..
+    make
+    make install
+    mkdir -p $DEPS_PATH/include/libpng
+    ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h
+    cd -
+fi
diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of libtiff that can be used as dependency of mxnet/opencv.
+TIFF_VERSION="4-0-9"
+if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then
+    # download and build libtiff
+    >&2 echo "Building libtiff..."
+    curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip
+    unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH
+    cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION
+    ./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH
+    make
+    make install
+    cd -
+fi
diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of libturbojpeg that can be used as dependency of
+# mxnet/opencv.
+TURBO_JPEG_VERSION=1.5.90
+if [[ $PLATFORM == 'darwin' ]]; then
+    JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
+fi
+
+if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
+    # download and build libjpeg
+    >&2 echo "Building libjpeg-turbo..."
+    curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip
+    unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
+    mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
+    cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
+    cmake \
+          -G"Unix Makefiles" \
+          -D CMAKE_BUILD_TYPE=RELEASE \
+          -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
+          -D CMAKE_C_FLAGS=-fPIC \
+          -D WITH_JAVA=FALSE \
+          -D WITH_JPEG7=TRUE \
+          -D WITH_JPEG8=TRUE \
+          $JPEG_NASM_OPTION \
+          -D ENABLE_SHARED=FALSE ..
+    make
+    make install
+    cd -
+fi
diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of libz that can be used as dependency of mxnet.
+ZLIB_VERSION=1.2.6
+if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
+    # Download and build zlib
+    >&2 echo "Building zlib..."
+    curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip
+    unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH
+    mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build
+    cd $DEPS_PATH/zlib-$ZLIB_VERSION/build
+    cmake \
+          -D CMAKE_BUILD_TYPE=RELEASE \
+          -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
+          -D BUILD_SHARED_LIBS=OFF ..
+    make
+    make install
+    cd -
+fi
diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of lz4 that can be used as dependency of mxnet.
+LZ4_VERSION=r130
+if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
+    # Download and build lz4
+    >&2 echo "Building lz4..."
+    curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip
+    unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH
+    cd $DEPS_PATH/lz4-$LZ4_VERSION
+    make
+    make PREFIX=$DEPS_PATH install
+    cd -
+fi
diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds the static library of openblas that can be used as dependency of mxnet.
+OPENBLAS_VERSION=0.3.3
+if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
+    # download and build openblas
+    >&2 echo "Building openblas..."
+
+    curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip
+    unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH
+    cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION
+
+    make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1
+    make PREFIX=$DEPS_PATH install
+    cd -
+    ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a
+    ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a
+fi