Skip to content

Commit

Permalink
Build armhf binary on docker (#356)
Browse files Browse the repository at this point in the history
* Build and test armhf wheel with docker.

* Fix dockerfile bug.

* Change Docker image names
  • Loading branch information
YukioOobuchi authored and YasunariZHashimoto committed Feb 18, 2019
1 parent 5d1ebb0 commit e8332fb
Show file tree
Hide file tree
Showing 5 changed files with 290 additions and 49 deletions.
93 changes: 44 additions & 49 deletions build-tools/make/build-with-docker.mk
Expand Up @@ -24,56 +24,51 @@ NNABLA_BUILD_WITH_DOCKER_INCLUDED = True
NNABLA_DIRECTORY ?= $(shell pwd)
include $(NNABLA_DIRECTORY)/build-tools/make/options.mk

DOCKER_IMAGE_NAME_BASE ?= nnabla-build

DOCKER_IMAGE_AUTO_FORMAT ?= $(DOCKER_IMAGE_NAME_BASE)-auto-format
DOCKER_IMAGE_DOC ?= $(DOCKER_IMAGE_NAME_BASE)-doc
DOCKER_IMAGE_BUILD ?= $(DOCKER_IMAGE_NAME_BASE)-build
DOCKER_IMAGE_BUILD_ANDROID ?= $(DOCKER_IMAGE_NAME_BASE)-build-android
DOCKER_IMAGE_NNABLA ?= $(DOCKER_IMAGE_NAME_BASE)-nnabla
DOCKER_IMAGE_ONNX_TEST ?= $(DOCKER_IMAGE_NAME_BASE)-onnx-test

DOCKER_RUN_OPTS +=--rm
DOCKER_RUN_OPTS += -v $$(pwd):$$(pwd)
DOCKER_RUN_OPTS += -w $$(pwd)
DOCKER_RUN_OPTS += -u $$(id -u):$$(id -g)
DOCKER_RUN_OPTS += -e HOME=/tmp
DOCKER_RUN_OPTS += -e CMAKE_OPTS=$(CMAKE_OPTS)
DOCKER_IMAGE_NAME_BASE ?= nnabla-py$(PYTHON_VERSION_MAJOR)$(PYTHON_VERSION_MINOR)

DOCKER_IMAGE_AUTO_FORMAT ?= $(DOCKER_IMAGE_NAME_BASE)-auto-format:$(shell md5sum $(NNABLA_DIRECTORY)/docker/development/Dockerfile.auto-format |cut -d \ -f 1)
DOCKER_IMAGE_DOC ?= $(DOCKER_IMAGE_NAME_BASE)-doc:$(shell md5sum $(NNABLA_DIRECTORY)/docker/development/Dockerfile.document |cut -d \ -f 1)
DOCKER_IMAGE_BUILD ?= $(DOCKER_IMAGE_NAME_BASE)-build$(ARCH_SUFFIX):$(shell md5sum $(NNABLA_DIRECTORY)/docker/development/Dockerfile.build$(ARCH_SUFFIX) |cut -d \ -f 1)
DOCKER_IMAGE_BUILD_ANDROID ?= $(DOCKER_IMAGE_NAME_BASE)-build-android:$(shell md5sum $(NNABLA_DIRECTORY)/docker/development/Dockerfile.android |cut -d \ -f 1)
DOCKER_IMAGE_NNABLA ?= $(DOCKER_IMAGE_NAME_BASE)-nnabla:$(shell md5sum $(NNABLA_DIRECTORY)/docker/development/Dockerfile.build |cut -d \ -f 1)
DOCKER_IMAGE_ONNX_TEST ?= $(DOCKER_IMAGE_NAME_BASE)-onnx-test$(ARCH_SUFFIX):$(shell md5sum $(NNABLA_DIRECTORY)/docker/development/Dockerfile.onnx-test$(ARCH_SUFFIX) |cut -d \ -f 1)

########################################################################################################################
# Docker images
.PHONY: docker_image_auto_format
docker_image_auto_format:
docker pull ubuntu:16.04
cd $(NNABLA_DIRECTORY) \
&& docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_AUTO_FORMAT) -f docker/development/Dockerfile.auto-format .

.PHONY: docker_image_doc
docker_image_doc:
docker pull ubuntu:16.04
cd $(NNABLA_DIRECTORY) \
&& docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_DOC) -f docker/development/Dockerfile.document .

.PHONY: docker_image_build
docker_image_build:
docker pull centos:6
cd $(NNABLA_DIRECTORY) \
&& docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_BUILD) \
-f docker/development/Dockerfile.build .

.PHONY: docker_image_onnx_test
docker_image_onnx_test:
docker pull ubuntu:16.04
cd $(NNABLA_DIRECTORY) \
&& docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_ONNX_TEST) \
-f docker/development/Dockerfile.onnx-test .

.PHONY: docker_image_build_android
docker_image_build_android:
docker pull ubuntu:16.04
cd $(NNABLA_DIRECTORY) \
&& docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_BUILD_ANDROID) \
-f docker/development/Dockerfile.android .
.PHONY: docker_image_auto_format$(DOCKER_IMAGE_TARGET_SUFFIX)
docker_image_auto_format$(DOCKER_IMAGE_TARGET_SUFFIX):
if ! docker image inspect $(DOCKER_IMAGE_AUTO_FORMAT) >/dev/null 2>/dev/null; then \
docker pull ubuntu:16.04 && \
(cd $(NNABLA_DIRECTORY) && docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_AUTO_FORMAT) -f docker/development/Dockerfile.auto-format .) \
fi

.PHONY: docker_image_doc$(DOCKER_IMAGE_TARGET_SUFFIX)
docker_image_doc$(DOCKER_IMAGE_TARGET_SUFFIX):
if ! docker image inspect $(DOCKER_IMAGE_DOC) >/dev/null 2>/dev/null; then \
docker pull ubuntu:16.04 && \
( cd $(NNABLA_DIRECTORY) && docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_DOC) -f docker/development/Dockerfile.document . ) \
fi

.PHONY: docker_image_build$(DOCKER_IMAGE_TARGET_SUFFIX)
docker_image_build$(DOCKER_IMAGE_TARGET_SUFFIX):
if ! docker image inspect $(DOCKER_IMAGE_BUILD) >/dev/null 2>/dev/null; then \
docker pull $(shell cat $(NNABLA_DIRECTORY)/docker/development/Dockerfile.build$(ARCH_SUFFIX) |grep ^FROM |awk '{print $$2}') && \
(cd $(NNABLA_DIRECTORY) && docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_BUILD) -f docker/development/Dockerfile.build$(ARCH_SUFFIX) .) \
fi

.PHONY: docker_image_onnx_test$(DOCKER_IMAGE_TARGET_SUFFIX)
docker_image_onnx_test$(DOCKER_IMAGE_TARGET_SUFFIX):
if ! docker image inspect $(DOCKER_IMAGE_ONNX_TEST) >/dev/null 2>/dev/null; then \
docker pull $(shell cat $(NNABLA_DIRECTORY)/docker/development/Dockerfile.onnx-test$(ARCH_SUFFIX) |grep ^FROM |awk '{print $$2}') && \
(cd $(NNABLA_DIRECTORY) && docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_ONNX_TEST) -f docker/development/Dockerfile.onnx-test$(ARCH_SUFFIX) .) \
fi

.PHONY: docker_image_build_android$(DOCKER_IMAGE_TARGET_SUFFIX)
docker_image_build_android$(DOCKER_IMAGE_TARGET_SUFFIX):
if ! docker image inspect $(DOCKER_IMAGE_BUILD_ANDROID) >/dev/null 2>/dev/null; then \
docker pull ubuntu:16.04 && \
(cd $(NNABLA_DIRECTORY) && docker build $(DOCKER_BUILD_ARGS) -t $(DOCKER_IMAGE_BUILD_ANDROID) -f docker/development/Dockerfile.android .) \
fi

########################################################################################################################
# Auto Format
Expand Down Expand Up @@ -110,12 +105,12 @@ bwd-nnabla-test-cpplib: docker_image_build
.PHONY: bwd-nnabla-wheel
bwd-nnabla-wheel: docker_image_build
cd $(NNABLA_DIRECTORY) \
&& docker run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_BUILD) make -f build-tools/make/build.mk MAKE_MANYLINUX_WHEEL=ON nnabla-wheel
&& docker run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_BUILD) make -f build-tools/make/build.mk MAKE_MANYLINUX_WHEEL=$(MAKE_MANYLINUX_WHEEL) nnabla-wheel

.PHONY: bwd-nnabla-test
bwd-nnabla-test: docker_image_onnx_test
cd $(NNABLA_DIRECTORY) \
&& docker run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_ONNX_TEST) make -f build-tools/make/build.mk nnabla-test-local
&& docker run $(DOCKER_RUN_OPTS) $(DOCKER_IMAGE_ONNX_TEST) make -f build-tools/make/build.mk nnabla-test

.PHONY: bwd-nnabla-shell
bwd-nnabla-shell: docker_image_build
Expand Down
11 changes: 11 additions & 0 deletions build-tools/make/options.mk
Expand Up @@ -17,6 +17,13 @@ NNABLA_OPTIONS_INCLUDED = True
########################################################################################################################
# Environments

DOCKER_RUN_OPTS =--rm
DOCKER_RUN_OPTS += -v $$(pwd):$$(pwd)
DOCKER_RUN_OPTS += -w $$(pwd)
DOCKER_RUN_OPTS += -u $$(id -u):$$(id -g)
DOCKER_RUN_OPTS += -e HOME=/tmp
DOCKER_RUN_OPTS += -e CMAKE_OPTS=$(CMAKE_OPTS)

DOCKER_RUN_OPTS += -v $(HOME)/.ccache:/tmp/.ccache

## If your environment is under proxy uncomment following lines.
Expand Down Expand Up @@ -62,6 +69,9 @@ DOCKER_RUN_OPTS += -e PARALLEL_BUILD_NUM=$(PARALLEL_BUILD_NUM)
WHEEL_SUFFIX ?=
DOCKER_RUN_OPTS += -e WHEEL_SUFFIX=$(WHEEL_SUFFIX)

ARCH_SUFFIX ?=
DOCKER_RUN_OPTS += -e ARCH_SUFFIX=$(ARCH_SUFFIX)

########################################################################################################################
# Output directories

Expand All @@ -81,6 +91,7 @@ DOCKER_RUN_OPTS += -e PYTEST_LD_LIBRARY_PATH_EXTRA=$(PYTEST_LD_LIBRARY_PATH_EXTR


export DOCKER_RUN_OPTS
export ARCH_SUFFIX

########################################################################################################################
# Functions for makefile
Expand Down
117 changes: 117 additions & 0 deletions docker/development/Dockerfile.build-armhf
@@ -0,0 +1,117 @@
# Copyright (c) 2017 Sony Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM multiarch/ubuntu-core:armhf-xenial

ENV LC_ALL C
ENV LANG C
ENV LANGUAGE C

RUN apt-get update && apt-get install -y --no-install-recommends \
bzip2 \
ca-certificates \
ccache \
clang-format-3.8 \
cmake \
curl \
g++ \
git \
libarchive-dev \
libatlas-dev \
libhdf5-dev \
liblapack-dev \
libopenmpi-dev \
make \
openmpi-bin \
python \
python-dev \
python-pip \
python-setuptools \
python-wheel \
python3 \
python3-dev \
python3-pip \
python3-setuptools \
python3-wheel \
unzip \
wget \
zip

RUN mkdir /tmp/deps \
&& cd /tmp/deps \
&& curl -L https://github.com/google/protobuf/archive/v3.1.0.tar.gz -o protobuf-v3.1.0.tar.gz \
&& tar xvf protobuf-v3.1.0.tar.gz \
&& cd protobuf-3.1.0 \
&& mkdir build \
&& cd build \
&& cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-Dprotobuf_BUILD_TESTS=OFF \
../cmake \
&& make \
&& make install \
&& cd / \
&& rm -rf /tmp/*

RUN python3 -m pip install --upgrade pip
RUN python2 -m pip install --upgrade pip

RUN pip install --no-cache-dir \
autopep8 \
boto3 \
cython \
mako \
numpy\<1.16 \
onnx==1.3.0 \
pillow \
protobuf \
pytest \
pyyaml \
requests \
scipy \
tqdm \
virtualenv

RUN pip install \
--no-cache-dir \
--global-option=build_ext \
--global-option="-I/usr/include/hdf5/serial" \
--global-option="-L/usr/lib/arm-linux-gnueabihf/hdf5/serial" \
h5py

RUN pip3 install --no-cache-dir \
autopep8 \
boto3 \
cython \
mako \
numpy\<1.16 \
onnx==1.3.0 \
pillow \
protobuf \
pytest \
pyyaml \
requests \
scipy \
tqdm \
virtualenv

RUN pip3 install \
--no-cache-dir \
--global-option=build_ext \
--global-option="-I/usr/include/hdf5/serial" \
--global-option="-L/usr/lib/arm-linux-gnueabihf/hdf5/serial" \
h5py

RUN pip install ipython==5.0
RUN pip3 install ipython
1 change: 1 addition & 0 deletions docker/development/Dockerfile.onnx-test
Expand Up @@ -119,6 +119,7 @@ RUN set -xe \
scipy \
tqdm \
wheel \
virtualenv \
&& pip install pyyaml onnx==1.3.0 future Cython autopep8 requests \
graphviz \
&& pip install cntk || true \
Expand Down
117 changes: 117 additions & 0 deletions docker/development/Dockerfile.onnx-test-armhf
@@ -0,0 +1,117 @@
# Copyright (c) 2017 Sony Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM multiarch/ubuntu-core:armhf-xenial

ENV LC_ALL C
ENV LANG C
ENV LANGUAGE C

RUN apt-get update && apt-get install -y --no-install-recommends \
bzip2 \
ca-certificates \
ccache \
clang-format-3.8 \
cmake \
curl \
g++ \
git \
libarchive-dev \
libatlas-dev \
libhdf5-dev \
liblapack-dev \
libopenmpi-dev \
make \
openmpi-bin \
python \
python-dev \
python-pip \
python-setuptools \
python-wheel \
python3 \
python3-dev \
python3-pip \
python3-setuptools \
python3-wheel \
unzip \
wget \
zip

RUN mkdir /tmp/deps \
&& cd /tmp/deps \
&& curl -L https://github.com/google/protobuf/archive/v3.1.0.tar.gz -o protobuf-v3.1.0.tar.gz \
&& tar xvf protobuf-v3.1.0.tar.gz \
&& cd protobuf-3.1.0 \
&& mkdir build \
&& cd build \
&& cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-Dprotobuf_BUILD_TESTS=OFF \
../cmake \
&& make \
&& make install \
&& cd / \
&& rm -rf /tmp/*

RUN python3 -m pip install --upgrade pip
RUN python2 -m pip install --upgrade pip

RUN pip install --no-cache-dir \
autopep8 \
boto3 \
cython \
mako \
numpy\<1.16 \
onnx==1.3.0 \
pillow \
protobuf \
pytest \
pyyaml \
requests \
scipy \
tqdm \
virtualenv

RUN pip install \
--no-cache-dir \
--global-option=build_ext \
--global-option="-I/usr/include/hdf5/serial" \
--global-option="-L/usr/lib/arm-linux-gnueabihf/hdf5/serial" \
h5py

RUN pip3 install --no-cache-dir \
autopep8 \
boto3 \
cython \
mako \
numpy\<1.16 \
onnx==1.3.0 \
pillow \
protobuf \
pytest \
pyyaml \
requests \
scipy \
tqdm \
virtualenv

RUN pip3 install \
--no-cache-dir \
--global-option=build_ext \
--global-option="-I/usr/include/hdf5/serial" \
--global-option="-L/usr/lib/arm-linux-gnueabihf/hdf5/serial" \
h5py

RUN pip install ipython==5.0
RUN pip3 install ipython

0 comments on commit e8332fb

Please sign in to comment.