Merge remote-tracking branch 'upstream/master' into fp32_update_on_ch…

…ainerx

.mergify.yml

@@ -15,6 +15,15 @@ pull_request_rules:
       - status-success=pfn-public-ci/chainer.py3.cv
       - status-success=pfn-public-ci/chainer.py3.cv.examples
       - status-success=pfn-public-ci/chainer.py3.cv.gpu
+      - status-success=pfn-public-ci/chainermn.cpu
+      - status-success=pfn-public-ci/chainermn.gpu
+      - status-success=pfn-public-ci/onnxchainer.py35.cpu
+      - status-success=pfn-public-ci/onnxchainer.py36.cpu
+      - status-success=pfn-public-ci/onnxchainer.py36.cpu.onnx14
+      - status-success=pfn-public-ci/onnxchainer.py37.cpu
+      - status-success=pfn-public-ci/onnxchainer.py37.cpu.onnx15
+      - status-success=pfn-public-ci/onnxchainer.py37.gpu
+      - status-success=pfn-public-ci/onnxchainer.win.py37.gpu
     actions:
       merge:
         method: merge

.pfnci/config.pbtxt

@@ -16,7 +16,10 @@ configs {
       memory: 30
     }
     time_limit: {
-      seconds: 900
+      seconds: 1200
+    }
+    checkout_strategy {
+      include_dot_git: true
     }
     environment_variables { key: "GPU" value: "0" }
     environment_variables {
@@ -35,7 +38,10 @@ configs {
       gpu: 2
     }
     time_limit: {
-      seconds: 900
+      seconds: 1200
+    }
+    checkout_strategy {
+      include_dot_git: true
     }
     environment_variables { key: "GPU" value: "2" }
     # NOTE: If # of threads / # of GPUs exceeds 10, GPU tests may cause
@@ -61,7 +67,10 @@ configs {
       memory: 30
     }
     time_limit: {
-      seconds: 900
+      seconds: 1200
+    }
+    checkout_strategy {
+      include_dot_git: true
     }
     environment_variables { key: "GPU" value: "0" }
     environment_variables {
@@ -80,7 +89,10 @@ configs {
       gpu: 2
     }
     time_limit: {
-      seconds: 900
+      seconds: 1200
+    }
+    checkout_strategy {
+      include_dot_git: true
     }
     environment_variables { key: "GPU" value: "2" }
     # NOTE: If # of threads / # of GPUs exceeds 10, GPU tests may cause
@@ -112,6 +124,9 @@ configs {
     time_limit: {
       seconds: 3600
     }
+    checkout_strategy {
+      include_dot_git: true
+    }
     command: "bash .pfnci/script.sh docker.py37.test"
   }
 }
@@ -125,6 +140,9 @@ configs {
     time_limit: {
       seconds: 3600
     }
+    checkout_strategy {
+      include_dot_git: true
+    }
     command: "bash .pfnci/script.sh docker.py27and35.test"
   }
 }
@@ -138,6 +156,9 @@ configs {
     time_limit: {
       seconds: 3600
     }
+    checkout_strategy {
+      include_dot_git: true
+    }
     command: "bash .pfnci/script.sh docker.py37.push"
   }
 }
@@ -151,6 +172,9 @@ configs {
     time_limit: {
       seconds: 3600
     }
+    checkout_strategy {
+      include_dot_git: true
+    }
     command: "bash .pfnci/script.sh docker.py27and35.push"
   }
 }
@@ -273,6 +297,9 @@ configs {
     time_limit: {
       seconds: 1200
     }
+    checkout_strategy {
+      include_dot_git: true
+    }
     command: "bash .pfnci/script.sh chainermn"
   }
 }
@@ -287,6 +314,9 @@ configs {
     }
     time_limit: {
       seconds: 900
+    }
+    checkout_strategy {
+      include_dot_git: true
     }
 	environment_variables { key: "GPU" value: "2" }
     command: "bash .pfnci/script.sh chainermn"

.pfnci/py27and35.Dockerfile

@@ -13,25 +13,16 @@ RUN apt-get update -y && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 RUN python3.5 -m pip install --upgrade pip setuptools
-RUN python2.7 -m pip install --upgrade pip setuptools
 
 RUN python3.5 -m pip install \
     'cython>=0.28.0' 'ideep4py<2.1' 'pytest==4.1.1' 'pytest-xdist==1.26.1' \
     mock setuptools typing \
-    typing_extensions filelock numpy>=1.9.0 protobuf>=3.0.0 six>=1.9.0
-
-RUN python2.7 -m pip install \
-    'cython>=0.28.0' 'ideep4py<2.1' 'pytest==4.1.1' 'pytest-xdist==1.26.1' \
-    mock setuptools typing \
-    typing_extensions filelock protobuf>=3.0.0 six>=1.9.0
-
-# Newer versions of numpy and more-itertools no longer support python2.
-RUN python2.7 -m pip install 'numpy<=1.16.5' 'more-itertools<=5.0.0'
+    typing_extensions filelock 'numpy>=1.9.0' 'protobuf>=3.0.0' 'six>=1.9.0'
 
 COPY --from=xpytest /usr/local/bin/xpytest /usr/local/bin/xpytest
 
 COPY . /cupy
 RUN cd /cupy && \
     echo 'install-%:\n\t$* -m pip install .\n' > /tmp/install.mk && \
     cat /tmp/install.mk && \
-    make -f /tmp/install.mk -j 2 install-python3.5 install-python2.7
+    make -f /tmp/install.mk -j 2 install-python3.5

.pfnci/run.sh

@@ -15,6 +15,8 @@
 
 set -eux
 
+export CHAINER_CI=flexci
+
 cp -a /src /chainer
 cp /chainer/setup.cfg /
 cd /
@@ -57,7 +59,8 @@ test_py37() {
   #-----------------------------------------------------------------------------
   # Install Chainer
   #-----------------------------------------------------------------------------
-  CHAINER_BUILD_CHAINERX=1 CHAINERX_BUILD_CUDA=1 MAKEFLAGS="-j$(nproc)" \
+  CHAINER_BUILD_CHAINERX=1 CHAINERX_BUILD_CUDA=1 \
+  MAKEFLAGS="-j$(get_build_concurrency)" \
   CHAINERX_NVCC_GENERATE_CODE=arch=compute_70,code=sm_70 \
       python3.7 -m pip install /chainer[test] 2>&1 >/tmp/install.log &
   install_pid=$!
@@ -77,7 +80,7 @@ test_py37() {
       -DCHAINERX_WARNINGS_AS_ERRORS=ON \
       /chainer/chainerx_cc
   # NOTE: Use nice to prioritize pip install process.
-  nice -n 19 make "-j$(nproc)"
+  nice -n 19 make "-j$(get_build_concurrency)"
   ctest --output-on-failure "-j$(nproc)" && :
   cc_test_status=$?
   popd
@@ -136,7 +139,8 @@ test_py27and35() {
   # Install Chainer
   #-----------------------------------------------------------------------------
   # Install Chainer for python3.5.
-  CHAINER_BUILD_CHAINERX=1 CHAINERX_BUILD_CUDA=1 MAKEFLAGS="-j$(nproc)" \
+  CHAINER_BUILD_CHAINERX=1 CHAINERX_BUILD_CUDA=1 \
+  MAKEFLAGS="-j$(get_build_concurrency)" \
   CHAINERX_NVCC_GENERATE_CODE=arch=compute_70,code=sm_70 \
       python3.5 -m pip install /chainer[test] 2>&1 >/tmp/install-py35.log &
   install_pid=$!
@@ -206,7 +210,8 @@ test_chainermn_sub() {
   #-----------------------------------------------------------------------------
   # Install Chainer
   #-----------------------------------------------------------------------------
-  CHAINER_BUILD_CHAINERX=1 CHAINERX_BUILD_CUDA=1 MAKEFLAGS="-j$(nproc)" \
+  CHAINER_BUILD_CHAINERX=1 CHAINERX_BUILD_CUDA=1 \
+  MAKEFLAGS="-j$(get_build_concurrency)" \
   CHAINERX_NVCC_GENERATE_CODE=arch=compute_70,code=sm_70 \
       python -m pip install /chainer[test] 2>&1 >/tmp/install-py3.log &
   install_pid=$!
@@ -227,6 +232,18 @@ test_chainermn_sub() {
 				   /chainer/tests/chainermn_tests
 }
 
+# get_build_concurrency determines the parallelism of the build process.
+# Currently maximum is set to 16 to avoid exhausting memory.
+get_build_concurrency() {
+    local num_cores="$(nproc)"
+    local num_cores_max="16"
+    if [ ${num_cores} -gt ${num_cores_max} ]; then
+        echo "${num_cores_max}"
+    else
+        echo "${num_cores}"
+    fi
+}
+
 ################################################################################
 # Bootstrap
 ################################################################################

.pfnci/script.sh

@@ -4,7 +4,7 @@
 # this should fall back to CPU testing automatically.  This script requires that
 # a corresponding Docker image is accessible from the machine.
 # TODO(imos): Enable external contributors to test this script on their
-# machines.  Specifically, locate a Dockerfile generating chainer-ci-prep.*.
+# machines.
 #
 # Usage: .pfnci/script.sh [target]
 # - target is a test target (e.g., "py37").
@@ -36,6 +36,9 @@ main() {
   # Prepare docker args.
   docker_args=(docker run  --rm)
 
+  # Base development branch name.
+  base_branch="$(get_base_branch)"
+
   # Run target-specific commands.
   case "${TARGET}" in
     # Unit tests.
@@ -55,7 +58,7 @@ main() {
         docker_args+=(--env="SPREADSHEET_ID=${SPREADSHEET_ID}")
       fi
       run "${docker_args[@]}" \
-          "asia.gcr.io/pfn-public-ci/chainer-ci-prep.${TARGET}" \
+          "asia.gcr.io/pfn-public-ci/chainer-ci-prep.${TARGET}:${base_branch}" \
           bash /src/.pfnci/run.sh "${TARGET}"
       ;;
     # Docker builds.
@@ -69,6 +72,11 @@ main() {
         echo "Unsupported docker target action: ${action}" >&2
         exit 1
       fi
+      if [ "${action}" = "push" ] && ! is_known_base_branch "${FLEXCI_BRANCH}"; then
+        echo "Branch invalid for docker push: ${FLEXCI_BRANCH}" >&2
+        exit 1
+      fi
+
       # This script can be run in CuPy repository to enable CI to build Chainer
       # base images with a specified CuPy version.  This block enables the
       # script can also be run even in Chainer repository.
@@ -78,15 +86,16 @@ main() {
       if [ ! -d "${cupy_directory}/cupy" ]; then
         if [ ! -d .pfnci/cupy ]; then
           run git clone https://github.com/cupy/cupy.git .pfnci/cupy
+          run git -C .pfnci/cupy checkout "${base_branch}"
         fi
         cupy_directory=.pfnci/cupy
       fi
       run docker build -t \
-          "asia.gcr.io/pfn-public-ci/chainer-ci-prep.${target}" \
+          "asia.gcr.io/pfn-public-ci/chainer-ci-prep.${target}:${base_branch}" \
           -f "$(dirname "${BASH_SOURCE}")/${target}.Dockerfile" \
           "${cupy_directory}"
       if [ "${action}" == 'push' ]; then
-        run docker push "asia.gcr.io/pfn-public-ci/chainer-ci-prep.${target}"
+        run docker push "asia.gcr.io/pfn-public-ci/chainer-ci-prep.${target}:${base_branch}"
       fi
       ;;
 	'chainermn' )
@@ -120,7 +129,7 @@ main() {
 
 # run executes a command.  If DRYRUN is enabled, run just prints the command.
 run() {
-  echo '+' "$@"
+  echo '+' "$@" >&2
   if [ "${DRYRUN:-}" == '' ]; then
     "$@"
   fi
@@ -139,6 +148,27 @@ prepare_docker() {
   run gcloud auth configure-docker
 }
 
+# is_known_base_branch returns 0 only if the given branch name is a known
+# base development branch.
+is_known_base_branch() {
+  local branch="${1##refs/heads/}"
+  for BASE_BRANCH in master v6; do
+    if [ "${branch}" = "${BASE_BRANCH}" ]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+# get_base_branch returns the base development branch for the current HEAD.
+get_base_branch() {
+  for BASE_BRANCH in master v6; do
+    run git merge-base --is-ancestor "origin/${BASE_BRANCH}" HEAD && echo "${BASE_BRANCH}" && return 0
+  done
+  echo "Base branch of HEAD is not valid." >&2
+  return 1
+}
+
 ################################################################################
 # Bootstrap
 ################################################################################

.travis.yml

@@ -27,16 +27,26 @@ matrix:
       env:
       - CHAINER_TRAVIS_TEST="chainer"
       - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5"
+      - GOROOT=$HOME/go
       if: type in (pull_request)
 
+    - name: "Examples"
+      language: python
+      dist: xenial
+      python: "3.5"
+      env:
+      - CHAINER_TRAVIS_TEST="examples"
+      - GOROOT=$HOME/go
+
     - name: "macOS Py35 | chainer, chainermn, chainerx, docs"
       os: osx
       language: generic
       env:
       - CHAINER_TRAVIS_TEST="chainer"
-      - PYTHON_VERSION=3.5.1
+      - PYTHON_VERSION=3.5.2
       - PYENV_ROOT=~/.pyenv
       - PATH=$PYENV_ROOT/shims:$PATH:$PYENV_ROOT/bin
+      - GOROOT=/usr/local/opt/go/libexec
       if: (branch = master OR branch = v6) AND (NOT type in (pull_request))
 
 before_install:

README.md

@@ -7,12 +7,13 @@
 [![travis](https://img.shields.io/travis/chainer/chainer/master.svg)](https://travis-ci.org/chainer/chainer)
 [![coveralls](https://img.shields.io/coveralls/chainer/chainer.svg)](https://coveralls.io/github/chainer/chainer)
 [![Read the Docs](https://readthedocs.org/projects/chainer/badge/?version=stable)](https://docs.chainer.org/en/stable/?badge=stable)
+[![Optuna](https://img.shields.io/badge/Optuna-integrated-blue)](https://optuna.org)
 
 [**Website**](https://chainer.org/)
 | [**Docs**](https://docs.chainer.org/en/stable/)
 | [**Install Guide**](https://docs.chainer.org/en/stable/install.html)
 | **Tutorials** ([ja](https://tutorials.chainer.org/ja/))
-| **Examples** ([Official](https://github.com/chainer/chainer/tree/master/examples), [External](https://github.com/chainer-community/awesome-chainer))
+| **Examples** ([Official](examples), [External](https://github.com/chainer-community/awesome-chainer))
 | [**Concepts**](https://docs.chainer.org/en/stable/guides/)
 | [**ChainerX**](#chainerx)
 
@@ -25,10 +26,6 @@ It provides automatic differentiation APIs based on the **define-by-run** approa
 It also supports CUDA/cuDNN using [CuPy](https://github.com/cupy/cupy) for high performance training and inference.
 For more details about Chainer, see the documents and resources listed above and join the community in Forum, Slack, and Twitter.
 
-## Stable version
-
-The stable version of current Chainer is separated in here: [v6](https://github.com/chainer/chainer/tree/v6).
-
 ## Installation
 
 *For more details, see the [installation guide](https://docs.chainer.org/en/stable/install.html).*

chainer/__init__.py

@@ -234,6 +234,7 @@ def __repr__(self):
                     'Only float16/32/64 are allowed.'.format(_chainer_dtype))
 global_config.in_recomputing = False
 global_config._will_recompute = False
+global_config.compute_mode = None
 
 
 def is_debug():
@@ -313,6 +314,10 @@ def get_dtype(dtype=None, map_mixed16=None):
     return numpy.dtype(dtype)
 
 
+def get_compute_mode():
+    return config.compute_mode
+
+
 basic_math.install_variable_arithmetics()
 array.get_item.install_variable_get_item()