Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[WIP] Windows dev environment configuration, update install instructions from source in the docs #17808

Closed
wants to merge 37 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
f80a477
new windows build
vexilligera Mar 11, 2020
be588c2
vs 2019 and cuda 10.2
vexilligera Mar 24, 2020
fae6d34
ami deployment
vexilligera Mar 25, 2020
5febea3
s3
vexilligera Mar 26, 2020
0d4e884
use vcvars64
vexilligera Mar 28, 2020
3416a64
stabilize gpu build by removing /MP
vexilligera Mar 28, 2020
33fc990
update dmlc-core
vexilligera Mar 28, 2020
395c513
refreshenv after build
vexilligera Mar 29, 2020
bc2d895
add /MP and /Zc:__cplusplus
vexilligera Mar 29, 2020
74a8d8c
add retry on failure
vexilligera Mar 29, 2020
ada659f
fix script
vexilligera Mar 29, 2020
977b56e
debug on ci
vexilligera Mar 30, 2020
7cbc4a9
probe cpu dir
vexilligera Mar 30, 2020
78b0f28
pack opencv though it's in the paht
vexilligera Mar 30, 2020
9c9758a
print lib dir
vexilligera Mar 30, 2020
5a2fa87
add lib to the path after build
vexilligera Mar 30, 2020
55c5e93
add test lib path
vexilligera Mar 30, 2020
f62198a
Add 5.2 arch
leezu Mar 31, 2020
af0d910
print to stderr as well
leezu Mar 31, 2020
ef64105
Drop 7.0
leezu Mar 31, 2020
73437de
Remove debug statements in load_lib
leezu Mar 31, 2020
19cc08b
add 7.0 back to build both
vexilligera Apr 1, 2020
01e3faf
print env vars in gpu ci
vexilligera Apr 1, 2020
2fef756
drop 7.0
vexilligera Apr 1, 2020
a786bfc
add path during test, print stuff
vexilligera Apr 1, 2020
f6f5a6a
build_windows.py cleanup
vexilligera Apr 1, 2020
5d14df3
print PATH
vexilligera Apr 1, 2020
9944a2e
update print and add 7.0 back
vexilligera Apr 1, 2020
ca81f3a
drop 7.0 and print instance type
vexilligera Apr 1, 2020
dddbf2a
print instance type
vexilligera Apr 1, 2020
3997316
add 7.0 again
vexilligera Apr 1, 2020
2054768
drop 7.0 before the previous build is done
vexilligera Apr 1, 2020
ba1896e
pip install -e python
vexilligera Apr 1, 2020
e0b8a23
Switch to g4 instance
leezu Apr 2, 2020
bac2b67
Remove debug statements
leezu Apr 2, 2020
27cce4a
Fix CMakeLists
leezu Apr 2, 2020
12e8a6c
Workaround broken gluon dataloader test_dataloader_context test
leezu Apr 2, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,8 @@ if(MSVC)
add_definitions(-DDMLC_STRICT_CXX11)
add_definitions(-DNOMINMAX)
set(CMAKE_C_FLAGS "/MP")
set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /bigobj")
# /Zc:__cplusplus to accurately report recent C++ language standards support
set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /bigobj /Zc:__cplusplus")
else()
include(CheckCXXCompilerFlag)
if(USE_CXX14_IF_AVAILABLE)
Expand Down
97 changes: 62 additions & 35 deletions ci/build_windows.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,20 @@
import zipfile
from distutils.dir_util import copy_tree
from enum import Enum
from subprocess import check_call
from subprocess import check_call, call

from util import *


# Fix for broken PATH with newline inserted presumably by VS studio installation of SQL server or
# other component which makes visual studio stop working.
os.environ['PATH']=os.environ.get('PATH').replace('\n','')

KNOWN_VCVARS = {
# https://gitlab.kitware.com/cmake/cmake/issues/18920
'VS 2015': r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat',
'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat',
'VS 2019': r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat'
}


Expand All @@ -54,10 +61,14 @@ class BuildFlavour(Enum):

CMAKE_FLAGS = {
'WIN_CPU': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
Expand All @@ -67,10 +78,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_CPU_MKLDNN': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
Expand All @@ -80,10 +95,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_CPU_MKLDNN_MKL': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=mkl '
'-DUSE_LAPACK=ON '
Expand All @@ -93,10 +112,14 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_CPU_MKL': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
'-DUSE_OPENCV=ON '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=mkl '
'-DUSE_LAPACK=ON '
Expand All @@ -106,29 +129,37 @@ class BuildFlavour(Enum):
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_GPU': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=ON '
'-DUSE_CUDNN=ON '
'-DENABLE_CUDA_RTC=ON '
'-DUSE_OPENCV=ON '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
'-DUSE_DIST_KVSTORE=OFF '
'-DMXNET_CUDA_ARCH="5.2" '
'-DMXNET_CUDA_ARCH="7.5" '
leezu marked this conversation as resolved.
Show resolved Hide resolved
'-DCMAKE_CXX_FLAGS="/FS /MD /O2 /Ob2" '
'-DUSE_MKL_IF_AVAILABLE=OFF '
'-DCMAKE_BUILD_TYPE=Release')

, 'WIN_GPU_MKLDNN': (
'-DCMAKE_C_COMPILER=cl '
'-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=ON '
'-DUSE_CUDNN=ON '
'-DENABLE_CUDA_RTC=ON '
'-DUSE_OPENCV=ON '
'-DOpenCV_RUNTIME=vc15 '
'-DOpenCV_ARCH=x64 '
'-DUSE_OPENMP=ON '
'-DUSE_BLAS=open '
'-DUSE_LAPACK=ON '
'-DUSE_DIST_KVSTORE=OFF '
'-DMXNET_CUDA_ARCH="5.2" '
'-DMXNET_CUDA_ARCH="7.5" '
'-DUSE_MKLDNN=ON '
'-DCMAKE_CXX_FLAGS="/FS /MD /O2 /Ob2" '
'-DCMAKE_BUILD_TYPE=Release')
Expand All @@ -140,38 +171,40 @@ def windows_build(args):
logging.info("Using vcvars environment:\n{}".format(args.vcvars))

path = args.output
os.makedirs(path, exist_ok=True)

mxnet_root = get_mxnet_root()
logging.info("Found MXNet root: {}".format(mxnet_root))
# cuda thrust + VS is flaky so try multiple times if fail
MAXIMUM_TRY = 5
build_try = 0

while build_try < MAXIMUM_TRY:
if os.path.exists(path):
shutil.rmtree(path)
os.makedirs(path, exist_ok=True)

url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip'
with tempfile.TemporaryDirectory() as tmpdir:
cmake_file_path = download_file(url, tmpdir)
with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref:
# Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe
zip_ref.extractall(tmpdir)
mxnet_root = get_mxnet_root()
logging.info("Found MXNet root: {}".format(mxnet_root))

with remember_cwd():
os.chdir(path)
cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
args.vcvars,
os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
CMAKE_FLAGS[args.flavour], mxnet_root)
cmd = "\"{}\" && cmake -G Ninja {} {}".format(args.vcvars,
CMAKE_FLAGS[args.flavour],
mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True)

cmd = "\"{}\" && jom".format(args.vcvars)
logging.info("Building with jom:\n{}".format(cmd))
cmd = "\"{}\" && ninja".format(args.vcvars)
logging.info("Building:\n{}".format(cmd))

t0 = int(time.time())
check_call(cmd, shell=True)

logging.info(
"Build flavour: {} complete in directory: \"{}\"".format(
args.flavour, os.path.abspath(path)))
logging.info("Build took {}".format(
datetime.timedelta(seconds=int(time.time() - t0))))
ret = call(cmd, shell=True)

if ret != 0:
build_try += 1
logging.info("{} build(s) have failed".format(build_try))
else:
logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
break
windows_package(args)


Expand All @@ -193,6 +226,7 @@ def windows_package(args):
for dll in dlls:
logging.info("packing dll: %s", dll)
shutil.copy(dll, pkgdir_lib)

os.chdir(get_mxnet_root())
logging.info('packing python bindings')
copy_tree('python', j(pkgdir, 'python'))
Expand Down Expand Up @@ -221,9 +255,6 @@ def main():
logging.getLogger().setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)-15s %(message)s')
logging.info("MXNet Windows build helper")
instance_info = ec2_instance_info()
if instance_info:
logging.info("EC2: %s", instance_info)

parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output",
Expand All @@ -233,7 +264,7 @@ def main():

parser.add_argument("--vcvars",
help="vcvars batch file location, typically inside vs studio install dir",
default=KNOWN_VCVARS['VS 2015'],
default=KNOWN_VCVARS['VS 2019'],
type=str)

parser.add_argument("--arch",
Expand All @@ -253,12 +284,8 @@ def main():
system = platform.system()
if system == 'Windows':
logging.info("Detected Windows platform")
if 'OpenBLAS_HOME' not in os.environ:
os.environ["OpenBLAS_HOME"] = "C:\\Program Files\\OpenBLAS-v0.2.19"
if 'OpenCV_DIR' not in os.environ:
os.environ["OpenCV_DIR"] = "C:\\Program Files\\OpenCV-v3.4.1\\build"
if 'CUDA_PATH' not in os.environ:
os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2"
os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is your concern?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We didn't agree on increasing the minimum cuda version

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We agreed on using VS 2019. Cuda 10 is a requirement for using VS 2019

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you sure about that? Usually visual studio can be upgraded by installing the vc++ toolset which then grants the possibility to compile other cuda versions. I'm not aware that this categorically excluded.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if 'MKL_ROOT' not in os.environ:
os.environ["MKL_ROOT"] = "C:\\Program Files (x86)\\IntelSWTools\\compilers_and_libraries\\windows\\mkl"
windows_build(args)
Expand Down
12 changes: 7 additions & 5 deletions ci/jenkins/Jenkins_steps.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def python3_gpu_ut_cython(docker_container_name) {
}

//------------------------------------------------------------------------------------------

/*
def compile_unix_cpu_openblas() {
return ['CPU: Openblas': {
node(NODE_LINUX_CPU) {
Expand Down Expand Up @@ -651,6 +651,7 @@ def compile_unix_amalgamation() {
}
}]
}
*/

def compile_windows_cpu() {
return ['Build CPU windows':{
Expand Down Expand Up @@ -708,6 +709,7 @@ def compile_windows_cpu_mkl() {
}]
}


def compile_windows_gpu() {
return ['Build GPU windows':{
node(NODE_WINDOWS_CPU) {
Expand Down Expand Up @@ -735,7 +737,7 @@ def compile_windows_gpu_mkldnn() {
}
}]
}

/*
def test_static_scala_cpu() {
return ['Static build CPU 14.04 Scala' : {
node(NODE_LINUX_CPU) {
Expand Down Expand Up @@ -1343,7 +1345,7 @@ def test_centos7_scala_cpu() {
}
}]
}

*/
def test_windows_python3_gpu() {
return ['Python 3: GPU Win':{
node(NODE_WINDOWS_GPU) {
Expand Down Expand Up @@ -1427,7 +1429,7 @@ def test_windows_julia10_cpu() {
}
}]
}

/*
def test_qemu_armv7_cpu() {
return ['ARMv7 QEMU': {
node(NODE_LINUX_CPU) {
Expand Down Expand Up @@ -1794,5 +1796,5 @@ def test_artifact_repository() {
}
}]
}

*/
return this
7 changes: 4 additions & 3 deletions ci/safe_docker_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@

from util import config_logging

DOCKER_STOP_TIMEOUT_SECONDS = 3
DOCKER_STOP_TIMEOUT_SECONDS = 10
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated change

CONTAINER_WAIT_SECONDS = 600
DOCKER_CLIENT_TIMEOUT = 600


class SafeDockerClient:
Expand All @@ -54,7 +55,7 @@ def _trim_container_id(cid):
return cid[:12]

def __init__(self):
self._docker_client = docker.from_env()
self._docker_client = docker.from_env(timeout=DOCKER_CLIENT_TIMEOUT)
self._containers = set()
self._docker_stop_timeout = DOCKER_STOP_TIMEOUT_SECONDS
self._container_wait_seconds = CONTAINER_WAIT_SECONDS
Expand Down Expand Up @@ -245,4 +246,4 @@ def main(command_line_arguments):


if __name__ == "__main__":
exit(main(sys.argv[1:]))
exit(main(sys.argv[1:]))
4 changes: 4 additions & 0 deletions ci/windows/test_py3_gpu.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
$env:MXNET_SUBGRAPH_VERBOSE=0
$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's print all ENV Vars to compare state of env vars in CI run vs manual powershell test
Get-ChildItem Env:

$env:PATH+=[io.path]::combine($pwd.Path,"windows_package\lib")

C:\Python37\Scripts\pip install -r tests\requirements.txt
C:\Python37\Scripts\pip install -e $env:PYTHONPATH

C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
Expand Down
Loading