python3.pkgs.mmcv: init at 1.7.1

pkgs/development/libraries/opencv/4.x.nix

@@ -37,9 +37,14 @@
 , enableContrib ? true
 
 , enableCuda ? (config.cudaSupport or false) && stdenv.hostPlatform.isx86_64
-, cudaPackages ? { }
+, enableCublas ? enableCuda
+, enableCudnn ? false # NOTE: CUDNN has a large impact on closure size so we disable it by default
+, enableCufft ? enableCuda
+, cudaPackages ? {}
+, symlinkJoin
 , nvidia-optical-flow-sdk
 
+, enableLto ? true
 , enableUnfree ? false
 , enableIpp ? false
 , enablePython ? false
@@ -79,9 +84,6 @@
 }:
 
 let
-  inherit (cudaPackages) cudatoolkit;
-  inherit (cudaPackages.cudaFlags) cudaCapabilities;
-
   version = "4.7.0";
 
   src = fetchFromGitHub {
@@ -227,6 +229,33 @@ let
   #multithreaded openblas conflicts with opencv multithreading, which manifest itself in hung tests
   #https://github.com/xianyi/OpenBLAS/wiki/Faq/4bded95e8dc8aadc70ce65267d1093ca7bdefc4c#multi-threaded
   openblas_ = blas.provider.override { singleThreaded = true; };
+
+  inherit (cudaPackages) backendStdenv cudaFlags cudaVersion;
+  inherit (cudaFlags) cudaCapabilities;
+
+  cuda-common-redist = with cudaPackages; [
+    cuda_cccl # <thrust/*>
+    libnpp # npp.h
+  ] ++ lib.optionals enableCublas [
+    libcublas # cublas_v2.h
+  ] ++ lib.optionals enableCudnn [
+    cudnn # cudnn.h
+  ] ++ lib.optionals enableCufft [
+    libcufft # cufft.h
+  ];
+
+  cuda-native-redist = symlinkJoin {
+    name = "cuda-native-redist-${cudaVersion}";
+    paths = with cudaPackages; [
+      cuda_cudart # cuda_runtime.h
+      cuda_nvcc
+    ] ++ cuda-common-redist;
+   };
+
+  cuda-redist = symlinkJoin {
+    name = "cuda-redist-${cudaVersion}";
+    paths = cuda-common-redist;
+   };
 in
 
 stdenv.mkDerivation {
@@ -298,17 +327,18 @@ stdenv.mkDerivation {
     ++ lib.optionals enableTesseract [ tesseract leptonica ]
     ++ lib.optional enableTbb tbb
     ++ lib.optionals stdenv.isDarwin [ bzip2 AVFoundation Cocoa VideoDecodeAcceleration CoreMedia MediaToolbox ]
-    ++ lib.optionals enableDocs [ doxygen graphviz-nox ];
+    ++ lib.optionals enableDocs [ doxygen graphviz-nox ]
+    ++ lib.optionals enableCuda [ cuda-redist ];
 
   propagatedBuildInputs = lib.optional enablePython pythonPackages.numpy
-    ++ lib.optionals enableCuda [ cudatoolkit nvidia-optical-flow-sdk ];
+    ++ lib.optionals enableCuda [ nvidia-optical-flow-sdk ];
 
   nativeBuildInputs = [ cmake pkg-config unzip ]
   ++ lib.optionals enablePython [
     pythonPackages.pip
     pythonPackages.wheel
     pythonPackages.setuptools
-  ];
+  ] ++ lib.optionals enableCuda [ cuda-native-redist ];
 
   env.NIX_CFLAGS_COMPILE = lib.optionalString enableEXR "-I${ilmbase.dev}/include/OpenEXR";
 
@@ -338,12 +368,30 @@ stdenv.mkDerivation {
     (opencvFlag "OPENEXR" enableEXR)
     (opencvFlag "OPENJPEG" enableJPEG2000)
     "-DWITH_JASPER=OFF" # OpenCV falls back to a vendored copy of Jasper when OpenJPEG is disabled
-    (opencvFlag "CUDA" enableCuda)
-    (opencvFlag "CUBLAS" enableCuda)
     (opencvFlag "TBB" enableTbb)
+
+    # CUDA options
+    (opencvFlag "CUDA" enableCuda)
+    (opencvFlag "CUDA_FAST_MATH" enableCuda)
+    (opencvFlag "CUBLAS" enableCublas)
+    (opencvFlag "CUDNN" enableCudnn)
+    (opencvFlag "CUFFT" enableCufft)
+
+    # LTO options
+    (opencvFlag "ENABLE_LTO" enableLto)
+    (opencvFlag "ENABLE_THIN_LTO" (
+      enableLto && (
+        # Only clang supports thin LTO, so we must either be using clang through the stdenv,
+        stdenv.cc.isClang ||
+          # or through the backend stdenv.
+          (enableCuda && backendStdenv.cc.isClang)
+      )
+    ))
   ] ++ lib.optionals enableCuda [
     "-DCUDA_FAST_MATH=ON"
-    "-DCUDA_HOST_COMPILER=${cudatoolkit.cc}/bin/cc"
+    # We need to set the C and C++ host compilers for CUDA to the same compiler.
+    "-DCMAKE_C_COMPILER=${backendStdenv.cc}/bin/cc"
+    "-DCMAKE_CXX_COMPILER=${backendStdenv.cc}/bin/c++"
     "-DCUDA_NVCC_FLAGS=--expt-relaxed-constexpr"
 
     # OpenCV respects at least three variables:

pkgs/development/python-modules/mmcv/default.nix

@@ -0,0 +1,122 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytestCheckHook
+, pythonOlder
+, torch
+, torchvision
+, opencv4
+, yapf
+, packaging
+, pillow
+, addict
+, ninja
+, which
+, onnx
+, onnxruntime
+, scipy
+, pyturbojpeg
+, tifffile
+, lmdb
+, symlinkJoin
+}:
+
+let
+  inherit (torch) cudaCapabilities cudaPackages cudaSupport;
+  inherit (cudaPackages) backendStdenv cudaVersion;
+
+  cuda-common-redist = with cudaPackages; [
+    cuda_cccl # <thrust/*>
+    libcublas # cublas_v2.h
+    libcusolver # cusolverDn.h
+    libcusparse # cusparse.h
+  ];
+
+  cuda-native-redist = symlinkJoin {
+    name = "cuda-native-redist-${cudaVersion}";
+    paths = with cudaPackages; [
+      cuda_cudart # cuda_runtime.h
+      cuda_nvcc
+    ] ++ cuda-common-redist;
+  };
+
+  cuda-redist = symlinkJoin {
+    name = "cuda-redist-${cudaVersion}";
+    paths = cuda-common-redist;
+  };
+
+in
+buildPythonPackage rec {
+  pname = "mmcv";
+  version = "1.7.1";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.6";
+
+  src = fetchFromGitHub {
+    owner = "open-mmlab";
+    repo = pname;
+    rev = "refs/tags/v${version}";
+    hash = "sha256-b4MLBPNRCcPq1osUvqo71PCWVX7lOjAH+dXttd4ZapU";
+  };
+
+  preConfigure = ''
+    export MMCV_WITH_OPS=1
+  '' + lib.optionalString cudaSupport ''
+    export CC=${backendStdenv.cc}/bin/cc
+    export CXX=${backendStdenv.cc}/bin/c++
+    export TORCH_CUDA_ARCH_LIST="${lib.concatStringsSep ";" cudaCapabilities}"
+    export FORCE_CUDA=1
+  '';
+
+
+  postPatch = ''
+    substituteInPlace setup.py --replace "cpu_use = 4" "cpu_use = $NIX_BUILD_CORES"
+  '';
+
+  # reason for not using pytestCheckHook is similiar to what
+  # is already mentioned in pkgs/development/python-modules/typed-ast
+  # test_cnn test_ops really requires gpus to be useful.
+  # some of the tests take exceedingly long time.
+  # the rest of the tests are disabled due to sandbox env.
+  checkPhase = ''
+   pytest --ignore=tests/test_cnn \
+           --ignore=tests/test_ops \
+           --ignore=tests/test_fileclient.py \
+           --ignore=tests/test_load_model_zoo.py \
+           --ignore=tests/test_runner/test_checkpoint.py \
+           --ignore=tests/test_video/test_processing.py \
+           --ignore=tests/test_utils/test_hub.py \
+           --ignore=tests/test_video/test_reader.py
+  '';
+
+  nativeBuildInputs = [ ninja which ]
+    ++ lib.optionals cudaSupport [cuda-native-redist];
+
+
+  buildInputs = [ torch torchvision ] ++ lib.optionals cudaSupport [cuda-redist];
+
+  nativeCheckInputs = [ pytestCheckHook ];
+
+  checkInputs = [ lmdb onnx onnxruntime scipy pyturbojpeg tifffile ];
+
+  propagatedBuildInputs = [
+    opencv4
+    yapf
+    packaging
+    pillow
+    addict
+  ];
+
+  pythonImportsCheck = [
+    "mmcv"
+  ];
+
+  meta = with lib; {
+    description = "A Foundational Library for Computer Vision Research";
+    homepage = "https://github.com/open-mmlab/mmcv";
+    changelog = "https://github.com/open-mmlab/mmcv/releases/tag/v${version}";
+    license = with licenses; [ asl20 ];
+    maintainers = with maintainers; [ rxiao ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -6078,6 +6078,8 @@ self: super: with self; {
     enablePython = true;
   });
 
+  mmcv = callPackage ../development/python-modules/mmcv { };
+
   mmh3 = callPackage ../development/python-modules/mmh3 { };
 
   mmpython = callPackage ../development/python-modules/mmpython { };