test_api.py

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

from __future__ import division

import unittest
import math
import numpy as np

from singa import singa_wrap as singa_api
from singa import tensor
from cuda_helper import gpu_dev, cpu_dev


def _np_bn_training(x, scale, bias, rm, rv, momentum=0.1, e=1e-5):
    channel = x.shape[1]
    np.testing.assert_array_almost_equal(scale.shape, (1, channel, 1, 1))
    np.testing.assert_array_almost_equal(bias.shape, (1, channel, 1, 1))
    np.testing.assert_array_almost_equal(rm.shape, (1, channel, 1, 1))
    np.testing.assert_array_almost_equal(rv.shape, (1, channel, 1, 1))

    batch_m = x.mean(axis=(0, 2, 3), keepdims=True)
    batch_v = x.var(axis=(0, 2, 3), keepdims=True)

    x_norm = (x - batch_m) / np.sqrt(batch_v + e)
    y_norm = x_norm * scale + bias

    # https://arxiv.org/pdf/1502.03167.pdf
    s = list(x.shape)
    s[1] = 1
    batch_v_unbiased = np.prod(s) * batch_v / (np.prod(s) - 1)

    rm = momentum * batch_m + (1 - momentum) * rm
    rv = momentum * batch_v_unbiased + (1 - momentum) * rv

    # https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnBatchNormalizationForwardTraining
    resultSaveInvVariance = 1 / np.sqrt(batch_v)
    return y_norm, rm, rv, batch_m, resultSaveInvVariance


def _np_bn_testing(x, scale, bias, rm, rv, momentum=0.1, e=1e-5):
    channel = x.shape[1]
    np.testing.assert_array_almost_equal(scale.shape, (1, channel, 1, 1))
    np.testing.assert_array_almost_equal(bias.shape, (1, channel, 1, 1))
    np.testing.assert_array_almost_equal(rm.shape, (1, channel, 1, 1))
    np.testing.assert_array_almost_equal(rv.shape, (1, channel, 1, 1))
    return scale * (x - rm) / np.sqrt(rv + e) + bias


def _cTensor_to_pyTensor(cTensor):
    new_t = tensor.Tensor()
    new_t.data = cTensor
    new_t.shape = tuple(new_t.data.shape())
    new_t.device = new_t.data.device()
    new_t.dtype = new_t.data.data_type()
    return new_t


def _ctensor_eq_ndarray(t1, np1):
    d = t1.device()
    t1.ToHost()
    if t1.data_type() == singa_api.kInt:
        np.testing.assert_array_almost_equal(t1.GetIntValue(t1.Size()),
                                             np1.flatten())
    elif t1.data_type() == singa_api.kFloat32:
        np.testing.assert_array_almost_equal(t1.GetFloatValue(t1.Size()),
                                             np1.flatten())

    if np1.dtype == np.float32:
        np.testing.assert_equal(t1.data_type(), singa_api.kFloat32)
    elif np1.dtype == np.int32:
        np.testing.assert_equal(t1.data_type(), singa_api.kInt)

    np.testing.assert_array_almost_equal(t1.shape(), np1.shape)
    t1.ToDevice(d)


def print_t(t1):
    d = t1.device()
    t1.ToHost()
    if t1.data_type() == singa_api.kInt:
        print(t1.GetIntValue(t1.Size()))
    elif t1.data_type() == singa_api.kFloat32:
        print(t1.GetFloatValue(t1.Size()))
    t1.ToDevice(d)


class TestAPI(unittest.TestCase):

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_batchnorm_training_gpu(self):
        dev = gpu_dev

        def _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.1):
            # np api
            (y_1, rm_1, rv_1, bm_1, bv_1) = _np_bn_training(x_0,
                                                            s_0,
                                                            b_0,
                                                            rm_0,
                                                            rv_0,
                                                            momentum=m_0)

            # singa api
            rm_t = tensor.Tensor(device=dev, data=rm_0)
            rv_t = tensor.Tensor(device=dev, data=rv_0)
            hndl = singa_api.CudnnBatchNormHandle(
                m_0,
                tensor.Tensor(device=dev, data=x_0).data)
            (y_2_c, bm_2_c, bv_2_c) = singa_api.GpuBatchNormForwardTraining(
                hndl,
                tensor.Tensor(device=dev, data=x_0).data,
                tensor.Tensor(device=dev, data=s_0).data,
                tensor.Tensor(device=dev, data=b_0).data, rm_t.data, rv_t.data)

            np.testing.assert_array_almost_equal(
                y_1, tensor.to_numpy(_cTensor_to_pyTensor(y_2_c)), decimal=4)
            np.testing.assert_array_almost_equal(
                bm_1, tensor.to_numpy(_cTensor_to_pyTensor(bm_2_c)))
            np.testing.assert_array_almost_equal(rm_1, tensor.to_numpy(rm_t))
            #print(bv_1)
            #print(tensor.to_numpy(_cTensor_to_pyTensor(bv_2_c)))
            np.testing.assert_array_almost_equal(
                bv_1, tensor.to_numpy(_cTensor_to_pyTensor(bv_2_c)), decimal=3)
            np.testing.assert_array_almost_equal(rv_1,
                                                 tensor.to_numpy(rv_t),
                                                 decimal=4)
            return

        x_0 = np.array([1, 1, 1, 1, 2, 2, 2, 2, 10, 10, 10, 10, 20, 20, 20, 20],
                       dtype=np.float32).reshape((2, 2, 2, 2))
        s_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        b_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rm_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rv_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.0)
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=1.0)
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.2)

        c = 10
        x_0 = np.random.random((10, c, 20, 20)).astype(np.float32)
        s_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        b_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rm_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rv_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.2)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_batchnorm_testing_gpu(self):
        dev = gpu_dev

        def _run_testing(x_0, s_0, b_0, rm_0, rv_0, m_0=0.1):
            # np api
            y_1 = _np_bn_testing(x_0, s_0, b_0, rm_0, rv_0, momentum=m_0)

            # singa api
            hndl = singa_api.CudnnBatchNormHandle(
                m_0,
                tensor.Tensor(device=dev, data=x_0).data)
            y_2_c = singa_api.GpuBatchNormForwardInference(
                hndl,
                tensor.Tensor(device=dev, data=x_0).data,
                tensor.Tensor(device=dev, data=s_0).data,
                tensor.Tensor(device=dev, data=b_0).data,
                tensor.Tensor(device=dev, data=rm_0).data,
                tensor.Tensor(device=dev, data=rv_0).data)
            #print(y_1)
            #print(tensor.to_numpy(_cTensor_to_pyTensor(y_2_c)))

            np.testing.assert_array_almost_equal(
                y_1, tensor.to_numpy(_cTensor_to_pyTensor(y_2_c)), decimal=3)
            return

        x_0 = np.array([1, 1, 1, 1, 2, 2, 2, 2, 10, 10, 10, 10, 20, 20, 20, 20],
                       dtype=np.float32).reshape((2, 2, 2, 2))
        s_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        b_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rm_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rv_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        _run_testing(x_0, s_0, b_0, rm_0, rv_0, m_0=1.0)
        c = 10
        x_0 = np.random.random((10, c, 20, 20)).astype(np.float32)
        s_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        b_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rm_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rv_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        _run_testing(x_0, s_0, b_0, rm_0, rv_0, m_0=1.0)

    def _softmax_api_helper(self, dev):

        def _run_test(dev, org_shape, axis, aft_shape):
            x_0 = np.random.random(org_shape).astype(np.float32)
            x_0 = x_0 + 1000
            x0 = tensor.Tensor(device=dev, data=x_0)

            # test with axis
            y0 = tensor._call_singa_func(singa_api.SoftMax, x0.data, axis)

            # test with numpy
            x_0 = x_0.reshape(aft_shape)
            x_0 = x_0 - np.max(x_0)
            y1 = np.divide(np.exp(x_0),
                           np.sum(np.exp(x_0), axis=1).reshape(x_0.shape[0],
                                                               1))  # 2d softmax
            y1 = y1.reshape(org_shape)

            np.testing.assert_array_almost_equal(tensor.to_numpy(y0), y1)

        _run_test(dev, [2, 2], 1, [2, 2])
        _run_test(dev, [2, 2], 0, [1, 4])
        _run_test(dev, [2, 2], -1, [2, 2])
        _run_test(dev, [2, 2], -2, [1, 4])
        _run_test(dev, [2, 2, 2], 2, [4, 2])
        _run_test(dev, [2, 2, 2], 1, [2, 4])
        _run_test(dev, [2, 2, 2], 0, [1, 8])
        _run_test(dev, [2, 2, 2], -1, [4, 2])
        _run_test(dev, [2, 2, 2], -2, [2, 4])
        _run_test(dev, [2, 2, 2], -3, [1, 8])
        _run_test(dev, [2, 2, 2, 2], 3, [8, 2])
        _run_test(dev, [2, 2, 2, 2], 2, [4, 4])
        _run_test(dev, [2, 2, 2, 2], 1, [2, 8])
        _run_test(dev, [2, 2, 2, 2], 0, [1, 16])
        _run_test(dev, [2, 2, 2, 2], -1, [8, 2])
        _run_test(dev, [2, 2, 2, 2], -2, [4, 4])
        _run_test(dev, [2, 2, 2, 2], -3, [2, 8])
        _run_test(dev, [2, 2, 2, 2], -4, [1, 16])

    def test_softmax_api_cpu(self):
        self._softmax_api_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_softmax_api_gpu(self):
        self._softmax_api_helper(gpu_dev)

    def _tensor_arithmetic_op_broadcast_helper(self, dev):

        def _run_test(dev, singa_op, np_op, s1, s2):
            x_0 = np.random.random(s1).astype(np.float32)
            y_0 = np.random.random(s2).astype(np.float32)
            x0 = tensor.Tensor(device=dev, data=x_0)
            y0 = tensor.Tensor(device=dev, data=y_0)

            z0 = tensor._call_singa_func(singa_op, x0.data, y0.data)
            z0.to_host()
            np.testing.assert_array_almost_equal(tensor.to_numpy(z0),
                                                 np_op(x_0, y_0))
            return

        for s_op, n_op in zip([
                singa_api.Pow,
                singa_api.__add__,
                singa_api.__div__,
                singa_api.__sub__,
                singa_api.__mul__,
        ], [np.power, np.add, np.divide, np.subtract, np.multiply]):
            _run_test(dev, s_op, n_op, [6], [1])
            _run_test(dev, s_op, n_op, [2, 3], [2, 3])
            _run_test(dev, s_op, n_op, [3, 2], [1])
            _run_test(dev, s_op, n_op, [3, 1, 2], [3, 1, 1])
            _run_test(dev, s_op, n_op, [2, 3, 4, 5], [5])
            _run_test(dev, s_op, n_op, [2, 3, 4, 5], [1, 1, 1])
            _run_test(dev, s_op, n_op, [2, 3, 4, 5], [1, 1, 1, 1])
            _run_test(dev, s_op, n_op, [2, 3, 4, 5], [4, 5])  # 45+2345=2345
            _run_test(dev, s_op, n_op, [3, 1, 2, 1], [3, 1, 2])
            _run_test(dev, s_op, n_op, [4, 5], [2, 3, 4, 5])  # 45+2345=2345
            _run_test(dev, s_op, n_op, [1, 4, 5], [2, 3, 1, 1])  # 145+2311=2345
            _run_test(dev, s_op, n_op, [3, 4, 5], [2, 1, 1, 1])  # 345+2111=2345

    def test_tensor_arithmetic_op_broadcast_cpu(self):
        self._tensor_arithmetic_op_broadcast_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_tensor_arithmetic_op_broadcast_gpu(self):
        self._tensor_arithmetic_op_broadcast_helper(gpu_dev)

    def _transpose_and_arithmetic_op_broadcast_helper(self, dev):

        def _test(s1, s2, axis1, axis2, s3, s_op, n_op, dev):
            x_0 = np.random.random(s1).astype(np.float32)
            y_0 = np.random.random(s2).astype(np.float32)

            x0 = tensor.Tensor(device=dev, data=x_0)
            y0 = tensor.Tensor(device=dev, data=y_0)

            x1 = x0.transpose(axis1)
            y1 = y0.transpose(axis2)

            z0 = tensor._call_singa_func(s_op, x1.data, y1.data)
            z0.to_host()

            np.testing.assert_array_almost_equal(
                tensor.to_numpy(z0),
                n_op(x_0.transpose(axis1), y_0.transpose(axis2)))
            np.testing.assert_array_almost_equal(z0.shape, s3)
            return

        for s_op, n_op in zip([
                singa_api.Pow,
                singa_api.__add__,
                singa_api.__div__,
                singa_api.__sub__,
                singa_api.__mul__,
        ], [np.power, np.add, np.divide, np.subtract, np.multiply]):
            s1 = [1, 5, 1, 3]
            s2 = [3, 1, 1, 4]
            axis1 = [3, 2, 1, 0]  # 3121
            axis2 = [1, 0, 2, 3]  # 1314
            s3 = [3, 3, 5, 4]
            _test(s1, s2, axis1, axis2, s3, s_op, n_op, dev)

            s1 = [1, 5, 1]
            s2 = [1, 3, 2]
            axis1 = [2, 1, 0]  # 151
            axis2 = [1, 0, 2]  # 312
            s3 = [3, 5, 2]
            _test(s1, s2, axis1, axis2, s3, s_op, n_op, dev)

            s1 = [5, 1]
            s2 = [1, 3]
            axis1 = [1, 0]  # 15
            axis2 = [1, 0]  # 31
            s3 = [3, 5]
            _test(s1, s2, axis1, axis2, s3, s_op, n_op, dev)

    def test_transpose_and_arithmetic_op_broadcast_cpu(self):
        self._transpose_and_arithmetic_op_broadcast_helper(cpu_dev)

    def _erf(self, dev=cpu_dev):
        np1 = np.random.random((2, 3)).astype(np.float32)

        x1 = tensor.from_numpy(np1)
        x1.to_device(dev)
        y1 = tensor.from_raw_tensor(singa_api.Erf(x1.data))

        # from scipy.special import erf
        # np.testing.assert_array_almost_equal(erf(np1), tensor.to_numpy(y1))

    def test_erf_cpu(self):
        self._erf(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_transpose_and_arithmetic_op_broadcast_gpu(self):
        self._transpose_and_arithmetic_op_broadcast_helper(gpu_dev)

    def test_batchnorm_training_dnnl(self):
        dev = cpu_dev

        def _np_bn_training(x, scale, bias, rm, rv, momentum=0.1, e=1e-5):
            channel = x.shape[1]
            np.testing.assert_array_almost_equal(scale.shape,
                                                 (1, channel, 1, 1))
            np.testing.assert_array_almost_equal(bias.shape, (1, channel, 1, 1))
            np.testing.assert_array_almost_equal(rm.shape, (1, channel, 1, 1))
            np.testing.assert_array_almost_equal(rv.shape, (1, channel, 1, 1))

            batch_m = x.mean(axis=(0, 2, 3), keepdims=True)
            batch_v = x.var(axis=(0, 2, 3), keepdims=True)

            x_norm = (x - batch_m) / np.sqrt(batch_v + e)
            y_norm = x_norm * scale + bias

            # https://arxiv.org/pdf/1502.03167.pdf
            s = list(x.shape)
            s[1] = 1
            batch_v_unbiased = np.prod(s) * batch_v / (np.prod(s) - 1)

            rm = momentum * batch_m + (1 - momentum) * rm
            rv = momentum * batch_v_unbiased + (1 - momentum) * rv

            # https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnBatchNormalizationForwardTraining
            # this value is useful for bwd computation
            resultSaveInvVariance = 1 / np.sqrt(batch_v)
            return y_norm, rm, rv, batch_m, resultSaveInvVariance

        def _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.1):
            # np api
            (y_1, rm_1, rv_1, bm_1, bv_1) = _np_bn_training(x_0,
                                                            s_0,
                                                            b_0,
                                                            rm_0,
                                                            rv_0,
                                                            momentum=m_0)

            # singa api
            hndl = singa_api.BatchNormHandle(
                m_0,
                tensor.Tensor(device=dev, data=x_0).data)
            (y_2_c, bm_2_c, bv_2_c) = singa_api.CpuBatchNormForwardTraining(
                hndl,
                tensor.Tensor(device=dev, data=x_0).data,
                tensor.Tensor(device=dev, data=s_0).data,
                tensor.Tensor(device=dev, data=b_0).data,
                tensor.Tensor(device=dev, data=rm_0).data,
                tensor.Tensor(device=dev, data=rv_0).data)

            np.testing.assert_array_almost_equal(
                y_1, tensor.to_numpy(_cTensor_to_pyTensor(y_2_c)), decimal=5)
            np.testing.assert_array_almost_equal(
                bm_1, tensor.to_numpy(_cTensor_to_pyTensor(bm_2_c)), decimal=5)
            #print(bv_1)
            #print(tensor.to_numpy(_cTensor_to_pyTensor(bv_2_c)))
            #np.testing.assert_array_almost_equal(
            #    bv_1, tensor.to_numpy(_cTensor_to_pyTensor(bv_2_c)), decimal=3)
            return

        x_0 = np.array([1, 1, 1, 1, 2, 2, 2, 2, 10, 10, 10, 10, 20, 20, 20, 20],
                       dtype=np.float32).reshape((2, 2, 2, 2))
        s_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        b_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rm_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rv_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=1.0)
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.0)
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.2)

        c = 10
        x_0 = np.random.random((10, c, 20, 20)).astype(np.float32)
        s_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        b_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rm_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rv_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        _run_training(x_0, s_0, b_0, rm_0, rv_0, m_0=0.2)

    def test_batchnorm_testing_dnnl(self):
        dev = cpu_dev

        def _np_bn_testing(x, scale, bias, rm, rv, momentum=0.1, e=1e-5):
            channel = x.shape[1]
            np.testing.assert_array_almost_equal(scale.shape,
                                                 (1, channel, 1, 1))
            np.testing.assert_array_almost_equal(bias.shape, (1, channel, 1, 1))
            np.testing.assert_array_almost_equal(rm.shape, (1, channel, 1, 1))
            np.testing.assert_array_almost_equal(rv.shape, (1, channel, 1, 1))
            return scale * (x - rm) / np.sqrt(rv + e) + bias

        def _run_testing(x_0, s_0, b_0, rm_0, rv_0, m_0=0.1):
            # np api
            y_1 = _np_bn_testing(x_0, s_0, b_0, rm_0, rv_0, momentum=m_0)

            # singa api
            hndl = singa_api.BatchNormHandle(
                m_0,
                tensor.Tensor(device=dev, data=x_0).data)
            y_2_c = singa_api.CpuBatchNormForwardInference(
                hndl,
                tensor.Tensor(device=dev, data=x_0).data,
                tensor.Tensor(device=dev, data=s_0).data,
                tensor.Tensor(device=dev, data=b_0).data,
                tensor.Tensor(device=dev, data=rm_0).data,
                tensor.Tensor(device=dev, data=rv_0).data)
            #print(y_1)
            #print(tensor.to_numpy(_cTensor_to_pyTensor(y_2_c)))

            np.testing.assert_array_almost_equal(
                y_1, tensor.to_numpy(_cTensor_to_pyTensor(y_2_c)), decimal=5)
            return

        x_0 = np.array([1, 1, 1, 1, 2, 2, 2, 2, 10, 10, 10, 10, 20, 20, 20, 20],
                       dtype=np.float32).reshape((2, 2, 2, 2))
        s_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        b_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rm_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        rv_0 = np.array([1, 10], dtype=np.float32).reshape((1, 2, 1, 1))
        _run_testing(x_0, s_0, b_0, rm_0, rv_0, m_0=1.0)
        c = 10
        x_0 = np.random.random((10, c, 20, 20)).astype(np.float32)
        s_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        b_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rm_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        rv_0 = np.random.random((1, c, 1, 1)).astype(np.float32)
        _run_testing(x_0, s_0, b_0, rm_0, rv_0, m_0=1.0)

    def test_batchnorm_backward_dnnl(self):
        dev = cpu_dev
        N = 1
        C = 3
        H = 2
        W = 2

        data_shape = [N, C, H, W]
        param_shape = [1, C, 1, 1]
        data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]

        x_0 = np.array(data, dtype=np.float32).reshape(data_shape)
        y_0 = np.array(data, dtype=np.float32).reshape(data_shape)
        dy_0 = np.array(data, dtype=np.float32).reshape(data_shape)
        scale_0 = np.array([1] * C, dtype=np.float32).reshape(param_shape)
        bias_0 = np.array([0] * C, dtype=np.float32).reshape(param_shape)

        mean_0 = x_0.mean(axis=(0, 2, 3), keepdims=True)
        var_0 = x_0.var(axis=(0, 2, 3), keepdims=True)

        hndl = singa_api.BatchNormHandle(
            0.1,
            tensor.Tensor(device=dev, data=x_0).data)
        (dx_2_c, _, _) = singa_api.CpuBatchNormBackwardx(
            hndl,
            tensor.Tensor(device=dev, data=y_0).data,
            tensor.Tensor(device=dev, data=dy_0).data,
            tensor.Tensor(device=dev, data=x_0).data,
            tensor.Tensor(device=dev, data=scale_0).data,
            tensor.Tensor(device=dev, data=bias_0).data,
            tensor.Tensor(device=dev, data=mean_0).data,
            tensor.Tensor(device=dev, data=var_0).data,
        )

        dx_truth = np.array([[[[-1.0769e-05, -3.5985e-06],
                               [3.5985e-06, 1.0769e-05]],
                              [[-1.0769e-05, -3.5985e-06],
                               [3.5985e-06, 1.0769e-05]],
                              [[-1.0769e-05, -3.5985e-06],
                               [3.5985e-06, 1.0769e-05]]]])
        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(dx_2_c)), dx_truth)

        return

    def test_softmax_api_dnnl_backend(self):
        dev = cpu_dev

        def _run_test(org_shape, axis, aft_shape):
            x_0 = np.random.random(org_shape).astype(np.float32)
            x_0 = x_0 + 1000
            x0 = tensor.Tensor(device=dev, data=x_0)

            # test with axis
            y0 = tensor._call_singa_func(singa_api.SoftMax, x0.data, axis)

            # test with numpy
            x_0 = x_0.reshape(aft_shape)
            x_0 = x_0 - np.max(x_0)
            y1 = np.divide(np.exp(x_0),
                           np.sum(np.exp(x_0), axis=1).reshape(x_0.shape[0],
                                                               1))  # 2d softmax
            y1 = y1.reshape(org_shape)

            np.testing.assert_array_almost_equal(tensor.to_numpy(y0), y1)

        _run_test([2, 2], 1, [2, 2])
        _run_test([2, 2], 0, [1, 4])
        _run_test([2, 2], -1, [2, 2])
        _run_test([2, 2], -2, [1, 4])

        _run_test([2, 2, 2], 2, [4, 2])
        _run_test([2, 2, 2], 1, [2, 4])
        _run_test([2, 2, 2], 0, [1, 8])
        _run_test([2, 2, 2], -1, [4, 2])
        _run_test([2, 2, 2], -2, [2, 4])
        _run_test([2, 2, 2], -3, [1, 8])

        _run_test([2, 2, 2, 2], 3, [8, 2])
        _run_test([2, 2, 2, 2], 2, [4, 4])
        _run_test([2, 2, 2, 2], 1, [2, 8])
        _run_test([2, 2, 2, 2], 0, [1, 16])
        _run_test([2, 2, 2, 2], -1, [8, 2])
        _run_test([2, 2, 2, 2], -2, [4, 4])
        _run_test([2, 2, 2, 2], -3, [2, 8])
        _run_test([2, 2, 2, 2], -4, [1, 16])

    def test_dnnl_pooling_max(self):
        dev = cpu_dev
        N = 1
        C = 3
        H = 2
        W = 2

        data_shape = [N, C, H, W]
        param_shape = [1, C, 1, 1]
        data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]

        x0 = np.array(data, dtype=np.float32).reshape(data_shape)
        x0_ct = tensor.Tensor(device=dev, data=x0).data

        dy0 = np.array([1, 2, 3], dtype=np.float32).reshape([1, 3, 1, 1])
        dy0_ct = tensor.Tensor(device=dev, data=dy0).data

        hndl = singa_api.PoolingHandle(x0_ct, [2, 2], [1, 1], [0, 0], True)

        y0_ct = singa_api.CpuPoolingForward(hndl, x0_ct)
        y1 = np.array([[[[4.]], [[8.]], [[12.]]]])
        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(y0_ct)), y1)

        dx0_ct = singa_api.CpuPoolingBackward(hndl, dy0_ct, x0_ct, y0_ct)
        dx1 = np.array([[[[0., 0.], [0., 1.]], [[0., 0.], [0., 2.]],
                         [[0., 0.], [0., 3.]]]])
        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(dx0_ct)), dx1)

    def test_dnnl_pooling_avg(self):
        dev = cpu_dev
        N = 1
        C = 3
        H = 2
        W = 2

        data_shape = [N, C, H, W]
        param_shape = [1, C, 1, 1]
        data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]

        x0 = np.array(data, dtype=np.float32).reshape(data_shape)
        x0_ct = tensor.Tensor(device=dev, data=x0).data

        dy0 = np.array([1, 2, 3], dtype=np.float32).reshape([1, 3, 1, 1])
        dy0_ct = tensor.Tensor(device=dev, data=dy0).data

        hndl = singa_api.PoolingHandle(x0_ct, [2, 2], [1, 1], [0, 0], False)

        y0_ct = singa_api.CpuPoolingForward(hndl, x0_ct)

        y1 = np.array([[[[2.5000]], [[6.5000]], [[10.5000]]]])
        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(y0_ct)), y1)
        dx0_ct = singa_api.CpuPoolingBackward(hndl, dy0_ct, x0_ct, y0_ct)
        dx1 = np.array([[[[0.2500, 0.2500], [0.2500, 0.2500]],
                         [[0.5000, 0.5000], [0.5000, 0.5000]],
                         [[0.7500, 0.7500], [0.7500, 0.7500]]]])
        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(dx0_ct)), dx1)

    def _concat_helper(self, dev):
        np1 = np.random.random([5, 6, 7, 8]).astype(np.float32)
        np2 = np.random.random([5, 6, 7, 1]).astype(np.float32)
        np3 = np.concatenate((np1, np2), axis=3)

        t1 = tensor.Tensor(device=dev, data=np1)
        t2 = tensor.Tensor(device=dev, data=np2)

        ctensors = singa_api.VecTensor()
        ctensors.append(t1.data)
        ctensors.append(t2.data)

        t3_ct = singa_api.ConcatOn(ctensors, 3)

        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(t3_ct)), np3)

    def test_concat_cpu(self):
        self._concat_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_concat_gpu(self):
        self._concat_helper(gpu_dev)

    def _ceil_helper(self, dev):

        np1 = np.random.random([5, 6, 7, 8]).astype(np.float32)

        np1 = np.random.random([5, 6, 7, 8]).astype(np.float32)
        np1 = np1 * 10
        np2 = np.ceil(np1)

        t1 = tensor.Tensor(device=dev, data=np1)

        t2_ct = singa_api.Ceil(t1.data)

        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(t2_ct)), np2)

    def test_ceil_cpu(self):
        self._ceil_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_ceil_gpu(self):
        self._ceil_helper(gpu_dev)

    def _floor_helper(self, dev):

        np1 = np.random.random([5, 6, 7, 8]).astype(np.float32)

        np1 = np.random.random([5, 6, 7, 8]).astype(np.float32)
        np1 = np1 * 10
        np2 = np.floor(np1)

        t1 = tensor.Tensor(device=dev, data=np1)

        t2_ct = singa_api.Floor(t1.data)

        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(t2_ct)), np2)

    def test_floor_cpu(self):
        self._floor_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_floor_gpu(self):
        self._floor_helper(gpu_dev)

    def _as_type_helper(self, dev):

        np1 = np.random.random([3]).astype(np.float32)
        np1 = np1 * 10 - 5
        np2 = np1.astype(np.int32)
        np3 = np2.astype(np.float32)

        t1 = tensor.Tensor(device=dev, data=np1)

        t1 = tensor.Tensor(device=dev, data=np1)

        t1_ct = t1.data

        self.assertEqual(t1_ct.data_type(), singa_api.kFloat32)

        t1_ct = t1_ct.AsType(singa_api.kInt)

        self.assertEqual(t1_ct.data_type(), singa_api.kInt)

        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(t1_ct)), np2)

        t1_ct = t1_ct.AsType(singa_api.kFloat32)

        self.assertEqual(t1_ct.data_type(), singa_api.kFloat32)

        np.testing.assert_array_almost_equal(
            tensor.to_numpy(_cTensor_to_pyTensor(t1_ct)), np3)

    def test_as_type_cpu(self):
        self._as_type_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_as_type_gpu(self):
        self._as_type_helper(gpu_dev)

    def _as_type2_helper(self, dev):
        shape1 = [1, 2, 3, 4]
        shape2 = [4, 3, 2, 1]
        np_int = np.random.randint(0, 10, shape1).astype(np.int32)
        np_flt = np_int.astype(np.float32)

        t1 = singa_api.Tensor(shape1, dev, singa_api.kInt)
        t1.CopyIntDataFromHostPtr(np_int.flatten())
        _ctensor_eq_ndarray(t1, np_int)

        t1 = singa_api.Reshape(t1, shape2)
        t2 = t1.AsType(singa_api.kFloat32)
        _ctensor_eq_ndarray(t2, np_flt.reshape(shape2))

        t3 = t2.AsType(singa_api.kInt)
        _ctensor_eq_ndarray(t3, np_int.reshape(shape2))

        t1 = singa_api.Reshape(t1, shape1)
        t4 = t1.AsType(singa_api.kFloat32)
        _ctensor_eq_ndarray(t4, np_flt.reshape(shape1))

    def test_as_type2_cpu(self):
        self._as_type2_helper(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_as_type2_gpu(self):
        self._as_type2_helper(gpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_rnn_relu(self):
        self._rnn_helper(0)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_rnn_tanh(self):
        self._rnn_helper(1)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_rnn_lstm(self):
        self._rnn_helper(2)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_rnn_gru(self):
        self._rnn_helper(3)

    def _rnn_helper(self, mode):
        dev = gpu_dev

        hidden_size = 7
        seq_length = 5
        batch_size = 6
        feature_size = 3
        directions = 2
        num_layers = 2

        x = tensor.Tensor(shape=(seq_length, batch_size, feature_size),
                          device=dev).gaussian(0, 1)
        hx = tensor.Tensor(shape=(num_layers * directions, batch_size,
                                  hidden_size),
                           device=dev).gaussian(0, 1)
        cx = tensor.Tensor(shape=(num_layers * directions, batch_size,
                                  hidden_size),
                           device=dev).gaussian(0, 1)

        rnn_handle = singa_api.CudnnRNNHandle(x.data,
                                              hidden_size,
                                              mode,
                                              num_layers=num_layers,
                                              dropout=0.1,
                                              bidirectional=1)

        w = tensor.Tensor(shape=(rnn_handle.weights_size,),
                          device=dev).gaussian(0, 1)
        # print("weights size is ", rnn_handle.weights_size)

        (y, hy, cy) = singa_api.GpuRNNForwardTraining(x.data, hx.data, cx.data,
                                                      w.data, rnn_handle)
        self.assertEqual(y.shape(),
                         (seq_length, batch_size, directions * hidden_size))
        self.assertEqual(hy.shape(), hx.shape)
        self.assertEqual(cy.shape(), cx.shape)

        (y2, hy2,
         cy2) = singa_api.GpuRNNForwardInference(x.data, hx.data, cx.data,
                                                 w.data, rnn_handle)
        self.assertEqual(y2.shape(),
                         (seq_length, batch_size, directions * hidden_size))
        self.assertEqual(hy2.shape(), hx.shape)
        self.assertEqual(cy2.shape(), cx.shape)

        dy = tensor.Tensor(shape=(seq_length, batch_size,
                                  directions * hidden_size),
                           device=dev).gaussian(0, 1)
        dhy = tensor.Tensor(shape=(num_layers * directions, batch_size,
                                   hidden_size),
                            device=dev).gaussian(0, 1)
        dcy = tensor.Tensor(shape=(num_layers * directions, batch_size,
                                   hidden_size),
                            device=dev).gaussian(0, 1)

        (dx, dhx, dcx) = singa_api.GpuRNNBackwardx(y, dy.data, dhy.data,
                                                   dcy.data, w.data, hx.data,
                                                   cx.data, rnn_handle)
        self.assertEqual(dx.shape(), (seq_length, batch_size, feature_size))
        self.assertEqual(dhx.shape(), hx.shape)
        self.assertEqual(dcx.shape(), cx.shape)

        dW = singa_api.GpuRNNBackwardW(x.data, hx.data, y, rnn_handle)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_rnn_with_seq_lengths(self):
        dev = gpu_dev

        # params
        hidden_size = 7
        seq_length = 5
        batch_size = 6
        feature_size = 3
        directions = 2
        num_layers = 2

        # shapes
        x_s = (seq_length, batch_size, feature_size)
        y_s = (seq_length, batch_size, hidden_size)
        states_s = (num_layers * directions, batch_size, hidden_size)

        # tensors
        x = tensor.Tensor(x_s, dev).gaussian(0, 1)
        y = tensor.Tensor(y_s, dev).gaussian(0, 1)
        dy = tensor.Tensor(y_s, dev).gaussian(0, 1)
        dhy = tensor.Tensor(states_s, dev).gaussian(0, 1)
        dcy = tensor.Tensor(states_s, dev).gaussian(0, 1)
        hx = tensor.Tensor(states_s, dev).gaussian(0, 1)
        cx = tensor.Tensor(states_s, dev).gaussian(0, 1)

        # handle
        rnn_handle = singa_api.CudnnRNNHandle(x.data, hidden_size, 2)
        w = tensor.Tensor((rnn_handle.weights_size,), dev).gaussian(0, 1)

        # seq lengths
        seq_lengths = tensor.from_numpy(np.array([seq_length] * batch_size))

        # operations
        (dx, dhx, dcx) = singa_api.GpuRNNBackwardxEx(y.data, dy.data, dhy.data,
                                                     dcy.data, w.data, hx.data,
                                                     cx.data, seq_lengths.data,
                                                     rnn_handle)


    def test_round_cpu(self):
        self._round(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_round_gpu(self):
        self._round(gpu_dev)

    def _round(self, dev=gpu_dev):
        x = tensor.Tensor(shape=(3,4,5), device=dev).gaussian(0, 1)
        y = tensor._call_singa_func(singa_api.Round, x.data)
        np.testing.assert_array_almost_equal(np.round(tensor.to_numpy(x)),
                                             tensor.to_numpy(y))

    def test_round_even_cpu(self):
        self._round_even(cpu_dev)

    @unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
    def test_round_even_gpu(self):
        self._round_even(gpu_dev)

    def _round_even(self, dev=gpu_dev):
        q=np.array([0.1, 0.5, 0.9, 1.2, 1.5,
                    1.8, 2.3, 2.5, 2.7, -1.1,
                    -1.5, -1.9, -2.2, -2.5, -2.8]).astype(np.float32)
        ans = np.array([0., 0., 1., 1., 2.,
                    2., 2., 2., 3., -1.,
                    -2., -2., -2., -2., -3.]).astype(np.float32)

        x = tensor.Tensor(shape=q.shape, device=dev)
        x.copy_from_numpy(q)
        y = tensor._call_singa_func(singa_api.RoundE, x.data)
        np.testing.assert_array_almost_equal(ans, tensor.to_numpy(y))


if __name__ == '__main__':
    unittest.main()