[pull] master from mindspore-ai:master

cmake/external_libs/opencl.cmake

@@ -1,4 +1,16 @@
-if(ENABLE_GITEE)
+if(ENABLE_GITEE_EULER)
+    set(GIT_REPOSITORY "git@gitee.com:src-openeuler/opencl-clhpp.git")
+    set(GIT_TAG "7347fa1bb52ebee9f3d6c44ff65ef3c4253cab79")
+    set(MD5 "d41d8cd98f00b204e9800998ecf8427e")
+
+    __download_pkg_with_git(OpenCL-CLHPP ${GIT_REPOSITORY} ${GIT_TAG} ${MD5})
+    set(OPENCL_CLHPP_SRC "${TOP_DIR}/mindspore/lite/build/_deps/opencl-clhpp-src")
+    execute_process(COMMAND tar -xf ${OPENCL_CLHPP_SRC}/v2.0.12.tar.gz --strip-components 1 -C ${OPENCL_CLHPP_SRC})
+
+    set(OPENCL_HEADER_SRC "${TOP_DIR}/mindspore/lite/build/_deps/opencl-headers-src")
+    execute_process(COMMAND mkdir -p ${OPENCL_HEADER_SRC})
+    execute_process(COMMAND tar -xf ${OPENCL_CLHPP_SRC}/v2020.12.18.tar.gz --strip-components 1 -C ${OPENCL_HEADER_SRC})
+elseif(ENABLE_GITEE)
     set(REQ_URL "https://gitee.com/mirrors/OpenCL-Headers/repository/archive/v2020.12.18.tar.gz")
     set(MD5 "9a34e50198ac54b795bf3822bb6e429a")
     __download_pkg(OpenCL-Headers ${REQ_URL} ${MD5})

docs/api/api_python/mindspore.communication.rst

@@ -31,7 +31,6 @@ mindspore.communication
 
     - **TypeError** – 参数 `backend_name` 不是字符串。
     - **RuntimeError** – 1）硬件设备类型无效；2）后台服务无效；3）分布式计算初始化失败；4）未设置环境变量 `RANK_ID` 或 `MINDSPORE_HCCL_CONFIG_PATH` 的情况下初始化HCCL服务。
-    - **ValueError** – 环境变量 `RANK_ID` 设置成非数字。
 
 .. py:class:: mindspore.communication.release()
 
@@ -221,4 +220,4 @@ mindspore.communication
 
 .. py:data:: mindspore.communication.NCCL_WORLD_COMM_GROUP
 
-    "nccl_world_group"字符串，指的是由NCCL创建的默认通信组。
+    "nccl_world_group"字符串，指的是由NCCL创建的默认通信组。

docs/api/api_python/mindspore.numpy.rst

@@ -59,6 +59,9 @@ Array Generation
     mindspore.numpy.ones
     mindspore.numpy.ones_like
     mindspore.numpy.pad
+    mindspore.numpy.rand
+    mindspore.numpy.randint
+    mindspore.numpy.randn
     mindspore.numpy.trace
     mindspore.numpy.tri
     mindspore.numpy.tril

docs/api/api_python/mindspore.ops.rst

@@ -334,6 +334,7 @@ functional算子是经过初始化后的Primitive，可以直接作为函数使
     :nosignatures:
     :template: classtemplate.rst
 
+    mindspore.ops.arange
     mindspore.ops.batch_dot
     mindspore.ops.clip_by_global_norm
     mindspore.ops.clip_by_value

docs/api/api_python/mindspore/mindspore.COOTensor.rst

@@ -5,6 +5,16 @@ mindspore.COOTensor
 
     用来表示某一张量在给定索引上非零元素的集合，其中索引(indices)指示了每一个非零元素的位置。
 
+    对一个稠密Tensor `dense` 来说，它对应的COOTensor(indices, values, dense_shape)，满足 `dense[indices[i]] = values[i]` 。
+
+    如果 `indices` 是[[0, 1], [1, 2]]， `values` 是[1, 2]， `dense_shape` 是(3, 4)，那么它对应的稠密Tensor如下：
+
+    .. code-block::
+
+        [[0, 1, 0, 0],
+         [0, 0, 2, 0],
+         [0, 0, 0, 0]]
+
     .. note::
         这是一个实验特性，在未来可能会发生API的变化。
 

docs/api/api_python/mindspore/mindspore.Tensor.rst

@@ -225,6 +225,20 @@ mindspore.Tensor
 
         维度与输入张量的相同的Tensor。输出张量的维度必须遵守广播规则。广播规则指输出张量的维度需要扩展为输入张量的维度，如果目标张量的维度大于输入张量的维度，则不满足广播规则。
 
+    .. py:method:: expand_dims(axis):
+
+        沿指定轴扩展Tensor维度。
+
+        **参数：**
+            - **axis** (int) - 扩展维度指定的轴。
+
+        **返回：**
+            Tensor, 指定轴上扩展的维度为1。
+
+        **异常：**
+            - **TypeError** - axis不是int类型。
+            - **ValueError** - axis的取值不在[-self.ndim - 1, self.ndim + 1)。
+
     .. py:method:: fill(value)
 
         用标量值填充数组。
@@ -343,6 +357,22 @@ mindspore.Tensor
 
         返回一个Tensor元素的长度（以字节为单位）。
 
+    .. py:method:: masked_fill(mask, value)
+
+        根据mask矩阵，将值为True对应位置填充为value值。mask矩阵的shape必须与原Tensor相同。
+
+        **参数：**
+            - **mask** (Tensor[bool]) - mask矩阵，值为bool类型的Tensor。
+            - **value** (Union[int, float]) - 填充值，可以为int或float类型。
+
+        **返回：**
+            Tensor, shape和dtype与原Tensor相同。
+
+        **异常：**
+            - **TypeError** - mask不是Tensor。
+            - **TypeError** - mask不是bool类型的Tensor。
+            - **TypeError** - value不是int或float类型。
+
     .. py:method:: max(axis=None, keepdims=False, initial=None, where=True)
 
         返回Tensor的最大值或轴方向上的最大值。
@@ -394,6 +424,26 @@ mindspore.Tensor
 
         - **TypeError** - 参数具有前面未指定的类型。
 
+    .. py:method:: narrow(axis, start, length):
+
+        沿指定轴，指定起始位置获取指定长度的Tensor。
+
+        **参数：**
+            - **axis** (int) - 指定的轴。
+            - **start** (int) - 指定的起始位置。
+            - **length** (int) - 指定的长度。
+
+        **返回：**
+            Tensor。
+
+        **异常：**
+            - **TypeError** - axis不是int类型。
+            - **TypeError** - start不是int类型。
+            - **TypeError** - length不是int类型。
+            - **ValueError** - axis超出Tensor的维度范围。
+            - **ValueError** - start超出Tensor的维度范围。
+            - **ValueError** - start+length超出Tensor的维度范围。
+
     .. py:method:: nbytes
         :property:
 

docs/api/api_python/ops/mindspore.ops.func_arange.rst

@@ -0,0 +1,22 @@
+mindspore.ops.arange
+===============================
+
+.. py:function:: mindspore.ops.arange(start=0, stop=None, step=1, rtype=None)
+
+    根据给定的范围返回指定均匀间隔的数据。
+
+    **参数：**
+
+    - **start** (Union[int, float]) - 指定范围的起始值，范围包含该值。类型为int或float。
+    - **stop** (Union[int, float]) - 指定范围的结束值，范围不包含该值。类型为int或float。
+    - **step** (Union[int, float]) - 指定取值的间隔。类型为int或float。
+    - **rtype** (Union[mindspore.dtype，str]) - 指定返回数据的类型，如果不指定，则会根据 `start` 、 `stop` 、 `step` 的值推断类型。
+
+    **返回：**
+
+    Tensor，值是均匀间隔的数据，类型为给定或推断的结果。
+
+    **异常：**
+
+    - **TypeError** -  `start` 、 `stop` 、 `step` 的类型不是int或float。
+    - **ValueError** - `start` 的值大于等于 `stop` 。

mindspore/ccsrc/backend/common/optimizer/helper.cc

@@ -827,6 +827,14 @@ AnfNodePtr GetAnfNodeByVar(const EquivPtr &equiv, const VarPtr &var_node) {
   return res;
 }
 
+int64_t GetGetitemIndex(const AnfNodePtr &getitem) {
+  if (!getitem->isa<CNode>() || IsPrimitive(getitem, prim::kPrimTupleGetItem)) {
+    MS_LOG(EXCEPTION) << "Expect TupleGetItem, but got " << getitem->DebugString();
+  }
+  auto vnode = GetValueNode(getitem->cast<CNodePtr>()->input(kInputNodeOutputIndexInTupleGetItem));
+  return GetValue<int64_t>(vnode);
+}
+
 bool CompareTupleGetitem(const AnfNodePtr &n1, const AnfNodePtr &n2) {
   MS_EXCEPTION_IF_NULL(n1);
   MS_EXCEPTION_IF_NULL(n2);

mindspore/ccsrc/backend/common/optimizer/helper.h

@@ -208,6 +208,9 @@ bool IsSameNode(const EquivPtr &equiv1, const EquivPtr &equiv2, const VarPtr &va
 // Get anf_node from equiv by var_node
 AnfNodePtr GetAnfNodeByVar(const EquivPtr &equiv, const VarPtr &var_node);
 
+// Get tuple getitem's index
+int64_t GetGetitemIndex(const AnfNodePtr &getitem);
+
 // Compare tuple getitem's index, return bool[n1's index < n2's index]
 bool CompareTupleGetitem(const AnfNodePtr &n1, const AnfNodePtr &n2);
 

mindspore/ccsrc/backend/common/pass/common_subexpression_elimination.cc

@@ -16,14 +16,20 @@
 #include "backend/common/pass/common_subexpression_elimination.h"
 
 #include <memory>
+#include <vector>
 #include "runtime/device/kernel_info.h"
+#include "base/core_ops.h"
 #include "utils/flags.h"
 #include "utils/ms_context.h"
+#include "include/common/utils/utils.h"
 #include "include/common/utils/anfalgo.h"
+#include "backend/common/optimizer/helper.h"
 
 namespace mindspore {
 namespace opt {
 namespace {
+using KernelWithIndex = std::pair<AnfNodePtr, int64_t>;
+
 bool CheckIgnoreCase(const AnfNodePtr &node) {
   MS_EXCEPTION_IF_NULL(node);
   if (common::AnfAlgo::GetCNodeName(node) != kTransDataOpName) {
@@ -42,6 +48,38 @@ bool CheckIgnoreCase(const AnfNodePtr &node) {
   }
   return need_ignore;
 }
+
+void EliminateDuplicatedTupleGetItem(const FuncGraphPtr &graph, const FuncGraphManagerPtr &manager) {
+  MS_EXCEPTION_IF_NULL(graph);
+  MS_EXCEPTION_IF_NULL(manager);
+
+  // key: (getitem_input, getitem_index), value: getitem_list
+  std::map<KernelWithIndex, std::vector<AnfNodePtr>> getitem_dup_map;
+  const auto &node_list = TopoSort(graph->get_return());
+  for (auto &node : node_list) {
+    if (!node->isa<CNode>() || !IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) {
+      continue;
+    }
+    auto getitem_cnode = node->cast<CNodePtr>();
+    KernelWithIndex input_with_index{getitem_cnode->input(kRealInputNodeIndexInTupleGetItem),
+                                     GetGetitemIndex(getitem_cnode)};
+    if (getitem_dup_map.count(input_with_index) == 0) {
+      getitem_dup_map.emplace(input_with_index, std::vector<AnfNodePtr>{node});
+    } else {
+      getitem_dup_map[input_with_index].push_back(node);
+    }
+  }
+
+  // remove duplicated
+  for (auto &item : getitem_dup_map) {
+    auto &getitem_list = item.second;
+    if (getitem_list.size() > 1) {
+      auto first_getitem = getitem_list[0];
+      std::for_each(getitem_list.begin() + 1, getitem_list.end(),
+                    [first_getitem, manager](const AnfNodePtr &getitem) { manager->Replace(getitem, first_getitem); });
+    }
+  }
+}
 }  // namespace
 
 bool BackendCSE::CheckEqualKernelBuildInfo(const AnfNodePtr &main, const AnfNodePtr &node) const {
@@ -141,7 +179,11 @@ bool BackendCSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) co
 
 bool BackendCSE::Cse(const FuncGraphPtr graph, const FuncGraphManagerPtr manager) const {
   MS_EXCEPTION_IF_NULL(manager);
-  return BuildOrderGroupAndDoReplaceForOneGraph(graph, manager);
+  auto ret = BuildOrderGroupAndDoReplaceForOneGraph(graph, manager);
+  if (ret) {
+    EliminateDuplicatedTupleGetItem(graph, manager);
+  }
+  return ret;
 }
 
 bool CommonSubexpressionElimination::Run(const FuncGraphPtr &func_graph) {

mindspore/ccsrc/common/mem_reuse/mem_dynamic_allocator.cc

@@ -75,11 +75,13 @@ std::vector<DeviceMemPtr> DynamicMemPoolBestFit::AllocContinuousTensorMem(size_t
   MS_EXCEPTION_IF_NULL(mem_block);
   const auto &iter = mem_block->block_all_mem_buf_map_.find(device_addr);
   if (iter == mem_block->block_all_mem_buf_map_.end()) {
+    DumpDynamicMemPoolDebugInfo();
     MS_LOG(EXCEPTION) << "Can't find the device address[" << device_addr << "].";
   }
   auto mem_buf = iter->second;
   MS_EXCEPTION_IF_NULL(mem_buf);
   if (mem_buf->size_ < total_size) {
+    DumpDynamicMemPoolDebugInfo();
     MS_LOG(EXCEPTION) << "The size of membuf is less than total_size.";
   }
   auto rest_size = mem_buf->size_ - total_size;
@@ -118,6 +120,7 @@ DeviceMemPtr DynamicMemPoolBestFit::FindIdleMemBuf(size_t size, bool from_persis
     auto mem_buf = iter->second;
     MS_EXCEPTION_IF_NULL(mem_buf);
     if (mem_buf->status_ != kMemBufIdle) {
+      DumpDynamicMemPoolDebugInfo();
       MS_LOG(EXCEPTION) << "Find the mem_buf is not idle, alloc_size[" << size << "] mem_buf_size[" << mem_buf->size_
                         << "] mem_buf_address[" << mem_buf->device_addr_ << "].";
     }
@@ -264,6 +267,7 @@ void DynamicMemPoolBestFit::SplitMemBuf(size_t size, const DynamicMemBufPtr &mem
   MS_EXCEPTION_IF_NULL(mem_block);
   // Divide new memory buf
   if (mem_buf->size_ < size) {
+    DumpDynamicMemPoolDebugInfo();
     MS_LOG(EXCEPTION) << "The size of membuf is less than size.";
   }
   size_t newbuf_size = mem_buf->size_ - size;
@@ -326,15 +330,18 @@ void DynamicMemPoolBestFit::CombineMemBuf(const DynamicMemBlockPtr &mem_block, c
   MS_EXCEPTION_IF_NULL(device_addr);
   const auto &iter = mem_block->block_all_mem_buf_map_.find(device_addr);
   if (iter == mem_block->block_all_mem_buf_map_.end()) {
+    DumpDynamicMemPoolDebugInfo();
     MS_LOG(EXCEPTION) << "Can't find the device address[" << device_addr << "].";
   }
   auto mem_buf = iter->second;
   MS_EXCEPTION_IF_NULL(mem_buf);
   if (mem_buf->status_ != kMemBufUsed) {
+    DumpDynamicMemPoolDebugInfo();
     MS_LOG(EXCEPTION) << "Find the mem_buf is not used, mem_buf_address[" << mem_buf->device_addr_ << "].";
   }
   mem_buf->status_ = kMemBufIdle;
   if (mem_mng->mps_.total_used_mem_size_ < mem_buf->size_) {
+    DumpDynamicMemPoolDebugInfo();
     MS_LOG(EXCEPTION) << "The total used mem size is less than the size of membuf.";
   }
   mem_mng->mps_.total_used_mem_size_ -= mem_buf->size_;
@@ -426,7 +433,7 @@ void DynamicMemPoolBestFit::DumpDynamicMemPoolStateInfo() {
           << " idle size:" << idle_size;
     }
     // Dump all the memory buf info
-    MS_LOG(WARNING) << mem_type << " pool info: block size " << mem_mng->unit_size_ << ", block counts "
+    MS_LOG(WARNING) << mem_type << " pool info: block unit size " << mem_mng->unit_size_ << ", block counts "
                     << mem_mng->mem_block_list_.size() << buf.str() << ". Total allocated mem "
                     << mem_mng->mps_.total_mem_size_ << ", peak used mem " << mem_mng->mps_.used_mem_peak_size_
                     << ", in used mem " << mem_mng->mps_.total_used_mem_size_ << ", total idle mem "
@@ -459,9 +466,8 @@ void DynamicMemPoolBestFit::DumpDynamicMemPoolDebugInfo() {
         } else {
           total_used_mem += mem_buf->size_;
         }
-        auto user_name = (mem_buf->status_ == kMemBufUsed) ? ("] name[" + mem_buf->allocator_name_ + "].") : "].";
         MS_LOG(INFO) << "  MemBuf info: address[" << mem_buf->device_addr_ << "] size[" << mem_buf->size_ << "] status["
-                     << kBufStatusString.at(mem_buf->status_) << user_name;
+                     << kBufStatusString.at(mem_buf->status_) << "] name[" << mem_buf->allocator_name_ << "].";
       }
     }
     // Dump all the idle memory buf info.

mindspore/ccsrc/plugin/device/ascend/hal/device/ascend_memory_manager.cc

@@ -86,12 +86,12 @@ uint8_t *AscendMemoryManager::MallocStaticMem(size_t size, bool communication_me
 #endif
 
   uint8_t *alloc_address = reinterpret_cast<uint8_t *>(AscendMemoryPool::GetInstance().AllocTensorMem(align_size));
-  if (alloc_address == nullptr) {
-    MS_LOG(EXCEPTION) << "Fail to alloc memory, size: " << align_size
-                      << ", memory statistics:" << AscendMemAdapter::GetInstance().DevMemStatistics();
+  if (alloc_address != nullptr) {
+    // create protect area [kMemAlignSize -- data -- kMemAlignSize] for communication node memory
+    return communication_mem ? alloc_address + kMemAlignSize : alloc_address;
   }
-  // create protect area [kMemAlignSize -- data -- kMemAlignSize] for communication node memory
-  return communication_mem ? alloc_address + kMemAlignSize : alloc_address;
+  MS_LOG(EXCEPTION) << "Fail to alloc memory, size: " << align_size
+                    << ", memory statistics:" << AscendMemAdapter::GetInstance().DevMemStatistics();
 }
 
 uint8_t *AscendMemoryManager::MallocDynamicMem(size_t size, bool communication_mem) {
@@ -124,11 +124,11 @@ void AscendMemoryManager::MallocSomasDynamicMem(const session::KernelGraph &grap
 uint8_t *AscendMemoryManager::MallocCommunicationMemFromMemPool(size_t size) {
   auto align_size = GetCommunicationAlignSize(size);
   uint8_t *base_ptr = reinterpret_cast<uint8_t *>(AscendMemoryPool::GetInstance().AllocTensorMem(align_size));
-  if (base_ptr == nullptr) {
-    MS_LOG(EXCEPTION) << "Fail to alloc memory, size: " << align_size
-                      << ", memory statistics:" << AscendMemAdapter::GetInstance().DevMemStatistics();
+  if (base_ptr != nullptr) {
+    return base_ptr + kMemAlignSize;
   }
-  return base_ptr + kMemAlignSize;
+  MS_LOG(EXCEPTION) << "Fail to alloc memory, size: " << align_size
+                    << ", memory statistics:" << AscendMemAdapter::GetInstance().DevMemStatistics();
 }
 
 bool AscendMemoryManager::MallocContinuousMemFromMemPool(const DeviceAddressPtrList &addr_list, size_t total_size,