use huggingface transformers official ut to test

.github/workflows/ci_pipeline.yaml

@@ -132,7 +132,9 @@ jobs:
         pip install mindspore
     - name: Test with pytest
       run: |
-        pytest -vs tests/transformers/models/${{ matrix.alpha }}*/test_modeling*
+        pip install transformers==4.51.2
+        git clone -b 4.51.2 https://gitee.com/mirrors/huggingface_transformers
+        python tests/run_test.py -vs huggingface_transformers/tests/models/${{ matrix.alpha }}*/test_modeling*
 
   kaggle-gpu-test:
     needs: pylint-check

.gitignore

@@ -164,4 +164,6 @@ aclinit.json
 xiyouji.txt
 *.safetensors
 *.jit
-flagged/
+flagged/
+
+huggingface_transformers/

mindnlp/__init__.py

@@ -17,7 +17,6 @@
 MindNLP library.
 """
 import os
-import sys
 import platform
 from packaging import version
 
@@ -51,6 +50,3 @@
 initialize_torch_proxy()
 setup_metadata_patch()
 setup_safetensors_patch()
-
-from . import core
-from . import transformers

mindnlp/core/__init__.py

@@ -36,7 +36,7 @@
 from ._C import *
 from ._dtype import *
 from ._tensor import Tensor, tensor, is_tensor, \
-    LongTensor, FloatTensor, BoolTensor, HalfTensor, BFloat16Tensor
+    LongTensor, FloatTensor, BoolTensor, HalfTensor, BFloat16Tensor, IntTensor
 from .types import device
 from ._C.size import Size
 from .types import device
@@ -46,9 +46,11 @@
 from ._bind import get_default_dtype, set_default_dtype
 
 from . import profiler, cuda, optim, amp, compiler, jit, version, __future__, overrides, \
-    return_types, linalg
+    return_types, linalg, fx
 
 from ._lowrank import svd_lowrank
+from .random import get_rng_state, initial_seed, manual_seed, seed, set_rng_state
+
 
 def _has_compatible_shallow_copy_type(tensor, other):
     """
@@ -70,4 +72,11 @@ def _has_compatible_shallow_copy_type(tensor, other):
         return False
 
     # Compatibility confirmed
-    return True
+    return True
+
+def compile(fn=None, *args, **kwargs):
+    def wrap_func(fn):
+        return fn
+    if fn is not None:
+        return wrap_func(fn)
+    return wrap_func

mindnlp/core/_tensor.py

@@ -9,11 +9,17 @@
 except:
     class StubTensor: pass
 
+try:
+    from mindspore._c_expression import TensorPy as Tensor_
+except:
+    from mindspore._c_expression import Tensor as Tensor_
+
 from . import ops, _dtype
 from ._dtype import dtype2np
 from ._bind import get_default_device, device_
 from .configs import use_pyboost, ON_A1
 from .storage import UntypedStorage
+from ._utils import _rebuild_tensor_v2
 
 DTYPE_ELEMENT_SIZE_MAP = {
     mindspore.float64: 8,
@@ -31,6 +37,11 @@ def __isinstancecheck__(self, instance):
             return False
         return instance.dtype == self.dtype
 
+class IntTensor(Tensor, metaclass=TypedTensorMeta):
+    dtype = _dtype.int
+    def __init__(self, data, device=None):
+        super().__init__(data, dtype=_dtype.int)
+
 class LongTensor(Tensor, metaclass=TypedTensorMeta):
     dtype = _dtype.long
     def __init__(self, data, device=None):
@@ -77,6 +88,22 @@ def is_tensor(x):
     return isinstance(x, Tensor)
 
 def enable_mindspore_patch():
+    def __reduce_ex__(self, protocol):
+        if isinstance(self, StubTensor):
+            data = Tensor_(self.stub_sync())
+        else:
+            data = Tensor_(self)
+        storage_offset = 0
+        size = data._shape
+        stride = data.stride()
+        requires_grad = False
+        args = (data, storage_offset, size, stride, requires_grad, None, None)
+        return (
+            _rebuild_from_type_v2, (_rebuild_tensor_v2, type(self), args, None))
+
+    Tensor.__reduce_ex__ = __reduce_ex__
+    StubTensor.__reduce_ex__ = __reduce_ex__
+
     def to_(self, *args, **kwargs):
         dtype_to = None
         if len(args) == 1:
@@ -260,3 +287,29 @@ def unfold(self, dimension, size, step):
 
     Tensor.unfold = unfold
     StubTensor.unfold = unfold
+
+    def new(self, data=None):
+        if data is None:
+            return Tensor([], dtype=self.dtype)
+        return Tensor(data, dtype=self.dtype)
+
+    Tensor.new = new
+    StubTensor.new = new
+
+    def view(self, *args):
+        if isinstance(args[0], (tuple, list)):
+            args = args[0]
+        return self.reshape(*args)
+
+    Tensor.view = view
+    StubTensor.view = view
+
+    def cpu(self):
+        return self
+
+    Tensor.cpu = cpu
+    StubTensor.cpu = cpu
+
+def _rebuild_from_type_v2(func, new_type, args, state):
+    ret = func(*args)
+    return ret

mindnlp/core/_utils.py

@@ -1,8 +1,16 @@
 import sys
 import traceback
+from functools import reduce
+import operator
 
-
+import numpy as np
 from mindnlp import core
+from .configs import SUPPORT_BF16
+
+if SUPPORT_BF16:
+    from mindspore.common.np_dtype import bfloat16 # pylint: disable=import-error
+else:
+    from ml_dtypes import bfloat16
 
 element_size_map = {
     core.float16: 2,
@@ -62,16 +70,47 @@ def _unflatten_dense_tensors(flat, tensors):
             offset += numel
     return outputs
 
-def _rebuild_tensor_v2(
-    storage,
-    storage_offset,
-    size,
-    stride,
-    requires_grad,
-    backward_hooks,
-    metadata=None,
-):
-    return core.Tensor(storage)
+def _rebuild_tensor_v2(storage, storage_offset, size, stride, requires_grad, backward_hooks=None, metadata=None):
+    '''Rebuilds a tensor based on the provided parameters.
+
+    Args:
+        storage (ndarray): The storage array from which the tensor is created.
+        storage_offset (int): The offset in the storage array from where the tensor data starts.
+        size (tuple): The size of the tensor.
+        stride (tuple or None): The stride of the tensor, or None if not applicable.
+        requires_grad (bool): Indicates if the tensor requires gradient computation.
+        backward_hooks (list): A list of backward hooks for the tensor.
+        metadata (Any, optional): Additional metadata associated with the tensor.
+
+    Returns:
+        None: This function does not have a return value.
+
+    Raises:
+        None: This function does not raise any exceptions.
+    '''
+    if size == ():
+        num_elemets = 1
+    else:
+        num_elemets = reduce(operator.mul, size)
+    array = storage[storage_offset: storage_offset + num_elemets]
+
+    if array.dtype == bfloat16 and not SUPPORT_BF16:
+        array = array.astype(np.float16)
+
+    if stride is not None and len(stride) > 1 and stride[0] == 1:
+        # stride = tuple((s * 4 for s in stride))
+        # # stride = tuple((s * 4 if s != 1 else s for s in stride))
+        # array = np.lib.stride_tricks.as_strided(array, size, stride)
+        order = "F"
+        array = array.reshape(size, order=order)
+    else:
+        order = "C"
+        array = array.reshape(size, order=order)
+
+    if isinstance(array, np.memmap):
+        array = array.copy()
+    param = core.from_numpy(array)
+    return param
 
 class KeyErrorMessage(str):
     r"""str subclass that returns itself in repr"""

mindnlp/core/distributed/fsdp/__init__.py

@@ -1,2 +1 @@
-class FullyShardedDataParallel:
-    pass
+from .fully_sharded_data_parallel import FullyShardedDataParallel

mindnlp/core/distributed/fsdp/fully_sharded_data_parallel.py

@@ -0,0 +1,2 @@
+class FullyShardedDataParallel:
+    pass

mindnlp/core/fx/__init__.py

@@ -9,3 +9,8 @@
 from .proxy import Proxy
 
 from . import _pytree
+
+class Graph: pass
+class GraphModule: pass
+class Node: pass
+class Tracer: pass