Merge pull request #2863 from alibaba/feature/sync_290

MNN:Sync Sync Internal 2.9.0

CMakeLists.txt

@@ -58,9 +58,18 @@ option(MNN_ENABLE_COVERAGE "Build with coverage enable" OFF)
 option(MNN_BUILD_PROTOBUFFER "Build with protobuffer in MNN" ON)
 option(MNN_BUILD_OPENCV "Build OpenCV api in MNN." OFF)
 option(MNN_BUILD_LLM "Build llm library based MNN." OFF)
+option(MNN_BUILD_DIFFUSION "Build diffusion demo based MNN." OFF)
 option(MNN_INTERNAL "Build with MNN internal features, such as model authentication, metrics logging" OFF)
 option(MNN_JNI "Build MNN Jni for java to use" OFF)
 
+IF (OHOS)
+  include($ENV{NODE_PATH}/@ali/tcpkg/tcpkg.cmake)
+  export_headers(DIR ${CMAKE_SOURCE_DIR}/include/MNN)
+  IF (MNN_BUILD_OPENCV)
+    export_headers(DIR ${CMAKE_SOURCE_DIR}/tools/cv/include/cv)
+  ENDIF()
+ENDIF()
+
 IF (NOT DEFINED MNN_USE_SPARSE_COMPUTE)
    set(MNN_USE_SPARSE_COMPUTE ON)
 ENDIF()
@@ -263,12 +272,12 @@ endif()
 option(MNN_USE_CPP11 "Enable MNN use c++11" ON)
 if (NOT MSVC)
     if(MNN_CUDA AND MNN_SUPPORT_TRANSFORMER_FUSE)
-        set(CMAKE_CXX_STANDARD 17)    
+        set(CMAKE_CXX_STANDARD 17)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
     elseif(MNN_USE_CPP11)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")     
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
     else()
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
@@ -493,7 +502,7 @@ endif()
 IF(MNN_COREML)
     add_definitions(-DMNN_COREML_ENABLED=1)
     add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/source/backend/coreml/)
-    
+
     IF(MNN_SEP_BUILD)
       list(APPEND MNN_DEPS MNNCoreML)
       list(APPEND MNN_EXTRA_DEPENDS MNNCoreML)
@@ -631,7 +640,7 @@ ENDIF()
 
 IF(MNN_BUILD_LLM)
     # add_definitions(-DMNN_BUILD_LLM)
-    include(${CMAKE_CURRENT_LIST_DIR}/llm/CMakeLists.txt)
+    include(${CMAKE_CURRENT_LIST_DIR}/transformers/llm/engine/CMakeLists.txt)
 ENDIF()
 
 # NPU
@@ -755,6 +764,9 @@ list(REMOVE_ITEM MNN_TARGETS MNN)
 IF(MNN_BUILD_DEMO)
 include(${CMAKE_CURRENT_LIST_DIR}/demo/exec/CMakeLists.txt)
 ENDIF()
+IF(MNN_BUILD_DIFFUSION AND MNN_BUILD_OPENCV AND MNN_IMGCODECS)
+include(${CMAKE_CURRENT_LIST_DIR}/transformers/diffusion/CMakeLists.txt)
+ENDIF()
 IF(MNN_BUILD_TOOLS)
 include(${CMAKE_CURRENT_LIST_DIR}/tools/cpp/CMakeLists.txt)
 ENDIF()

benchmark/benchmark.cpp

@@ -127,7 +127,7 @@ std::vector<float> doBench(Model& model, int loop, int warmup = 10, int forward
 
     auto modelBuffer      = revertor->getBuffer();
     const auto bufferSize = revertor->getBufferSize();
-    auto net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromBuffer(modelBuffer, bufferSize));
+    auto net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromBuffer(modelBuffer, bufferSize), MNN::Interpreter::destroy);
     revertor.reset();
     net->setSessionMode(MNN::Interpreter::Session_Release);
     MNN::ScheduleConfig config;

docs/compile/cmake.md

@@ -80,6 +80,7 @@ MNN使用CMake构建项目，CMake中的宏定义列表如下：
 | MNN_OPENCV_BENCH     | 构建MNN的OpenCV功能是否开启性能benchmark，默认为`OFF` |
 | MNN_VULKAN_IMAGE     | 构建MNN的Vulkan后端时采用Image内存模式，以便支持FP16和部分移动端上GPU的加速，默认为`ON` |
 | MNN_LOW_MEMORY       | 是否支持低内存模式，支持低内存模式使用权值量化模型并设置`low_memory`则会使用计算时反量化，默认为`OFF` |
-| MNN_SUPPORT_RENDER       | 是否支持图形渲染相关算子实现，默认为 `OFF` |
+| MNN_SUPPORT_RENDER   | 是否支持图形渲染相关算子实现，默认为 `OFF` |
 | MNN_SUPPORT_TRANSFORMER_FUSE | 是否支持Fuse Transformer相关OP实现，默认为 `OFF` |
 | MNN_BUILD_LLM        | 是否构建基于MNN的llm库和demo，默认为`OFF` |
+| MNN_BUILD_DIFFUSION  | 是否构建基于MNN的diffusion demo，需要打开MNN_BUILD_OPENCV和MNN_IMGCODECS宏使用 默认为`OFF` |

docs/compile/tools.md

@@ -60,6 +60,7 @@
   - `winogradExample.out` winograd示例
   - `fuseTest` 测试 GPU 自定义算子的功能，目前仅支持 Vulkan Buffer 模式
   - `GpuInterTest.out` 测试 GPU 内存输入的功能，目前仅支持 OpenCL Buffer 模式与 OpenGL texture 模式，编译时许打开 MNN_OPENCL 与 MNN_OPENGL
+  - `LoRA` 将LorA权重添加到模型权重中
 ## Benchmark工具
 - 相关编译选项
   - `MNN_BUILD_BENCHMARK` 是否编译Benchmark工具

express/Executor.cpp

@@ -27,6 +27,7 @@ namespace Express {
 
 void Executor::setGlobalExecutorConfig(MNNForwardType type, const BackendConfig& config, int numberThread) {
     std::lock_guard<std::mutex> _l(mMutex);
+
     if(type == MNN_FORWARD_AUTO) {
         ScheduleConfig sConfig;
         sConfig.type = type;
@@ -41,10 +42,12 @@ void Executor::setGlobalExecutorConfig(MNNForwardType type, const BackendConfig&
             info.numThread = 4;
         }
         mAttr->firstType = std::make_pair(type, info.numThread);
-
-        info.user = (BackendConfig*)&config;
-        std::shared_ptr<Runtime> bn(creator->onCreate(info));
-        mRuntimes[mAttr->firstType] = bn;
+        auto firstIter = mRuntimes.find(mAttr->firstType);
+        if (firstIter == mRuntimes.end()) {
+            info.user = (BackendConfig*)&config;
+            std::shared_ptr<Runtime> bn(creator->onCreate(info));
+            mRuntimes[mAttr->firstType] = bn;
+        }
     } else {
         auto creator = MNNGetExtraRuntimeCreator(type);
         if (nullptr == creator) {
@@ -56,11 +59,14 @@ void Executor::setGlobalExecutorConfig(MNNForwardType type, const BackendConfig&
         Backend::Info info;
         info.type = type;
         mAttr->firstType = std::make_pair(type, numberThread);
-        info.mode = Backend::Info::DIRECT;
-        info.numThread = numberThread;
-        info.user = (BackendConfig*)&config;
-        std::shared_ptr<Runtime> bn(creator->onCreate(info));
-        mRuntimes[mAttr->firstType] = bn;
+        auto firstIter = mRuntimes.find(mAttr->firstType);
+        if (firstIter == mRuntimes.end()) {
+            info.mode = Backend::Info::DIRECT;
+            info.numThread = numberThread;
+            info.user = (BackendConfig*)&config;
+            std::shared_ptr<Runtime> bn(creator->onCreate(info));
+            mRuntimes[mAttr->firstType] = bn;
+        }
     }
     _refreshRuntime();
 }
@@ -155,6 +161,10 @@ std::shared_ptr<Executor> Executor::newExecutor(MNNForwardType type,
                                                 const BackendConfig& config,
                                                 int numberThread) {
     auto creator = MNNGetExtraRuntimeCreator(type);
+    if(nullptr == creator) {
+        MNN_ERROR("Don't support %d\n", type);
+        return nullptr;
+    }
     Backend::Info info;
     info.type = type;
     info.numThread = numberThread;

express/module/StaticModule.cpp

@@ -98,6 +98,21 @@ static std::vector<std::shared_ptr<BufferStorage>> preRearrangeWeights( // NOLIN
                 }
                 break;
             }
+            case MNN::OpType_Attention: {
+                exe.reset(backend->onCreate({}, {}, op));
+                if (exe.get() == nullptr) {
+                    exe.reset(backupBackend->onCreate({}, {}, op));
+                }
+                if (nullptr == exe) {
+                    break;
+                }
+                // The exe can't clone
+                if (!exe->onClone(nullptr, op, nullptr)) {
+                    exe = nullptr;
+                    break;
+                }
+                break;
+            }
             default: {
                 break;
             }

include/MNN/MNNDefine.h

@@ -68,7 +68,7 @@ MNN_ERROR("Check failed: %s ==> %s\n", #success, #log); \
 #define STR_IMP(x) #x
 #define STR(x) STR_IMP(x)
 #define MNN_VERSION_MAJOR 2
-#define MNN_VERSION_MINOR 8
-#define MNN_VERSION_PATCH 4
+#define MNN_VERSION_MINOR 9
+#define MNN_VERSION_PATCH 0
 #define MNN_VERSION STR(MNN_VERSION_MAJOR) "." STR(MNN_VERSION_MINOR) "." STR(MNN_VERSION_PATCH)
 #endif /* MNNDefine_h */

project/harmony/build_64.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+cmake ../../../ \
+-DCMAKE_TOOLCHAIN_FILE=$HARMONY_HOME/native/build/cmake/ohos.toolchain.cmake \
+-DCMAKE_BUILD_TYPE=Release \
+-DOHOS_ARCH="arm64-v8a" \
+-DOHOS_STL=c++_static \
+-DMNN_USE_LOGCAT=false \
+-DMNN_BUILD_BENCHMARK=ON \
+-DMNN_USE_SSE=OFF \
+-DMNN_SUPPORT_BF16=OFF \
+-DMNN_BUILD_TEST=ON \
+-DOHOS_PLATFORM_LEVEL=9  \
+-DMNN_BUILD_FOR_ANDROID_COMMAND=true \
+-DNATIVE_LIBRARY_OUTPUT=. -DNATIVE_INCLUDE_OUTPUT=. $1 $2 $3
+
+make -j4

project/harmony/updateTest.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+DIR=yanxing
+
+make -j16
+hdc file send ./libMNN.so /data/local/tmp/$DIR/libMNN.so
+hdc file send ./libMNN_Express.so /data/local/tmp/$DIR/libMNN_Express.so
+hdc file send ./MNNV2Basic.out /data/local/tmp/$DIR/MNNV2Basic.out
+hdc file send ./ModuleBasic.out /data/local/tmp/$DIR/ModuleBasic.out
+# hdc shell "cd /data/local/tmp/$DIR && rm -r output"
+# hdc shell "cd /data/local/tmp/$DIR && mkdir output"
+hdc file send ./unitTest.out /data/local/tmp/$DIR/unitTest.out
+hdc file send ./testModel.out /data/local/tmp/$DIR/testModel.out
+hdc file send ./testModelWithDescribe.out /data/local/tmp/$DIR/testModelWithDescribe.out
+hdc file send ./backendTest.out /data/local/tmp/$DIR/backendTest.out
+hdc file send ./timeProfile.out /data/local/tmp/$DIR/timeProfile.out
+hdc file send ./run_test.out /data/local/tmp/$DIR/run_test.out

project/ios/MNN.xcodeproj/project.pbxproj

@@ -174,7 +174,6 @@
 		489D7A7B2550FDC800AD896A /* MetalUnary.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A2A2550FDC800AD896A /* MetalUnary.hpp */; };
 		489D7A7D2550FDC900AD896A /* MetalConvolution.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A2C2550FDC800AD896A /* MetalConvolution.mm */; };
 		489D7A7E2550FDC900AD896A /* MNNMetalContext.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A2D2550FDC800AD896A /* MNNMetalContext.mm */; };
-		489D7A7F2550FDC900AD896A /* MetalReLU.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A2E2550FDC800AD896A /* MetalReLU.hpp */; };
 		489D7A802550FDC900AD896A /* MetalEltwise.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A2F2550FDC800AD896A /* MetalEltwise.hpp */; };
 		489D7A812550FDC900AD896A /* MetalPooling.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A302550FDC800AD896A /* MetalPooling.hpp */; };
 		489D7A822550FDC900AD896A /* MetalPReLU.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A312550FDC800AD896A /* MetalPReLU.hpp */; };
@@ -184,7 +183,6 @@
 		489D7A8A2550FDC900AD896A /* MetalConvolutionDepthwise.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A392550FDC800AD896A /* MetalConvolutionDepthwise.mm */; };
 		489D7A8B2550FDC900AD896A /* MetalConvolutionWinograd.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A3A2550FDC800AD896A /* MetalConvolutionWinograd.hpp */; };
 		489D7A8C2550FDC900AD896A /* MetalDeconvolution.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A3B2550FDC800AD896A /* MetalDeconvolution.mm */; };
-		489D7A8D2550FDC900AD896A /* MetalReLU.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A3C2550FDC800AD896A /* MetalReLU.mm */; };
 		489D7A8E2550FDC900AD896A /* MetalPooling.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A3D2550FDC800AD896A /* MetalPooling.mm */; };
 		489D7A902550FDC900AD896A /* MetalConvolution.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 489D7A3F2550FDC800AD896A /* MetalConvolution.hpp */; };
 		489D7A912550FDC900AD896A /* MetalScale.mm in Sources */ = {isa = PBXBuildFile; fileRef = 489D7A402550FDC800AD896A /* MetalScale.mm */; };
@@ -746,6 +744,7 @@
 		9558333D29B0947300488807 /* MNNGelu.S in Sources */ = {isa = PBXBuildFile; fileRef = 9558333C29B0947300488807 /* MNNGelu.S */; };
 		9558334729B09A2300488807 /* MNNGelu.S in Sources */ = {isa = PBXBuildFile; fileRef = 9558334629B09A2300488807 /* MNNGelu.S */; };
 		9558334B29B09A7B00488807 /* MNNGeluFP16.S in Sources */ = {isa = PBXBuildFile; fileRef = 9558334A29B09A7B00488807 /* MNNGeluFP16.S */; settings = {COMPILER_FLAGS = "-march=armv8.2-a+fp16"; }; };
+		9560EAD62BDE426A00C8D0B6 /* GeometryLayernorm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 9560EAD52BDE426A00C8D0B6 /* GeometryLayernorm.cpp */; };
 		956F52E12AB2D692004B13D9 /* ImageProcessUtils.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 956F52E02AB2D692004B13D9 /* ImageProcessUtils.cpp */; };
 		956F52E32AB2D6A1004B13D9 /* ImageProcessUtils.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 956F52E22AB2D6A1004B13D9 /* ImageProcessUtils.hpp */; };
 		958375352A496E5C007C0A3E /* MNNLineDepthWiseInt8AddBiasScale_ARMV82_Unit3X3.S in Sources */ = {isa = PBXBuildFile; fileRef = 958375342A496E5C007C0A3E /* MNNLineDepthWiseInt8AddBiasScale_ARMV82_Unit3X3.S */; };
@@ -1005,7 +1004,6 @@
 		489D7A2A2550FDC800AD896A /* MetalUnary.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalUnary.hpp; sourceTree = "<group>"; };
 		489D7A2C2550FDC800AD896A /* MetalConvolution.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MetalConvolution.mm; sourceTree = "<group>"; };
 		489D7A2D2550FDC800AD896A /* MNNMetalContext.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MNNMetalContext.mm; sourceTree = "<group>"; };
-		489D7A2E2550FDC800AD896A /* MetalReLU.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalReLU.hpp; sourceTree = "<group>"; };
 		489D7A2F2550FDC800AD896A /* MetalEltwise.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalEltwise.hpp; sourceTree = "<group>"; };
 		489D7A302550FDC800AD896A /* MetalPooling.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalPooling.hpp; sourceTree = "<group>"; };
 		489D7A312550FDC800AD896A /* MetalPReLU.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalPReLU.hpp; sourceTree = "<group>"; };
@@ -1015,7 +1013,6 @@
 		489D7A392550FDC800AD896A /* MetalConvolutionDepthwise.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MetalConvolutionDepthwise.mm; sourceTree = "<group>"; };
 		489D7A3A2550FDC800AD896A /* MetalConvolutionWinograd.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalConvolutionWinograd.hpp; sourceTree = "<group>"; };
 		489D7A3B2550FDC800AD896A /* MetalDeconvolution.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MetalDeconvolution.mm; sourceTree = "<group>"; };
-		489D7A3C2550FDC800AD896A /* MetalReLU.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MetalReLU.mm; sourceTree = "<group>"; };
 		489D7A3D2550FDC800AD896A /* MetalPooling.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MetalPooling.mm; sourceTree = "<group>"; };
 		489D7A3F2550FDC800AD896A /* MetalConvolution.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MetalConvolution.hpp; sourceTree = "<group>"; };
 		489D7A402550FDC800AD896A /* MetalScale.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MetalScale.mm; sourceTree = "<group>"; };
@@ -1587,6 +1584,7 @@
 		9558333C29B0947300488807 /* MNNGelu.S */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = MNNGelu.S; sourceTree = "<group>"; };
 		9558334629B09A2300488807 /* MNNGelu.S */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = MNNGelu.S; sourceTree = "<group>"; };
 		9558334A29B09A7B00488807 /* MNNGeluFP16.S */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; name = MNNGeluFP16.S; path = ../../../arm82/asm/arm64/MNNGeluFP16.S; sourceTree = "<group>"; };
+		9560EAD52BDE426A00C8D0B6 /* GeometryLayernorm.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = GeometryLayernorm.cpp; sourceTree = "<group>"; };
 		956F52E02AB2D692004B13D9 /* ImageProcessUtils.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ImageProcessUtils.cpp; sourceTree = "<group>"; };
 		956F52E22AB2D6A1004B13D9 /* ImageProcessUtils.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = ImageProcessUtils.hpp; sourceTree = "<group>"; };
 		958375342A496E5C007C0A3E /* MNNLineDepthWiseInt8AddBiasScale_ARMV82_Unit3X3.S */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; name = MNNLineDepthWiseInt8AddBiasScale_ARMV82_Unit3X3.S; path = arm/arm64/MNNLineDepthWiseInt8AddBiasScale_ARMV82_Unit3X3.S; sourceTree = "<group>"; };
@@ -1842,6 +1840,7 @@
 				4819FB2624C139690050BD09 /* GeometryLSTM.cpp */,
 				4819FB2424C139680050BD09 /* GeometryPoolGrad.cpp */,
 				4819FB2A24C139690050BD09 /* GeometryReduce.cpp */,
+				9560EAD52BDE426A00C8D0B6 /* GeometryLayernorm.cpp */,
 				489404DD24A2FC2B001E456C /* GeometryReverseSequence.cpp */,
 				48FD0349246AA40300456AF5 /* GeometryConvert.cpp */,
 				48FD12BD2466A88D009E9102 /* GeometryConv2DBackPropFilter.cpp */,
@@ -2150,7 +2149,6 @@
 				489D7A2A2550FDC800AD896A /* MetalUnary.hpp */,
 				489D7A2C2550FDC800AD896A /* MetalConvolution.mm */,
 				489D7A2D2550FDC800AD896A /* MNNMetalContext.mm */,
-				489D7A2E2550FDC800AD896A /* MetalReLU.hpp */,
 				489D7A2F2550FDC800AD896A /* MetalEltwise.hpp */,
 				489D7A302550FDC800AD896A /* MetalPooling.hpp */,
 				489D7A312550FDC800AD896A /* MetalPReLU.hpp */,
@@ -2160,7 +2158,6 @@
 				489D7A392550FDC800AD896A /* MetalConvolutionDepthwise.mm */,
 				489D7A3A2550FDC800AD896A /* MetalConvolutionWinograd.hpp */,
 				489D7A3B2550FDC800AD896A /* MetalDeconvolution.mm */,
-				489D7A3C2550FDC800AD896A /* MetalReLU.mm */,
 				489D7A3D2550FDC800AD896A /* MetalPooling.mm */,
 				489D7A3F2550FDC800AD896A /* MetalConvolution.hpp */,
 				489D7A402550FDC800AD896A /* MetalScale.mm */,
@@ -3070,7 +3067,6 @@
 				EBECA39924643D320062C7A3 /* Arm82Relu.hpp in Headers */,
 				4838EA7C2611BFE20027232C /* CPUGridSample.hpp in Headers */,
 				92FF03A523AA0B5A00AC97F6 /* DeconvolutionWithStride.hpp in Headers */,
-				489D7A7F2550FDC900AD896A /* MetalReLU.hpp in Headers */,
 				92FF03D123AA0B5A00AC97F6 /* CPUTopKV2.hpp in Headers */,
 				92FF033F23AA0B5A00AC97F6 /* CPUArgMax.hpp in Headers */,
 				92FF034C23AA0B5A00AC97F6 /* CPUSetDiff1D.hpp in Headers */,
@@ -3602,7 +3598,6 @@
 				486E1A9924F5078D00C16006 /* CPURandomUniform.cpp in Sources */,
 				92FF02C823AA0B5A00AC97F6 /* MNNNV21ToBGRUnit.S in Sources */,
 				92FF045C23AA0B7100AC97F6 /* ShapeBroadcastTo.cpp in Sources */,
-				489D7A8D2550FDC900AD896A /* MetalReLU.mm in Sources */,
 				48747D49245D9D24000B9709 /* RuntimeFactory.cpp in Sources */,
 				92FF02AE23AA0B5A00AC97F6 /* CPUProposal.cpp in Sources */,
 				92FF042723AA0B7100AC97F6 /* ShapeMatMul.cpp in Sources */,
@@ -3627,6 +3622,7 @@
 				92FF025923AA0B5A00AC97F6 /* CPUPoolInt8.cpp in Sources */,
 				92FF045B23AA0B7100AC97F6 /* ShapeShape.cpp in Sources */,
 				CECF8C87299CAD9400D3875B /* sds.c in Sources */,
+				9560EAD62BDE426A00C8D0B6 /* GeometryLayernorm.cpp in Sources */,
 				4D6D7FD72656896D00F80814 /* SparseConvolutionTiledExecutor.cpp in Sources */,
 				CECF8C82299CAD9400D3875B /* log_api.cpp in Sources */,
 				92FF03A823AA0B5A00AC97F6 /* WinogradOptFunction.cpp in Sources */,
@@ -4131,7 +4127,7 @@
 				CODE_SIGN_STYLE = Automatic;
 				DEAD_CODE_STRIPPING = YES;
 				DEFINES_MODULE = YES;
-				DEVELOPMENT_TEAM = 6G7464HHUS;
+				DEVELOPMENT_TEAM = Q48UX93J22;
 				DYLIB_COMPATIBILITY_VERSION = 1;
 				DYLIB_CURRENT_VERSION = 1;
 				DYLIB_INSTALL_NAME_BASE = "@rpath";
@@ -4218,7 +4214,7 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				ASSETCATALOG_COMPILER_LAUNCHIMAGE_NAME = LaunchImage;
 				CODE_SIGN_STYLE = Automatic;
-				DEVELOPMENT_TEAM = 6G7464HHUS;
+				DEVELOPMENT_TEAM = Q48UX93J22;
 				GCC_ENABLE_CPP_EXCEPTIONS = NO;
 				GCC_ENABLE_CPP_RTTI = NO;
 				HEADER_SEARCH_PATHS = (