From 060f098762af3af239bafcd72dc7111e90468b3b Mon Sep 17 00:00:00 2001 From: Andrzej Warzynski Date: Wed, 17 Sep 2025 20:12:15 +0000 Subject: [PATCH 1/3] [mlir][vector] Refine Vector to LLVM lowering options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a follow-up to #144307, where we removed `vector.matrix_multiply` and `vector.flat_transpose` from the Vector dialect. This PR: * Updates comments that were missed in the previous change. * Renames relevant `-convert-vector-to-llvm=` options: - `vector-contract-lowering=matmul` → `vector-contract-lowering=llvm` - `vector-transpose-lowering=flat_transpose` → `vector-transpose-lowering=llvm` These new names better reflect the actual transformation target — LLVM intrinsics — rather than the now-removed abstract operations. --- mlir/include/mlir/Conversion/Passes.td | 8 +-- .../Vector/Transforms/VectorTransformsBase.td | 17 +++---- .../VectorToLLVM/ConvertVectorToLLVM.cpp | 49 +++++++++++-------- .../VectorToLLVM/ConvertVectorToLLVMPass.cpp | 4 +- .../pass-option-serialization.mlir | 6 +-- ...tract-to-matrix-intrinsics-transforms.mlir | 2 +- ...nspose-to-matrix-intrinsics-transform.mlir | 2 +- 7 files changed, 47 insertions(+), 41 deletions(-) diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index 1a37d057776e2..aca0963478e63 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -1489,8 +1489,8 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> { VectorContractLoweringAttr.summary, [{::llvm::cl::values( clEnumValN(::mlir::vector::VectorContractLowering::Dot, "dot", "Progressively lower to finer grained `vector.contract` and dot-products. (default)"), - clEnumValN(::mlir::vector::VectorContractLowering::Matmul, "matmul", - "Lower to `vector.matrix_multiply`, maps 1-1 to LLVM matrix intrinsics."), + clEnumValN(::mlir::vector::VectorContractLowering::LLVM, "llvm", + "Lower directly to `llvm.intr.matrix.multiply`."), clEnumValN(::mlir::vector::VectorContractLowering::OuterProduct, "outerproduct", "Lower to `vector.outerproduct`."), clEnumValN(::mlir::vector::VectorContractLowering::ParallelArith, "parallelarith", @@ -1502,8 +1502,8 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> { VectorTransposeLoweringAttr.summary, [{::llvm::cl::values( clEnumValN(::mlir::vector::VectorTransposeLowering::EltWise, "eltwise", "Lower transpose into element-wise extract and inserts (default)"), - clEnumValN(::mlir::vector::VectorTransposeLowering::Flat, "flat", - "Lower 2-D transpose to `vector.flat_transpose`, maps 1-1 to LLVM matrix intrinsics"), + clEnumValN(::mlir::vector::VectorTransposeLowering::LLVM, "llvm", + "Lower 2-D transpose directly to `llvm.intr.matrix.transpose`"), clEnumValN(::mlir::vector::VectorTransposeLowering::Shuffle1D, "shuffle1d", "Lower 2-D transpose to `vector.shuffle` on 1-D vector."), clEnumValN(::mlir::vector::VectorTransposeLowering::Shuffle16x16, "shuffle16x16", diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td index ef0951ab1d166..cbba44ae4dc8a 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td @@ -14,10 +14,9 @@ include "mlir/IR/EnumAttr.td" // Lower transpose into element-wise extract and inserts. def VectorTransposeLowering_Elementwise: I32EnumAttrCase<"EltWise", 0, "eltwise">; -// Lower 2-D transpose to `vector.flat_transpose`, maps 1-1 to LLVM matrix -// intrinsics. -def VectorTransposeLowering_FlatTranspose: - I32EnumAttrCase<"Flat", 1, "flat_transpose">; +// Lower directly to LLVM matrix intrinsics. +def VectorTransposeLowering_LLVM: + I32EnumAttrCase<"LLVM", 1, "llvm">; // Lower 2-D transpose to `vector.shuffle` on 1-D vector. def VectorTransposeLowering_Shuffle1D: I32EnumAttrCase<"Shuffle1D", 2, "shuffle_1d">; @@ -27,7 +26,7 @@ def VectorTransposeLowering_Shuffle16x16: def VectorTransposeLoweringAttr : I32EnumAttr< "VectorTransposeLowering", "control the lowering of `vector.transpose` operations.", - [VectorTransposeLowering_Elementwise, VectorTransposeLowering_FlatTranspose, + [VectorTransposeLowering_Elementwise, VectorTransposeLowering_LLVM, VectorTransposeLowering_Shuffle1D, VectorTransposeLowering_Shuffle16x16]> { let cppNamespace = "::mlir::vector"; } @@ -48,9 +47,9 @@ def VectorMultiReductionLoweringAttr: I32EnumAttr< // Progressively lower to finer grained `vector.contract` and dot-products. def VectorContractLowering_Dot: I32EnumAttrCase<"Dot", 0, "dot">; -// Lower to `vector.matrix_multiply`, maps 1-1 to LLVM matrix intrinsics. -def VectorContractLowering_Matmul: - I32EnumAttrCase<"Matmul", 1, "matmulintrinsics">; +// Lower directly to LLVM intrinsics. +def VectorContractLowering_LLVM: + I32EnumAttrCase<"LLVM", 1, "llvm">; // Lower to `vector.outerproduct`. def VectorContractLowering_OuterProduct: I32EnumAttrCase<"OuterProduct", 2, "outerproduct">; @@ -61,7 +60,7 @@ def VectorContractLowering_ParallelArith: def VectorContractLoweringAttr: I32EnumAttr< "VectorContractLowering", "control the lowering of `vector.contract` operations.", - [VectorContractLowering_Dot, VectorContractLowering_Matmul, + [VectorContractLowering_Dot, VectorContractLowering_LLVM, VectorContractLowering_OuterProduct, VectorContractLowering_ParallelArith]> { let cppNamespace = "::mlir::vector"; } diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp index e7266740894b1..4d60450e7fde8 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp @@ -1987,17 +1987,13 @@ struct VectorScalableStepOpLowering /// %e = add %c, %d /// ``` /// `vector.matrix_multiply` later lowers to `llvm.matrix.multiply`. -// -/// This only kicks in when vectorContractLowering is set to Matmul and -/// the vector.contract op is a row-major matrix multiply. class ContractionOpToMatmulOpLowering : public vector::MaskableOpRewritePattern { public: using MaskableOpRewritePattern::MaskableOpRewritePattern; - ContractionOpToMatmulOpLowering( - vector::VectorContractLowering vectorContractLowering, - MLIRContext *context, PatternBenefit benefit = 100) + ContractionOpToMatmulOpLowering(MLIRContext *context, + PatternBenefit benefit = 100) : MaskableOpRewritePattern(context, benefit) {} FailureOr @@ -2005,23 +2001,22 @@ class ContractionOpToMatmulOpLowering PatternRewriter &rewriter) const override; }; -/// Progressively lower a `vector.contract %a, %b, %c` with row-major matmul -/// semantics to: +/// Lower a qualifying `vector.contract %a, %b, %c` (with row-major matmul +/// semantics directly into `llvm.intr.matrix.multiply`: +/// BEFORE: /// ``` -/// %mta = maybe_transpose -/// %mtb = maybe_transpose -/// %flattened_a = vector.shape_cast %mta -/// %flattened_b = vector.shape_cast %mtb -/// %flattened_d = llvm.intr.matrix.multiply %flattened_a, %flattened_b -/// %mtd = vector.shape_cast %flattened_d -/// %d = maybe_untranspose %mtd -/// %e = add %c, %d +/// %res = vector.contract #matmat_trait %lhs, %rhs, %acc +/// : vector<2x4xf32>, vector<4x3xf32> into vector<2x3xf32> /// ``` -// -/// This only kicks in when vectorContractLowering is set to `Matmul`. -/// vector.transpose operations are inserted if the vector.contract op is not a -/// row-major matrix multiply. /// +/// AFTER: +/// ``` +/// %lhs = vector.shape_cast %arg0 : vector<2x4xf32> to vector<8xf32> +/// %rhs = vector.shape_cast %arg1 : vector<4x3xf32> to vector<12xf32> +/// %matmul = llvm.intr.matrix.multiply %lhs, %rhs +/// %res = arith.addf %acc, %matmul : vector<2x3xf32> +/// ``` +// /// Scalable vectors are not supported. FailureOr ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp( vector::ContractionOp op, MaskingOpInterface maskOp, @@ -2116,7 +2111,19 @@ FailureOr ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp( return res; } -/// Lowers vector.transpose to llvm.intr.matrix.transpose +/// Lowers vector.transpose directly to llvm.intr.matrix.transpose +/// +/// BEFORE: +/// ``` +/// %tr = vector.transpose %vec, [1, 0] : vector<2x4xf32> to vector<4x2xf32> +/// ``` +/// AFTER: +/// ``` +/// %vec_cs = vector.shape_cast %vec : vector<2x4xf32> to vector<8xf32> +/// %tr = llvm.intr.matrix.transpose %vec_sc +/// {columns = 2 : i32, rows = 4 : i32} : vector<8xf32> into vector<8xf32> +/// %res = vector.shape_cast %tr : vector<8xf32> to vector<4x2xf32> +/// ``` class TransposeOpToMatrixTransposeOpLowering : public OpRewritePattern { public: diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp index 0b44ca7ceee42..a65f8ba233b76 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp @@ -70,7 +70,7 @@ void ConvertVectorToLLVMPass::runOnOperation() { populateVectorBitCastLoweringPatterns(patterns); populateVectorBroadcastLoweringPatterns(patterns); populateVectorContractLoweringPatterns(patterns, vectorContractLowering); - if (vectorContractLowering == vector::VectorContractLowering::Matmul) { + if (vectorContractLowering == vector::VectorContractLowering::LLVM) { // This pattern creates a dependency on the LLVM dialect, hence we don't // include it in `populateVectorContractLoweringPatterns` that is part of // the Vector dialect (and should not depend on LLVM). @@ -80,7 +80,7 @@ void ConvertVectorToLLVMPass::runOnOperation() { populateVectorShapeCastLoweringPatterns(patterns); populateVectorInterleaveLoweringPatterns(patterns); populateVectorTransposeLoweringPatterns(patterns, vectorTransposeLowering); - if (vectorTransposeLowering == vector::VectorTransposeLowering::Flat) { + if (vectorTransposeLowering == vector::VectorTransposeLowering::LLVM) { // This pattern creates a dependency on the LLVM dialect, hence we don't // include it in `populateVectorTransposeLoweringPatterns` that is part of // the Vector dialect (and should not depend on LLVM). diff --git a/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir b/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir index 323d86ac40988..9b18d67e037db 100644 --- a/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir +++ b/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir @@ -13,7 +13,7 @@ // RUN: mlir-opt --convert-vector-to-llvm --dump-pass-pipeline %s 2>&1 | FileCheck %s --check-prefix=DEFAULT -// RUN: mlir-opt --convert-vector-to-llvm='vector-contract-lowering=matmul vector-transpose-lowering=flat' \ +// RUN: mlir-opt --convert-vector-to-llvm='vector-contract-lowering=llvm vector-transpose-lowering=llvm' \ // RUN: --dump-pass-pipeline %s 2>&1 | FileCheck %s --check-prefix=NON-DEFAULT // CHECK: builtin.module( @@ -26,5 +26,5 @@ // CHECK-SAME: reassociate-fp-reductions={{[aA-zZ0-9]+}} // DEFAULT: vector-contract-lowering=dot // DEFAULT: vector-transpose-lowering=eltwise -// NON-DEFAULT: vector-contract-lowering=matmul -// NON-DEFAULT: vector-transpose-lowering=flat +// NON-DEFAULT: vector-contract-lowering=llvm +// NON-DEFAULT: vector-transpose-lowering=llvm diff --git a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir index 3950e54006eec..fd5bbaaadc331 100644 --- a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --convert-vector-to-llvm='vector-contract-lowering=matmul' | FileCheck %s +// RUN: mlir-opt %s --convert-vector-to-llvm='vector-contract-lowering=llvm' | FileCheck %s #matmat_accesses = [ affine_map<(i, j, k) -> (i, k)>, diff --git a/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir b/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir index 94689fa0dfb88..66032014d9307 100644 --- a/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir +++ b/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --convert-vector-to-llvm='vector-transpose-lowering=flat' --split-input-file | FileCheck %s +// RUN: mlir-opt %s --convert-vector-to-llvm='vector-transpose-lowering=llvm' --split-input-file | FileCheck %s // CHECK-LABEL: func @transpose( func.func @transpose(%arg0: vector<2x4xf32>) -> vector<4x2xf32> { From 9b36aca5551dcd92f2d7e6610422cbb98def5eba Mon Sep 17 00:00:00 2001 From: Andrzej Warzynski Date: Thu, 18 Sep 2025 18:41:35 +0000 Subject: [PATCH 2/3] Fix Python test --- mlir/test/python/dialects/transform_vector_ext.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mlir/test/python/dialects/transform_vector_ext.py b/mlir/test/python/dialects/transform_vector_ext.py index 28902b012f7cb..8f44d86fd5180 100644 --- a/mlir/test/python/dialects/transform_vector_ext.py +++ b/mlir/test/python/dialects/transform_vector_ext.py @@ -74,9 +74,9 @@ def enum_configurable_patterns(): # CHECK: transform.apply_patterns.vector.lower_contraction vector.ApplyLowerContractionPatternsOp() # CHECK: transform.apply_patterns.vector.lower_contraction - # CHECK-SAME: lowering_strategy = matmulintrinsics + # CHECK-SAME: lowering_strategy = llvm vector.ApplyLowerContractionPatternsOp( - lowering_strategy=vector.VectorContractLowering.Matmul + lowering_strategy=vector.VectorContractLowering.LLVM ) # CHECK: transform.apply_patterns.vector.lower_contraction # CHECK-SAME: lowering_strategy = parallelarith @@ -105,9 +105,9 @@ def enum_configurable_patterns(): lowering_strategy=vector.VectorTransposeLowering.EltWise ) # CHECK: transform.apply_patterns.vector.lower_transpose - # CHECK-SAME: lowering_strategy = flat_transpose + # CHECK-SAME: lowering_strategy = llvm vector.ApplyLowerTransposePatternsOp( - lowering_strategy=vector.VectorTransposeLowering.Flat + lowering_strategy=vector.VectorTransposeLowering.LLVM ) # CHECK: transform.apply_patterns.vector.lower_transpose # CHECK-SAME: lowering_strategy = shuffle_1d @@ -120,10 +120,10 @@ def enum_configurable_patterns(): lowering_strategy=vector.VectorTransposeLowering.Shuffle16x16 ) # CHECK: transform.apply_patterns.vector.lower_transpose - # CHECK-SAME: lowering_strategy = flat_transpose + # CHECK-SAME: lowering_strategy = llvm # CHECK-SAME: avx2_lowering_strategy = true vector.ApplyLowerTransposePatternsOp( - lowering_strategy=vector.VectorTransposeLowering.Flat, + lowering_strategy=vector.VectorTransposeLowering.LLVM, avx2_lowering_strategy=True, ) From 126975098cbdba8ea46893b9b1833af4143e64c8 Mon Sep 17 00:00:00 2001 From: Andrzej Warzynski Date: Mon, 22 Sep 2025 07:49:14 +0000 Subject: [PATCH 3/3] Addressing comments from Jakub and Diego --- mlir/include/mlir/Conversion/Passes.td | 4 ++-- .../Vector/Transforms/VectorTransformsBase.td | 12 ++++++------ .../Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp | 8 ++++---- .../VectorToLLVM/ConvertVectorToLLVMPass.cpp | 4 ++-- .../VectorToLLVM/pass-option-serialization.mlir | 2 +- ...tor-contract-to-matrix-intrinsics-transforms.mlir | 2 +- ...tor-transpose-to-matrix-intrinsics-transform.mlir | 2 +- mlir/test/python/dialects/transform_vector_ext.py | 12 ++++++------ 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index aca0963478e63..3c18ecc753d0f 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -1489,7 +1489,7 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> { VectorContractLoweringAttr.summary, [{::llvm::cl::values( clEnumValN(::mlir::vector::VectorContractLowering::Dot, "dot", "Progressively lower to finer grained `vector.contract` and dot-products. (default)"), - clEnumValN(::mlir::vector::VectorContractLowering::LLVM, "llvm", + clEnumValN(::mlir::vector::VectorContractLowering::LLVMIntr, "llvmintr", "Lower directly to `llvm.intr.matrix.multiply`."), clEnumValN(::mlir::vector::VectorContractLowering::OuterProduct, "outerproduct", "Lower to `vector.outerproduct`."), @@ -1502,7 +1502,7 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> { VectorTransposeLoweringAttr.summary, [{::llvm::cl::values( clEnumValN(::mlir::vector::VectorTransposeLowering::EltWise, "eltwise", "Lower transpose into element-wise extract and inserts (default)"), - clEnumValN(::mlir::vector::VectorTransposeLowering::LLVM, "llvm", + clEnumValN(::mlir::vector::VectorTransposeLowering::LLVMIntr, "llvmintr", "Lower 2-D transpose directly to `llvm.intr.matrix.transpose`"), clEnumValN(::mlir::vector::VectorTransposeLowering::Shuffle1D, "shuffle1d", "Lower 2-D transpose to `vector.shuffle` on 1-D vector."), diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td index cbba44ae4dc8a..34febf2c4ff4b 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransformsBase.td @@ -15,8 +15,8 @@ include "mlir/IR/EnumAttr.td" def VectorTransposeLowering_Elementwise: I32EnumAttrCase<"EltWise", 0, "eltwise">; // Lower directly to LLVM matrix intrinsics. -def VectorTransposeLowering_LLVM: - I32EnumAttrCase<"LLVM", 1, "llvm">; +def VectorTransposeLowering_LLVMIntr: + I32EnumAttrCase<"LLVMIntr", 1, "llvmintr">; // Lower 2-D transpose to `vector.shuffle` on 1-D vector. def VectorTransposeLowering_Shuffle1D: I32EnumAttrCase<"Shuffle1D", 2, "shuffle_1d">; @@ -26,7 +26,7 @@ def VectorTransposeLowering_Shuffle16x16: def VectorTransposeLoweringAttr : I32EnumAttr< "VectorTransposeLowering", "control the lowering of `vector.transpose` operations.", - [VectorTransposeLowering_Elementwise, VectorTransposeLowering_LLVM, + [VectorTransposeLowering_Elementwise, VectorTransposeLowering_LLVMIntr, VectorTransposeLowering_Shuffle1D, VectorTransposeLowering_Shuffle16x16]> { let cppNamespace = "::mlir::vector"; } @@ -48,8 +48,8 @@ def VectorMultiReductionLoweringAttr: I32EnumAttr< // Progressively lower to finer grained `vector.contract` and dot-products. def VectorContractLowering_Dot: I32EnumAttrCase<"Dot", 0, "dot">; // Lower directly to LLVM intrinsics. -def VectorContractLowering_LLVM: - I32EnumAttrCase<"LLVM", 1, "llvm">; +def VectorContractLowering_LLVMIntr: + I32EnumAttrCase<"LLVMIntr", 1, "llvmintr">; // Lower to `vector.outerproduct`. def VectorContractLowering_OuterProduct: I32EnumAttrCase<"OuterProduct", 2, "outerproduct">; @@ -60,7 +60,7 @@ def VectorContractLowering_ParallelArith: def VectorContractLoweringAttr: I32EnumAttr< "VectorContractLowering", "control the lowering of `vector.contract` operations.", - [VectorContractLowering_Dot, VectorContractLowering_LLVM, + [VectorContractLowering_Dot, VectorContractLowering_LLVMIntr, VectorContractLowering_OuterProduct, VectorContractLowering_ParallelArith]> { let cppNamespace = "::mlir::vector"; } diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp index 4d60450e7fde8..e0b1a88d01cdc 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp @@ -2004,13 +2004,13 @@ class ContractionOpToMatmulOpLowering /// Lower a qualifying `vector.contract %a, %b, %c` (with row-major matmul /// semantics directly into `llvm.intr.matrix.multiply`: /// BEFORE: -/// ``` +/// ```mlir /// %res = vector.contract #matmat_trait %lhs, %rhs, %acc /// : vector<2x4xf32>, vector<4x3xf32> into vector<2x3xf32> /// ``` /// /// AFTER: -/// ``` +/// ```mlir /// %lhs = vector.shape_cast %arg0 : vector<2x4xf32> to vector<8xf32> /// %rhs = vector.shape_cast %arg1 : vector<4x3xf32> to vector<12xf32> /// %matmul = llvm.intr.matrix.multiply %lhs, %rhs @@ -2114,11 +2114,11 @@ FailureOr ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp( /// Lowers vector.transpose directly to llvm.intr.matrix.transpose /// /// BEFORE: -/// ``` +/// ```mlir /// %tr = vector.transpose %vec, [1, 0] : vector<2x4xf32> to vector<4x2xf32> /// ``` /// AFTER: -/// ``` +/// ```mlir /// %vec_cs = vector.shape_cast %vec : vector<2x4xf32> to vector<8xf32> /// %tr = llvm.intr.matrix.transpose %vec_sc /// {columns = 2 : i32, rows = 4 : i32} : vector<8xf32> into vector<8xf32> diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp index a65f8ba233b76..e2e6f1530bd2f 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp @@ -70,7 +70,7 @@ void ConvertVectorToLLVMPass::runOnOperation() { populateVectorBitCastLoweringPatterns(patterns); populateVectorBroadcastLoweringPatterns(patterns); populateVectorContractLoweringPatterns(patterns, vectorContractLowering); - if (vectorContractLowering == vector::VectorContractLowering::LLVM) { + if (vectorContractLowering == vector::VectorContractLowering::LLVMIntr) { // This pattern creates a dependency on the LLVM dialect, hence we don't // include it in `populateVectorContractLoweringPatterns` that is part of // the Vector dialect (and should not depend on LLVM). @@ -80,7 +80,7 @@ void ConvertVectorToLLVMPass::runOnOperation() { populateVectorShapeCastLoweringPatterns(patterns); populateVectorInterleaveLoweringPatterns(patterns); populateVectorTransposeLoweringPatterns(patterns, vectorTransposeLowering); - if (vectorTransposeLowering == vector::VectorTransposeLowering::LLVM) { + if (vectorTransposeLowering == vector::VectorTransposeLowering::LLVMIntr) { // This pattern creates a dependency on the LLVM dialect, hence we don't // include it in `populateVectorTransposeLoweringPatterns` that is part of // the Vector dialect (and should not depend on LLVM). diff --git a/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir b/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir index 9b18d67e037db..7e7925a473d53 100644 --- a/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir +++ b/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir @@ -13,7 +13,7 @@ // RUN: mlir-opt --convert-vector-to-llvm --dump-pass-pipeline %s 2>&1 | FileCheck %s --check-prefix=DEFAULT -// RUN: mlir-opt --convert-vector-to-llvm='vector-contract-lowering=llvm vector-transpose-lowering=llvm' \ +// RUN: mlir-opt --convert-vector-to-llvm='vector-contract-lowering=llvmintr vector-transpose-lowering=llvmintr' \ // RUN: --dump-pass-pipeline %s 2>&1 | FileCheck %s --check-prefix=NON-DEFAULT // CHECK: builtin.module( diff --git a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir index fd5bbaaadc331..bf4f094263545 100644 --- a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --convert-vector-to-llvm='vector-contract-lowering=llvm' | FileCheck %s +// RUN: mlir-opt %s --convert-vector-to-llvm='vector-contract-lowering=llvmintr' | FileCheck %s #matmat_accesses = [ affine_map<(i, j, k) -> (i, k)>, diff --git a/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir b/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir index 66032014d9307..f68badaa122cd 100644 --- a/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir +++ b/mlir/test/Dialect/Vector/vector-transpose-to-matrix-intrinsics-transform.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --convert-vector-to-llvm='vector-transpose-lowering=llvm' --split-input-file | FileCheck %s +// RUN: mlir-opt %s --convert-vector-to-llvm='vector-transpose-lowering=llvmintr' --split-input-file | FileCheck %s // CHECK-LABEL: func @transpose( func.func @transpose(%arg0: vector<2x4xf32>) -> vector<4x2xf32> { diff --git a/mlir/test/python/dialects/transform_vector_ext.py b/mlir/test/python/dialects/transform_vector_ext.py index 8f44d86fd5180..0cd9333dc1218 100644 --- a/mlir/test/python/dialects/transform_vector_ext.py +++ b/mlir/test/python/dialects/transform_vector_ext.py @@ -74,9 +74,9 @@ def enum_configurable_patterns(): # CHECK: transform.apply_patterns.vector.lower_contraction vector.ApplyLowerContractionPatternsOp() # CHECK: transform.apply_patterns.vector.lower_contraction - # CHECK-SAME: lowering_strategy = llvm + # CHECK-SAME: lowering_strategy = llvmintr vector.ApplyLowerContractionPatternsOp( - lowering_strategy=vector.VectorContractLowering.LLVM + lowering_strategy=vector.VectorContractLowering.LLVMIntr ) # CHECK: transform.apply_patterns.vector.lower_contraction # CHECK-SAME: lowering_strategy = parallelarith @@ -105,9 +105,9 @@ def enum_configurable_patterns(): lowering_strategy=vector.VectorTransposeLowering.EltWise ) # CHECK: transform.apply_patterns.vector.lower_transpose - # CHECK-SAME: lowering_strategy = llvm + # CHECK-SAME: lowering_strategy = llvmintr vector.ApplyLowerTransposePatternsOp( - lowering_strategy=vector.VectorTransposeLowering.LLVM + lowering_strategy=vector.VectorTransposeLowering.LLVMIntr ) # CHECK: transform.apply_patterns.vector.lower_transpose # CHECK-SAME: lowering_strategy = shuffle_1d @@ -120,10 +120,10 @@ def enum_configurable_patterns(): lowering_strategy=vector.VectorTransposeLowering.Shuffle16x16 ) # CHECK: transform.apply_patterns.vector.lower_transpose - # CHECK-SAME: lowering_strategy = llvm + # CHECK-SAME: lowering_strategy = llvmintr # CHECK-SAME: avx2_lowering_strategy = true vector.ApplyLowerTransposePatternsOp( - lowering_strategy=vector.VectorTransposeLowering.LLVM, + lowering_strategy=vector.VectorTransposeLowering.LLVMIntr, avx2_lowering_strategy=True, )