208 changes: 208 additions & 0 deletions mlir/include/mlir/Dialect/Affine/Passes.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,138 @@ def AffineDataCopyGeneration : Pass<"affine-data-copy-generate", "FuncOp"> {
];
}

def AffineLoopFusion : Pass<"affine-loop-fusion", "FuncOp"> {
let summary = "Fuse affine loop nests";
let description = [{
This pass performs fusion of loop nests using a slicing-based approach. It
combines two fusion strategies: producer-consumer fusion and sibling fusion.
Producer-consumer fusion is aimed at fusing pairs of loops where the first
one writes to a memref that the second reads. Sibling fusion targets pairs
of loops that share no dependences between them but that load from the same
memref. The fused loop nests, when possible, are rewritten to access
significantly smaller local buffers instead of the original memref's, and
the latter are often either completely optimized away or contracted. This
transformation leads to enhanced locality and lower memory footprint through
the elimination or contraction of temporaries/intermediate memref's. These
benefits are sometimes achieved at the expense of redundant computation
through a cost model that evaluates available choices such as the depth at
which a source slice should be materialized in the designation slice.

Example 1: Producer-consumer fusion.
Input:
```mlir
func @producer_consumer_fusion(%arg0: memref<10xf32>, %arg1: memref<10xf32>) {
%0 = memref.alloc() : memref<10xf32>
%1 = memref.alloc() : memref<10xf32>
%cst = arith.constant 0.000000e+00 : f32
affine.for %arg2 = 0 to 10 {
affine.store %cst, %0[%arg2] : memref<10xf32>
affine.store %cst, %1[%arg2] : memref<10xf32>
}
affine.for %arg2 = 0 to 10 {
%2 = affine.load %0[%arg2] : memref<10xf32>
%3 = arith.addf %2, %2 : f32
affine.store %3, %arg0[%arg2] : memref<10xf32>
}
affine.for %arg2 = 0 to 10 {
%2 = affine.load %1[%arg2] : memref<10xf32>
%3 = arith.mulf %2, %2 : f32
affine.store %3, %arg1[%arg2] : memref<10xf32>
}
return
}
```
Output:
```mlir
func @producer_consumer_fusion(%arg0: memref<10xf32>, %arg1: memref<10xf32>) {
%0 = memref.alloc() : memref<1xf32>
%1 = memref.alloc() : memref<1xf32>
%cst = arith.constant 0.000000e+00 : f32
affine.for %arg2 = 0 to 10 {
affine.store %cst, %0[0] : memref<1xf32>
affine.store %cst, %1[0] : memref<1xf32>
%2 = affine.load %1[0] : memref<1xf32>
%3 = arith.mulf %2, %2 : f32
affine.store %3, %arg1[%arg2] : memref<10xf32>
%4 = affine.load %0[0] : memref<1xf32>
%5 = arith.addf %4, %4 : f32
affine.store %5, %arg0[%arg2] : memref<10xf32>
}
return
}
```

Example 2: Sibling fusion.
Input:
```mlir
func @sibling_fusion(%arg0: memref<10x10xf32>, %arg1: memref<10x10xf32>,
%arg2: memref<10x10xf32>, %arg3: memref<10x10xf32>,
%arg4: memref<10x10xf32>) {
affine.for %arg5 = 0 to 3 {
affine.for %arg6 = 0 to 3 {
%0 = affine.load %arg0[%arg5, %arg6] : memref<10x10xf32>
%1 = affine.load %arg1[%arg5, %arg6] : memref<10x10xf32>
%2 = arith.mulf %0, %1 : f32
affine.store %2, %arg3[%arg5, %arg6] : memref<10x10xf32>
}
}
affine.for %arg5 = 0 to 3 {
affine.for %arg6 = 0 to 3 {
%0 = affine.load %arg0[%arg5, %arg6] : memref<10x10xf32>
%1 = affine.load %arg2[%arg5, %arg6] : memref<10x10xf32>
%2 = arith.addf %0, %1 : f32
affine.store %2, %arg4[%arg5, %arg6] : memref<10x10xf32>
}
}
return
}
```
Output:
```mlir
func @sibling_fusion(%arg0: memref<10x10xf32>, %arg1: memref<10x10xf32>,
%arg2: memref<10x10xf32>, %arg3: memref<10x10xf32>,
%arg4: memref<10x10xf32>) {
affine.for %arg5 = 0 to 3 {
affine.for %arg6 = 0 to 3 {
%0 = affine.load %arg0[%arg5, %arg6] : memref<10x10xf32>
%1 = affine.load %arg1[%arg5, %arg6] : memref<10x10xf32>
%2 = arith.mulf %0, %1 : f32
affine.store %2, %arg3[%arg5, %arg6] : memref<10x10xf32>
%3 = affine.load %arg0[%arg5, %arg6] : memref<10x10xf32>
%4 = affine.load %arg2[%arg5, %arg6] : memref<10x10xf32>
%5 = arith.addf %3, %4 : f32
affine.store %5, %arg4[%arg5, %arg6] : memref<10x10xf32>
}
}
return
}
```
}];
let constructor = "mlir::createLoopFusionPass()";
let options = [
Option<"computeToleranceThreshold", "fusion-compute-tolerance", "double",
/*default=*/"0.30f", "Fractional increase in additional computation "
"tolerated while fusing">,
Option<"fastMemorySpace", "fusion-fast-mem-space", "unsigned",
/*default=*/"0",
"Faster memory space number to promote fusion buffers to">,
Option<"localBufSizeThreshold", "fusion-local-buf-threshold", "uint64_t",
/*default=*/"0", "Threshold size (KiB) for promoting local buffers "
"to fast memory space">,
Option<"maximalFusion", "fusion-maximal", "bool", /*default=*/"false",
"Enables maximal loop fusion">,
Option<"affineFusionMode", "mode", "enum FusionMode",
"mlir::FusionMode::Greedy", "fusion mode to attempt",
"llvm::cl::values(clEnumValN(mlir::FusionMode::Greedy,"
" \"greedy\", \"Perform greedy (both producer-consumer and sibling) fusion\"), "
"clEnumValN( mlir::FusionMode::ProducerConsumer, "
"\"producer\", \"Perform only producer-consumer fusion\"), "
"clEnumValN( mlir::FusionMode::Sibling, "
"\"sibling\", \"Perform only sibling fusion\"))">,
];
let dependentDialects = ["memref::MemRefDialect"];
}

def AffineLoopInvariantCodeMotion
: Pass<"affine-loop-invariant-code-motion", "FuncOp"> {
let summary = "Hoist loop invariant instructions outside of affine loops";
Expand Down Expand Up @@ -94,6 +226,75 @@ def AffineLoopUnrollAndJam : Pass<"affine-loop-unroll-jam", "FuncOp"> {
];
}

def AffinePipelineDataTransfer
: Pass<"affine-pipeline-data-transfer", "FuncOp"> {
let summary = "Pipeline non-blocking data transfers between explicitly "
"managed levels of the memory hierarchy";
let description = [{
This pass performs a transformation to overlap non-blocking DMA operations
in a loop with computations through double buffering. This is achieved by
advancing dma_start operations with respect to other operations.

Input

```mlir
func @pipelinedatatransfer() {
%0 = memref.alloc() : memref<256xf32>
%1 = memref.alloc() : memref<32xf32, 1>
%2 = memref.alloc() : memref<1xf32>
%c0 = arith.constant 0 : index
%c128 = arith.constant 128 : index
affine.for %i0 = 0 to 8 {
affine.dma_start %0[%i0], %1[%i0], %2[%c0], %c128 : memref<256xf32>, memref<32xf32, 1>, memref<1xf32>
affine.dma_wait %2[%c0], %c128 : memref<1xf32>
%3 = affine.load %1[%i0] : memref<32xf32, 1>
%4 = "compute"(%3) : (f32) -> f32
affine.store %4, %1[%i0] : memref<32xf32, 1>
}
return
}
```

Output

```mlir
module {
func @pipelinedatatransfer() {
%c8 = arith.constant 8 : index
%c0 = arith.constant 0 : index
%0 = memref.alloc() : memref<256xf32>
%c0_0 = arith.constant 0 : index
%c128 = arith.constant 128 : index
%1 = memref.alloc() : memref<2x32xf32, 1>
%2 = memref.alloc() : memref<2x1xf32>
affine.dma_start %0[%c0], %1[%c0 mod 2, %c0], %2[%c0 mod 2, symbol(%c0_0)], %c128 : memref<256xf32>, memref<2x32xf32, 1>, memref<2x1xf32>
affine.for %arg0 = 1 to 8 {
affine.dma_start %0[%arg0], %1[%arg0 mod 2, %arg0], %2[%arg0 mod 2, symbol(%c0_0)], %c128 : memref<256xf32>, memref<2x32xf32, 1>, memref<2x1xf32>
%8 = affine.apply #map3(%arg0)
%9 = affine.apply #map4(%8)
%10 = affine.apply #map4(%8)
affine.dma_wait %2[%8 mod 2, symbol(%c0_0)], %c128 : memref<2x1xf32>
%11 = affine.load %1[%8 mod 2, %8] : memref<2x32xf32, 1>
%12 = "compute"(%11) : (f32) -> f32
affine.store %12, %1[%8 mod 2, %8] : memref<2x32xf32, 1>
}
%3 = affine.apply #map3(%c8)
%4 = affine.apply #map4(%3)
%5 = affine.apply #map4(%3)
affine.dma_wait %2[%3 mod 2, symbol(%c0_0)], %c128 : memref<2x1xf32>
%6 = affine.load %1[%3 mod 2, %3] : memref<2x32xf32, 1>
%7 = "compute"(%6) : (f32) -> f32
affine.store %7, %1[%3 mod 2, %3] : memref<2x32xf32, 1>
memref.dealloc %2 : memref<2x1xf32>
memref.dealloc %1 : memref<2x32xf32, 1>
return
}
}
```
}];
let constructor = "mlir::createPipelineDataTransferPass()";
}

def AffineScalarReplacement : Pass<"affine-scalrep", "FuncOp"> {
let summary = "Replace affine memref acceses by scalars by forwarding stores "
"to loads and eliminating redundant loads";
Expand Down Expand Up @@ -184,6 +385,13 @@ def AffineLoopNormalize : Pass<"affine-loop-normalize", "FuncOp"> {
let constructor = "mlir::createAffineLoopNormalizePass()";
}

def LoopCoalescing : Pass<"loop-coalescing", "FuncOp"> {
let summary = "Coalesce nested loops with independent bounds into a single "
"loop";
let constructor = "mlir::createLoopCoalescingPass()";
let dependentDialects = ["arith::ArithmeticDialect"];
}

def SimplifyAffineStructures : Pass<"simplify-affine-structures", "FuncOp"> {
let summary = "Simplify affine expressions in maps/sets and normalize "
"memrefs";
Expand Down
119 changes: 119 additions & 0 deletions mlir/include/mlir/Dialect/Affine/Utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ class DominanceInfo;
class Operation;
class PostDominanceInfo;

namespace memref {
class AllocOp;
} // namespace memref

struct LogicalResult;

using ReductionLoopMap = DenseMap<Operation *, SmallVector<LoopReduction, 2>>;
Expand Down Expand Up @@ -168,6 +172,121 @@ void normalizeAffineFor(AffineForOp op);
AffineExpr substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
AffineExpr max, bool positivePath = true);

/// Replaces all "dereferencing" uses of `oldMemRef` with `newMemRef` while
/// optionally remapping the old memref's indices using the supplied affine map,
/// `indexRemap`. The new memref could be of a different shape or rank.
/// `extraIndices` provides any additional access indices to be added to the
/// start.
///
/// `indexRemap` remaps indices of the old memref access to a new set of indices
/// that are used to index the memref. Additional input operands to indexRemap
/// can be optionally provided in `extraOperands`, and they occupy the start
/// of its input list. `indexRemap`'s dimensional inputs are expected to
/// correspond to memref's indices, and its symbolic inputs if any should be
/// provided in `symbolOperands`.
///
/// `domOpFilter`, if non-null, restricts the replacement to only those
/// operations that are dominated by the former; similarly, `postDomOpFilter`
/// restricts replacement to only those operations that are postdominated by it.
///
/// 'allowNonDereferencingOps', if set, allows replacement of non-dereferencing
/// uses of a memref without any requirement for access index rewrites as long
/// as the user operation has the MemRefsNormalizable trait. The default value
/// of this flag is false.
///
/// 'replaceInDeallocOp', if set, lets DeallocOp, a non-dereferencing user, to
/// also be a candidate for replacement. The default value of this flag is
/// false.
///
/// Returns true on success and false if the replacement is not possible,
/// whenever a memref is used as an operand in a non-dereferencing context and
/// 'allowNonDereferencingOps' is false, except for dealloc's on the memref
/// which are left untouched. See comments at function definition for an
/// example.
//
// Ex: to replace load %A[%i, %j] with load %Abuf[%t mod 2, %ii - %i, %j]:
// The SSA value corresponding to '%t mod 2' should be in 'extraIndices', and
// index remap will perform (%i, %j) -> (%ii - %i, %j), i.e., indexRemap = (d0,
// d1, d2) -> (d0 - d1, d2), and %ii will be the extra operand. Without any
// extra operands, note that 'indexRemap' would just be applied to existing
// indices (%i, %j).
// TODO: allow extraIndices to be added at any position.
LogicalResult replaceAllMemRefUsesWith(
Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices = {},
AffineMap indexRemap = AffineMap(), ArrayRef<Value> extraOperands = {},
ArrayRef<Value> symbolOperands = {}, Operation *domOpFilter = nullptr,
Operation *postDomOpFilter = nullptr, bool allowNonDereferencingOps = false,
bool replaceInDeallocOp = false);

/// Performs the same replacement as the other version above but only for the
/// dereferencing uses of `oldMemRef` in `op`, except in cases where
/// 'allowNonDereferencingOps' is set to true where we replace the
/// non-dereferencing uses as well.
LogicalResult replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
Operation *op,
ArrayRef<Value> extraIndices = {},
AffineMap indexRemap = AffineMap(),
ArrayRef<Value> extraOperands = {},
ArrayRef<Value> symbolOperands = {},
bool allowNonDereferencingOps = false);

/// Rewrites the memref defined by this alloc op to have an identity layout map
/// and updates all its indexing uses. Returns failure if any of its uses
/// escape (while leaving the IR in a valid state).
LogicalResult normalizeMemRef(memref::AllocOp *op);

/// Uses the old memref type map layout and computes the new memref type to have
/// a new shape and a layout map, where the old layout map has been normalized
/// to an identity layout map. It returns the old memref in case no
/// normalization was needed or a failure occurs while transforming the old map
/// layout to an identity layout map.
MemRefType normalizeMemRefType(MemRefType memrefType, OpBuilder builder,
unsigned numSymbolicOperands);

/// Creates and inserts into 'builder' a new AffineApplyOp, with the number of
/// its results equal to the number of operands, as a composition
/// of all other AffineApplyOps reachable from input parameter 'operands'. If
/// different operands were drawing results from multiple affine apply ops,
/// these will also be collected into a single (multi-result) affine apply op.
/// The final results of the composed AffineApplyOp are returned in output
/// parameter 'results'. Returns the affine apply op created.
Operation *createComposedAffineApplyOp(OpBuilder &builder, Location loc,
ArrayRef<Value> operands,
ArrayRef<Operation *> affineApplyOps,
SmallVectorImpl<Value> *results);

/// Given an operation, inserts one or more single result affine apply
/// operations, results of which are exclusively used by this operation.
/// The operands of these newly created affine apply ops are
/// guaranteed to be loop iterators or terminal symbols of a function.
///
/// Before
///
/// affine.for %i = 0 to #map(%N)
/// %idx = affine.apply (d0) -> (d0 mod 2) (%i)
/// send %A[%idx], ...
/// %v = "compute"(%idx, ...)
///
/// After
///
/// affine.for %i = 0 to #map(%N)
/// %idx = affine.apply (d0) -> (d0 mod 2) (%i)
/// send %A[%idx], ...
/// %idx_ = affine.apply (d0) -> (d0 mod 2) (%i)
/// %v = "compute"(%idx_, ...)

/// This allows the application of different transformations on send and
/// compute (for eg. different shifts/delays)
///
/// Fills `sliceOps` with the list of affine.apply operations.
/// In the following cases, `sliceOps` remains empty:
/// 1. If none of opInst's operands were the result of an affine.apply
/// (i.e., there was no affine computation slice to create).
/// 2. If all the affine.apply op's supplying operands to this opInst did not
/// have any uses other than those in this opInst.
void createAffineComputationSlice(Operation *opInst,
SmallVectorImpl<AffineApplyOp> *sliceOps);

} // namespace mlir

#endif // MLIR_DIALECT_AFFINE_UTILS_H
1 change: 0 additions & 1 deletion mlir/include/mlir/Dialect/ArmSVE/ArmSVE.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

include "mlir/Interfaces/SideEffectInterfaces.td"
include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
include "mlir/Dialect/Arithmetic/IR/ArithmeticBase.td"

//===----------------------------------------------------------------------===//
// ArmSVE dialect definition
Expand Down
17 changes: 9 additions & 8 deletions mlir/include/mlir/Transforms/BufferUtils.h → ...ct/Bufferization/Transforms/BufferUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,24 @@
//
//===----------------------------------------------------------------------===//

#ifndef MLIR_TRANSFORMS_BUFFERUTILS_H
#define MLIR_TRANSFORMS_BUFFERUTILS_H
#ifndef MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERUTILS_H
#define MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERUTILS_H

#include "mlir/Analysis/BufferViewFlowAnalysis.h"
#include "mlir/Analysis/Liveness.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/IR/Operation.h"
#include "mlir/Transforms/DialectConversion.h"

namespace mlir {
namespace memref {
class GlobalOp;
} // namespace memref

namespace bufferization {

/// A simple analysis that detects allocation operations.
class BufferPlacementAllocs {
Expand Down Expand Up @@ -117,10 +121,6 @@ class BufferPlacementTransformationBase {
Liveness liveness;
};

namespace memref {
class GlobalOp;
} // namespace memref

// Support class to create global ops for tensor-valued constants in the
// program. Globals are created lazily at the top of the `moduleOp` with pretty
// names. Duplicates are avoided.
Expand All @@ -137,6 +137,7 @@ class GlobalCreator {
// dependence to the memref dialect for this.
DenseMap<Attribute, Operation *> globals;
};
} // namespace bufferization
} // namespace mlir

#endif // MLIR_TRANSFORMS_BUFFERUTILS_H
#endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERUTILS_H
24 changes: 24 additions & 0 deletions mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,34 @@ namespace bufferization {
/// buffers.
std::unique_ptr<Pass> createBufferDeallocationPass();

/// Creates a pass that moves allocations upwards to reduce the number of
/// required copies that are inserted during the BufferDeallocation pass.
std::unique_ptr<Pass> createBufferHoistingPass();

/// Creates a pass that moves allocations upwards out of loops. This avoids
/// reallocations inside of loops.
std::unique_ptr<Pass> createBufferLoopHoistingPass();

/// Creates a pass that converts memref function results to out-params.
std::unique_ptr<Pass> createBufferResultsToOutParamsPass();

/// Creates a pass that finalizes a partial bufferization by removing remaining
/// bufferization.to_tensor and bufferization.to_memref operations.
std::unique_ptr<OperationPass<FuncOp>> createFinalizingBufferizePass();

/// Creates a pass that promotes heap-based allocations to stack-based ones.
/// Only buffers smaller than the provided size are promoted.
/// Dynamic shaped buffers are promoted up to the given rank.
std::unique_ptr<Pass>
createPromoteBuffersToStackPass(unsigned maxAllocSizeInBytes = 1024,
unsigned bitwidthOfIndexType = 64,
unsigned maxRankOfAllocatedMemRef = 1);

/// Creates a pass that promotes heap-based allocations to stack-based ones.
/// Only buffers smaller with `isSmallAlloc(alloc) == true` are promoted.
std::unique_ptr<Pass>
createPromoteBuffersToStackPass(std::function<bool(Value)> isSmallAlloc);

//===----------------------------------------------------------------------===//
// Registration
//===----------------------------------------------------------------------===//
Expand Down
69 changes: 69 additions & 0 deletions mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,51 @@ def BufferDeallocation : Pass<"buffer-deallocation", "FuncOp"> {
let constructor = "mlir::bufferization::createBufferDeallocationPass()";
}

def BufferHoisting : Pass<"buffer-hoisting", "FuncOp"> {
let summary = "Optimizes placement of allocation operations by moving them "
"into common dominators and out of nested regions";
let description = [{
This pass implements an approach to aggressively move allocations upwards
into common dominators and out of nested regions.
}];
let constructor = "mlir::bufferization::createBufferHoistingPass()";
}

def BufferLoopHoisting : Pass<"buffer-loop-hoisting", "FuncOp"> {
let summary = "Optimizes placement of allocation operations by moving them "
"out of loop nests";
let description = [{
This pass implements an approach to aggressively move allocations upwards
out of loop nests. It does not move allocations into common dominators.
}];
let constructor = "mlir::bufferization::createBufferLoopHoistingPass()";
}

def BufferResultsToOutParams : Pass<"buffer-results-to-out-params", "ModuleOp"> {
let summary = "Converts memref-typed function results to out-params";
let description = [{
Some calling conventions prefer to pass output memrefs as "out params". The
conversion to this calling convention must be done as an atomic
transformation of the entire program (hence this is a module pass).

For example, if a call is rewritten, the callee needs to be rewritten
otherwise the IR will end up invalid. Thus, this transformation
require an atomic change to the entire program (e.g. the whole module).

This pass is expected to run immediately after bufferization is finished.
At that point, tensor-typed results will have been converted to memref-typed
results, and can be consistently converted to out params.

All memref-typed results are appended to the function argument list.

The main issue with this pass (and the out-param calling convention) is that
buffers for results need to be allocated in the caller. This currently only
works for static shaped memrefs.
}];
let constructor = "mlir::bufferization::createBufferResultsToOutParamsPass()";
let dependentDialects = ["memref::MemRefDialect"];
}

def FinalizingBufferize : Pass<"finalizing-bufferize", "FuncOp"> {
let summary = "Finalize a partial bufferization";
let description = [{
Expand All @@ -104,4 +149,28 @@ def FinalizingBufferize : Pass<"finalizing-bufferize", "FuncOp"> {
let constructor = "mlir::bufferization::createFinalizingBufferizePass()";
}

def PromoteBuffersToStack : Pass<"promote-buffers-to-stack", "FuncOp"> {
let summary = "Promotes heap-based allocations to automatically managed "
"stack-based allocations";
let description = [{
This pass implements a simple algorithm to convert heap-based memory
allocations to stack-based ones. It uses a built-in heuristic to decide
whether it makes sense to convert an allocation. Furthermore, dynamic
shaped buffers that are limited by the rank of the tensor can be
converted. They are only transformed if they are considered to be small.
}];
let constructor = "mlir::bufferization::createPromoteBuffersToStackPass()";
let options = [
Option<"maxAllocSizeInBytes", "max-alloc-size-in-bytes", "unsigned",
/*default=*/"1024",
"Maximal size in bytes to promote allocations to stack.">,
Option<"bitwidthOfIndexType", "bitwidth-of-index-type", "unsigned",
/*default=*/"64",
"Bitwidth of the index type. Used for size estimation.">,
Option<"maxRankOfAllocatedMemRef", "max-rank-of-allocated-memref", "unsigned",
/*default=*/"1",
"Maximal memref rank to promote dynamic buffers.">,
];
}

#endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_PASSES
1 change: 0 additions & 1 deletion mlir/include/mlir/Dialect/GPU/GPUOps.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

include "mlir/Dialect/DLTI/DLTIBase.td"
include "mlir/Dialect/GPU/GPUBase.td"
include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
include "mlir/IR/EnumAttr.td"
include "mlir/IR/FunctionInterfaces.td"
include "mlir/IR/SymbolInterfaces.td"
Expand Down
4 changes: 4 additions & 0 deletions mlir/include/mlir/Dialect/MemRef/Transforms/Passes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ void populateResolveShapedTypeResultDimsPatterns(RewritePatternSet &patterns);
/// load/store ops into `patterns`.
std::unique_ptr<Pass> createFoldSubViewOpsPass();

/// Creates an interprocedural pass to normalize memrefs to have a trivial
/// (identity) layout map.
std::unique_ptr<OperationPass<ModuleOp>> createNormalizeMemRefsPass();

/// Creates an operation pass to resolve `memref.dim` operations with values
/// that are defined by operations that implement the
/// `ReifyRankedShapeTypeShapeOpInterface`, in terms of shapes of its input
Expand Down
116 changes: 116 additions & 0 deletions mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,122 @@ def FoldSubViewOps : Pass<"fold-memref-subview-ops"> {
];
}

def NormalizeMemRefs : Pass<"normalize-memrefs", "ModuleOp"> {
let summary = "Normalize memrefs";
let description = [{
This pass transforms memref types with a non-trivial
[layout map](https://mlir.llvm.org/docs/LangRef/#layout-map) into
memref types with an identity layout map, e.g. (i, j) -> (i, j). This
pass is inter-procedural, in the sense that it can modify function
interfaces and call sites that pass memref types. In order to modify
memref types while preserving the original behavior, users of those
memref types are also modified to incorporate the resulting layout map.
For instance, an [AffineLoadOp]
(https://mlir.llvm.org/docs/Dialects/Affine/#affineload-affineloadop)
will be updated to compose the layout map with with the affine expression
contained in the op. Operations marked with the [MemRefsNormalizable]
(https://mlir.llvm.org/docs/Traits/#memrefsnormalizable) trait are
expected to be normalizable. Supported operations include affine
operations, memref.alloc, memref.dealloc, and std.return.

Given an appropriate layout map specified in the code, this transformation
can express tiled or linearized access to multi-dimensional data
structures, but will not modify memref types without an explicit layout
map.

Currently this pass is limited to only modify
functions where all memref types can be normalized. If a function
contains any operations that are not MemRefNormalizable, then the function
and any functions that call or call it will not be modified.

Input

```mlir
#tile = affine_map<(i) -> (i floordiv 4, i mod 4)>
func @matmul(%A: memref<16xf64, #tile>,
%B: index, %C: memref<16xf64>) -> (memref<16xf64, #tile>) {
affine.for %arg3 = 0 to 16 {
%a = affine.load %A[%arg3] : memref<16xf64, #tile>
%p = arith.mulf %a, %a : f64
affine.store %p, %A[%arg3] : memref<16xf64, #tile>
}
%c = memref.alloc() : memref<16xf64, #tile>
%d = affine.load %c[0] : memref<16xf64, #tile>
return %A: memref<16xf64, #tile>
}
```

Output

```mlir
func @matmul(%arg0: memref<4x4xf64>, %arg1: index, %arg2: memref<16xf64>)
-> memref<4x4xf64> {
affine.for %arg3 = 0 to 16 {
%3 = affine.load %arg0[%arg3 floordiv 4, %arg3 mod 4]: memref<4x4xf64>
%4 = arith.mulf %3, %3 : f64
affine.store %4, %arg0[%arg3 floordiv 4, %arg3 mod 4]: memref<4x4xf64>
}
%0 = memref.alloc() : memref<4x4xf64>
%1 = affine.apply #map1()
%2 = affine.load %0[0, 0] : memref<4x4xf64>
return %arg0 : memref<4x4xf64>
}
```

Input

```
#linear8 = affine_map<(i, j) -> (i * 8 + j)>
func @linearize(%arg0: memref<8x8xi32, #linear8>,
%arg1: memref<8x8xi32, #linear8>,
%arg2: memref<8x8xi32, #linear8>) {
%c8 = arith.constant 8 : index
%c0 = arith.constant 0 : index
%c1 = arith.constant 1 : index
affine.for %arg3 = %c0 to %c8 {
affine.for %arg4 = %c0 to %c8 {
affine.for %arg5 = %c0 to %c8 {
%0 = affine.load %arg0[%arg3, %arg5] : memref<8x8xi32, #linear8>
%1 = affine.load %arg1[%arg5, %arg4] : memref<8x8xi32, #linear8>
%2 = affine.load %arg2[%arg3, %arg4] : memref<8x8xi32, #linear8>
%3 = arith.muli %0, %1 : i32
%4 = arith.addi %2, %3 : i32
affine.store %4, %arg2[%arg3, %arg4] : memref<8x8xi32, #linear8>
}
}
}
return
}
```

Output

```mlir
func @linearize(%arg0: memref<64xi32>,
%arg1: memref<64xi32>,
%arg2: memref<64xi32>) {
%c8 = arith.constant 8 : index
%c0 = arith.constant 0 : index
affine.for %arg3 = %c0 to %c8 {
affine.for %arg4 = %c0 to %c8 {
affine.for %arg5 = %c0 to %c8 {
%0 = affine.load %arg0[%arg3 * 8 + %arg5] : memref<64xi32>
%1 = affine.load %arg1[%arg5 * 8 + %arg4] : memref<64xi32>
%2 = affine.load %arg2[%arg3 * 8 + %arg4] : memref<64xi32>
%3 = arith.muli %0, %1 : i32
%4 = arith.addi %2, %3 : i32
affine.store %4, %arg2[%arg3 * 8 + %arg4] : memref<64xi32>
}
}
}
return
}
```
}];
let constructor = "mlir::memref::createNormalizeMemRefsPass()";
let dependentDialects = ["AffineDialect"];
}

def ResolveRankedShapeTypeResultDims :
Pass<"resolve-ranked-shaped-type-result-dims"> {
let summary = "Resolve memref.dim of result values of ranked shape type";
Expand Down
4 changes: 4 additions & 0 deletions mlir/include/mlir/Dialect/SCF/Passes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ std::unique_ptr<Pass> createForLoopPeelingPass();
/// inside of scf.for loops with known lower and upper bounds.
std::unique_ptr<Pass> createSCFForLoopCanonicalizationPass();

/// Creates a pass that transforms a single ParallelLoop over N induction
/// variables into another ParallelLoop over less than N induction variables.
std::unique_ptr<Pass> createParallelLoopCollapsingPass();

/// Creates a loop fusion pass which fuses parallel loops.
std::unique_ptr<Pass> createParallelLoopFusionPass();

Expand Down
16 changes: 16 additions & 0 deletions mlir/include/mlir/Dialect/SCF/Passes.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,22 @@ def SCFParallelLoopFusion : Pass<"parallel-loop-fusion"> {
let constructor = "mlir::createParallelLoopFusionPass()";
}

def SCFParallelLoopCollapsing : Pass<"parallel-loop-collapsing"> {
let summary = "Collapse parallel loops to use less induction variables";
let constructor = "mlir::createParallelLoopCollapsingPass()";
let options = [
ListOption<"clCollapsedIndices0", "collapsed-indices-0", "unsigned",
"Which loop indices to combine 0th loop index",
"llvm::cl::MiscFlags::CommaSeparated">,
ListOption<"clCollapsedIndices1", "collapsed-indices-1", "unsigned",
"Which loop indices to combine into the position 1 loop index",
"llvm::cl::MiscFlags::CommaSeparated">,
ListOption<"clCollapsedIndices2", "collapsed-indices-2", "unsigned",
"Which loop indices to combine into the position 2 loop index",
"llvm::cl::MiscFlags::CommaSeparated">,
];
}

def SCFParallelLoopSpecialization
: Pass<"parallel-loop-specialization", "FuncOp"> {
let summary = "Specialize parallel loops for vectorization";
Expand Down
60 changes: 60 additions & 0 deletions mlir/include/mlir/Dialect/SCF/Utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,5 +98,65 @@ getSCFMinMaxExpr(Value value, SmallVectorImpl<Value> &dims,
SmallVectorImpl<Value> &symbols,
llvm::function_ref<bool(Operation *)> loopFilter = nullptr);

/// Replace a perfect nest of "for" loops with a single linearized loop. Assumes
/// `loops` contains a list of perfectly nested loops with bounds and steps
/// independent of any loop induction variable involved in the nest.
void coalesceLoops(MutableArrayRef<scf::ForOp> loops);

/// Take the ParallelLoop and for each set of dimension indices, combine them
/// into a single dimension. combinedDimensions must contain each index into
/// loops exactly once.
void collapseParallelLoops(scf::ParallelOp loops,
ArrayRef<std::vector<unsigned>> combinedDimensions);

/// Promotes the loop body of a scf::ForOp to its containing block if the loop
/// was known to have a single iteration.
LogicalResult promoteIfSingleIteration(scf::ForOp forOp);

/// Unrolls this for operation by the specified unroll factor. Returns failure
/// if the loop cannot be unrolled either due to restrictions or due to invalid
/// unroll factors. Requires positive loop bounds and step. If specified,
/// annotates the Ops in each unrolled iteration by applying `annotateFn`.
LogicalResult loopUnrollByFactor(
scf::ForOp forOp, uint64_t unrollFactor,
function_ref<void(unsigned, Operation *, OpBuilder)> annotateFn = nullptr);

/// Tile a nest of standard for loops rooted at `rootForOp` by finding such
/// parametric tile sizes that the outer loops have a fixed number of iterations
/// as defined in `sizes`.
using Loops = SmallVector<scf::ForOp, 8>;
using TileLoops = std::pair<Loops, Loops>;
TileLoops extractFixedOuterLoops(scf::ForOp rootFOrOp, ArrayRef<int64_t> sizes);

/// Performs tiling fo imperfectly nested loops (with interchange) by
/// strip-mining the `forOps` by `sizes` and sinking them, in their order of
/// occurrence in `forOps`, under each of the `targets`.
/// Returns the new AffineForOps, one per each of (`forOps`, `targets`) pair,
/// nested immediately under each of `targets`.
SmallVector<Loops, 8> tile(ArrayRef<scf::ForOp> forOps, ArrayRef<Value> sizes,
ArrayRef<scf::ForOp> targets);

/// Performs tiling (with interchange) by strip-mining the `forOps` by `sizes`
/// and sinking them, in their order of occurrence in `forOps`, under `target`.
/// Returns the new AffineForOps, one per `forOps`, nested immediately under
/// `target`.
Loops tile(ArrayRef<scf::ForOp> forOps, ArrayRef<Value> sizes,
scf::ForOp target);

/// Tile a nest of scf::ForOp loops rooted at `rootForOp` with the given
/// (parametric) sizes. Sizes are expected to be strictly positive values at
/// runtime. If more sizes than loops are provided, discard the trailing values
/// in sizes. Assumes the loop nest is permutable.
/// Returns the newly created intra-tile loops.
Loops tilePerfectlyNested(scf::ForOp rootForOp, ArrayRef<Value> sizes);

/// Get perfectly nested sequence of loops starting at root of loop nest
/// (the first op being another AffineFor, and the second op - a terminator).
/// A loop is perfectly nested iff: the first op in the loop's body is another
/// AffineForOp, and the second op is a terminator).
void getPerfectlyNestedLoops(SmallVectorImpl<scf::ForOp> &nestedLoops,
scf::ForOp root);

} // namespace mlir

#endif // MLIR_DIALECT_SCF_UTILS_H_
4 changes: 2 additions & 2 deletions mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
namespace mlir {
namespace bufferization {
class BufferizeTypeConverter;
class GlobalCreator;
} // namespace bufferization

class GlobalCreator;
class RewritePatternSet;
using OwningRewritePatternList = RewritePatternSet;

Expand All @@ -38,7 +38,7 @@ std::unique_ptr<Pass> createFuncBufferizePass();
/// Add patterns to bufferize tensor constants into global memrefs to the given
/// pattern list.
void populateTensorConstantBufferizePatterns(
GlobalCreator &globalCreator,
bufferization::GlobalCreator &globalCreator,
bufferization::BufferizeTypeConverter &typeConverter,
RewritePatternSet &patterns);

Expand Down
13 changes: 13 additions & 0 deletions mlir/include/mlir/Interfaces/LoopLikeInterface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,20 @@

#include "mlir/IR/OpDefinition.h"

//===----------------------------------------------------------------------===//
// LoopLike Interfaces
//===----------------------------------------------------------------------===//

/// Include the generated interface declarations.
#include "mlir/Interfaces/LoopLikeInterface.h.inc"

//===----------------------------------------------------------------------===//
// LoopLike Utilities
//===----------------------------------------------------------------------===//

namespace mlir {
/// Move loop invariant code out of a `looplike` operation.
LogicalResult moveLoopInvariantCode(LoopLikeOpInterface looplike);
} // namespace mlir

#endif // MLIR_INTERFACES_LOOPLIKEINTERFACE_H_
70 changes: 70 additions & 0 deletions mlir/include/mlir/Transforms/ControlFlowSinkUtils.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
//===- ControlFlowSinkUtils.h - ControlFlow Sink Utils ----------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef MLIR_TRANSFORMS_CONTROLFLOWSINKUTILS_H
#define MLIR_TRANSFORMS_CONTROLFLOWSINKUTILS_H

#include "mlir/Support/LLVM.h"

namespace mlir {

class DominanceInfo;
class Operation;
class Region;
class RegionBranchOpInterface;

/// Given a list of regions, perform control flow sinking on them. For each
/// region, control-flow sinking moves operations that dominate the region but
/// whose only users are in the region into the regions so that they aren't
/// executed on paths where their results are not needed.
///
/// TODO: For the moment, this is a *simple* control-flow sink, i.e., no
/// duplicating of ops. It should be made to accept a cost model to determine
/// whether duplicating a particular op is profitable.
///
/// Example:
///
/// ```mlir
/// %0 = arith.addi %arg0, %arg1
/// scf.if %cond {
/// scf.yield %0
/// } else {
/// scf.yield %arg2
/// }
/// ```
///
/// After control-flow sink:
///
/// ```mlir
/// scf.if %cond {
/// %0 = arith.addi %arg0, %arg1
/// scf.yield %0
/// } else {
/// scf.yield %arg2
/// }
/// ```
///
/// Users must supply a callback `shouldMoveIntoRegion` that determines whether
/// the given operation that only has users in the given operation should be
/// moved into that region.
///
/// Returns the number of operations sunk.
size_t
controlFlowSink(ArrayRef<Region *> regions, DominanceInfo &domInfo,
function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion);

/// Populates `regions` with regions of the provided region branch op that are
/// executed at most once at that are reachable given the current operands of
/// the op. These regions can be passed to `controlFlowSink` to perform sinking
/// on the regions of the operation.
void getSinglyExecutedRegionsToSink(RegionBranchOpInterface branch,
SmallVectorImpl<Region *> &regions);

} // namespace mlir

#endif // MLIR_TRANSFORMS_CONTROLFLOWSINKUTILS_H
54 changes: 0 additions & 54 deletions mlir/include/mlir/Transforms/Passes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,41 +22,12 @@

namespace mlir {

class AffineForOp;
class GreedyRewriteConfig;

/// Fusion mode to attempt. The default mode `Greedy` does both
/// producer-consumer and sibling fusion.
enum FusionMode { Greedy, ProducerConsumer, Sibling };

//===----------------------------------------------------------------------===//
// Passes
//===----------------------------------------------------------------------===//

/// Creates a pass that moves allocations upwards to reduce the number of
/// required copies that are inserted during the BufferDeallocation pass.
std::unique_ptr<Pass> createBufferHoistingPass();

/// Creates a pass that moves allocations upwards out of loops. This avoids
/// reallocations inside of loops.
std::unique_ptr<Pass> createBufferLoopHoistingPass();

/// Creates a pass that promotes heap-based allocations to stack-based ones.
/// Only buffers smaller than the provided size are promoted.
/// Dynamic shaped buffers are promoted up to the given rank.
std::unique_ptr<Pass>
createPromoteBuffersToStackPass(unsigned maxAllocSizeInBytes = 1024,
unsigned bitwidthOfIndexType = 64,
unsigned maxRankOfAllocatedMemRef = 1);

/// Creates a pass that promotes heap-based allocations to stack-based ones.
/// Only buffers smaller with `isSmallAlloc(alloc) == true` are promoted.
std::unique_ptr<Pass>
createPromoteBuffersToStackPass(std::function<bool(Value)> isSmallAlloc);

/// Creates a pass that converts memref function results to out-params.
std::unique_ptr<Pass> createBufferResultsToOutParamsPass();

/// Creates an instance of the Canonicalizer pass, configured with default
/// settings (which can be overridden by pass options on the command line).
std::unique_ptr<Pass> createCanonicalizerPass();
Expand All @@ -80,31 +51,10 @@ std::unique_ptr<Pass> createControlFlowSinkPass();
/// Creates a pass to perform common sub expression elimination.
std::unique_ptr<Pass> createCSEPass();

/// Creates a loop fusion pass which fuses loops according to type of fusion
/// specified in `fusionMode`. Buffers of size less than or equal to
/// `localBufSizeThreshold` are promoted to memory space `fastMemorySpace`.
std::unique_ptr<OperationPass<FuncOp>>
createLoopFusionPass(unsigned fastMemorySpace = 0,
uint64_t localBufSizeThreshold = 0,
bool maximalFusion = false,
enum FusionMode fusionMode = FusionMode::Greedy);

/// Creates a loop invariant code motion pass that hoists loop invariant
/// instructions out of the loop.
std::unique_ptr<Pass> createLoopInvariantCodeMotionPass();

/// Creates a pass to pipeline explicit movement of data across levels of the
/// memory hierarchy.
std::unique_ptr<OperationPass<FuncOp>> createPipelineDataTransferPass();

/// Creates a pass that transforms perfectly nested loops with independent
/// bounds into a single loop.
std::unique_ptr<OperationPass<FuncOp>> createLoopCoalescingPass();

/// Creates a pass that transforms a single ParallelLoop over N induction
/// variables into another ParallelLoop over less than N induction variables.
std::unique_ptr<Pass> createParallelLoopCollapsingPass();

/// Creates a pass to strip debug information from a function.
std::unique_ptr<Pass> createStripDebugInfoPass();

Expand Down Expand Up @@ -137,10 +87,6 @@ std::unique_ptr<Pass> createSCCPPass();
/// pass may *only* be scheduled on an operation that defines a SymbolTable.
std::unique_ptr<Pass> createSymbolDCEPass();

/// Creates an interprocedural pass to normalize memrefs to have a trivial
/// (identity) layout map.
std::unique_ptr<OperationPass<ModuleOp>> createNormalizeMemRefsPass();

//===----------------------------------------------------------------------===//
// Registration
//===----------------------------------------------------------------------===//
Expand Down
409 changes: 0 additions & 409 deletions mlir/include/mlir/Transforms/Passes.td
View file

Large diffs are not rendered by default.

200 changes: 0 additions & 200 deletions mlir/include/mlir/Transforms/Utils.h
View file

This file was deleted.

1 change: 0 additions & 1 deletion mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
#include "mlir/IR/Builders.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/RegionUtils.h"
#include "llvm/ADT/Sequence.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Conversion/SCFToStandard/SCFToStandard.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
#include "mlir/IR/PatternMatch.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"

using namespace mlir;
using namespace mlir::scf;
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
#include "mlir/Support/MathExtras.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

#include "../PassDetail.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/SCF.h"
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
#include "PassDetail.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
Expand Down
4 changes: 2 additions & 2 deletions mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Utils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallPtrSet.h"
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Affine/Passes.h.inc"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/Support/Debug.h"
#include <deque>

Expand Down
3 changes: 3 additions & 0 deletions mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,12 @@ add_mlir_dialect_library(MLIRAffineTransforms
AffineLoopNormalize.cpp
AffineParallelize.cpp
AffineScalarReplacement.cpp
LoopCoalescing.cpp
LoopFusion.cpp
LoopTiling.cpp
LoopUnroll.cpp
LoopUnrollAndJam.cpp
PipelineDataTransfer.cpp
SuperVectorize.cpp
SimplifyAffineStructures.cpp

Expand Down
3 changes: 2 additions & 1 deletion mlir/lib/Transforms/LoopCoalescing.cpp → ...lect/Affine/Transforms/LoopCoalescing.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@

#include "PassDetail.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Dialect/SCF/Utils.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/RegionUtils.h"
#include "llvm/Support/Debug.h"
Expand Down
6 changes: 3 additions & 3 deletions mlir/lib/Transforms/LoopFusion.cpp → .../Dialect/Affine/Transforms/LoopFusion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopFusionUtils.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/LoopFusionUtils.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SetVector.h"
Expand Down
4 changes: 2 additions & 2 deletions mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Utils.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
using namespace mlir;
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Affine/Transforms/LoopUnroll.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
#include "PassDetail.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@
#include "mlir/Dialect/Affine/Analysis/AffineAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/Support/CommandLine.h"

Expand Down
5 changes: 5 additions & 0 deletions mlir/lib/Dialect/Affine/Transforms/PassDetail.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,18 @@
#ifndef DIALECT_AFFINE_TRANSFORMS_PASSDETAIL_H_
#define DIALECT_AFFINE_TRANSFORMS_PASSDETAIL_H_

#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Pass/Pass.h"

namespace mlir {
// Forward declaration from Dialect.h
template <typename ConcreteDialect>
void registerDialect(DialectRegistry &registry);

namespace arith {
class ArithmeticDialect;
} // namespace arith

namespace linalg {
class LinalgDialect;
} // namespace linalg
Expand Down
7 changes: 3 additions & 4 deletions mlir/lib/Transforms/PipelineDataTransfer.cpp → ...ffine/Transforms/PipelineDataTransfer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,16 @@
//===----------------------------------------------------------------------===//

#include "PassDetail.h"
#include "mlir/Transforms/Passes.h"

#include "mlir/Dialect/Affine/Analysis/AffineAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/Utils/Utils.h"
#include "mlir/IR/Builders.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Utils.h"
#include "mlir/Transforms/Passes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/Support/Debug.h"

Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/IR/IntegerSet.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/Utils.h"

#define DEBUG_TYPE "simplify-affine-structure"

Expand Down
3 changes: 3 additions & 0 deletions mlir/lib/Dialect/Affine/Utils/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
add_mlir_dialect_library(MLIRAffineUtils
LoopFusionUtils.cpp
LoopUtils.cpp
Utils.cpp

ADDITIONAL_HEADER_DIRS
Expand All @@ -7,5 +9,6 @@ add_mlir_dialect_library(MLIRAffineUtils
LINK_LIBS PUBLIC
MLIRAffine
MLIRAnalysis
MLIRMemRef
MLIRTransformUtils
)
5 changes: 2 additions & 3 deletions .../lib/Transforms/Utils/LoopFusionUtils.cpp → .../Dialect/Affine/Utils/LoopFusionUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,20 @@
//
//===----------------------------------------------------------------------===//

#include "mlir/Transforms/LoopFusionUtils.h"

#include "mlir/Dialect/Affine/LoopFusionUtils.h"
#include "mlir/Analysis/SliceAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/AffineAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Operation.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/Debug.h"
Expand Down
663 changes: 31 additions & 632 deletions mlir/lib/Transforms/Utils/LoopUtils.cpp → mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
View file

Large diffs are not rendered by default.

741 changes: 740 additions & 1 deletion mlir/lib/Dialect/Affine/Utils/Utils.cpp
View file

Large diffs are not rendered by default.

35 changes: 18 additions & 17 deletions mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,33 @@

#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/TypeUtilities.h"
#include "llvm/ADT/TypeSwitch.h"

using namespace mlir;
using namespace arm_sve;
using namespace mlir::arm_sve;

#include "mlir/Dialect/ArmSVE/ArmSVEDialect.cpp.inc"
//===----------------------------------------------------------------------===//
// ScalableVector versions of general helpers for comparison ops
//===----------------------------------------------------------------------===//

/// Return the scalable vector of the same shape and containing i1.
static Type getI1SameShape(Type type) {
auto i1Type = IntegerType::get(type.getContext(), 1);
if (auto sVectorType = type.dyn_cast<VectorType>())
return VectorType::get(sVectorType.getShape(), i1Type,
sVectorType.getNumScalableDims());
return nullptr;
}

//===----------------------------------------------------------------------===//
// Tablegen Definitions
//===----------------------------------------------------------------------===//

static Type getI1SameShape(Type type);
#include "mlir/Dialect/ArmSVE/ArmSVEDialect.cpp.inc"

#define GET_OP_CLASSES
#include "mlir/Dialect/ArmSVE/ArmSVE.cpp.inc"
Expand All @@ -38,16 +52,3 @@ void ArmSVEDialect::initialize() {
#include "mlir/Dialect/ArmSVE/ArmSVE.cpp.inc"
>();
}

//===----------------------------------------------------------------------===//
// ScalableVector versions of general helpers for comparison ops
//===----------------------------------------------------------------------===//

// Return the scalable vector of the same shape and containing i1.
static Type getI1SameShape(Type type) {
auto i1Type = IntegerType::get(type.getContext(), 1);
if (auto sVectorType = type.dyn_cast<VectorType>())
return VectorType::get(sVectorType.getShape(), i1Type,
sVectorType.getNumScalableDims());
return nullptr;
}
1 change: 0 additions & 1 deletion mlir/lib/Dialect/ArmSVE/IR/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,5 @@ add_mlir_dialect_library(MLIRArmSVE
LINK_LIBS PUBLIC
MLIRIR
MLIRLLVMIR
MLIRStandard
MLIRSideEffectInterfaces
)
14 changes: 0 additions & 14 deletions mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
#include "mlir/Dialect/ArmSVE/Transforms.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/PatternMatch.h"

Expand All @@ -34,19 +33,6 @@ class ForwardOperands : public OpConversionPattern<OpTy> {
}
};

class ReturnOpTypeConversion : public OpConversionPattern<ReturnOp> {
public:
using OpConversionPattern<ReturnOp>::OpConversionPattern;

LogicalResult
matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const final {
rewriter.updateRootInPlace(
op, [&]() { op->setOperands(adaptor.getOperands()); });
return success();
}
};

using SdotOpLowering = OneToOneConvertToLLVMPattern<SdotOp, SdotIntrOp>;
using SmmlaOpLowering = OneToOneConvertToLLVMPattern<SmmlaOp, SmmlaIntrOp>;
using UdotOpLowering = OneToOneConvertToLLVMPattern<UdotOp, UdotIntrOp>;
Expand Down
3 changes: 2 additions & 1 deletion mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,13 @@

#include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
#include "mlir/Dialect/Bufferization/Transforms/BufferUtils.h"
#include "mlir/Dialect/Bufferization/Transforms/Passes.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Transforms/BufferUtils.h"
#include "llvm/ADT/SetOperations.h"

using namespace mlir;
using namespace mlir::bufferization;

/// Walks over all immediate return-like terminators in the given region.
static LogicalResult
Expand Down
20 changes: 10 additions & 10 deletions mlir/lib/Transforms/BufferOptimizations.cpp → ...zation/Transforms/BufferOptimizations.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,15 @@
// convert heap-based allocations to stack-based allocations, if possible.

#include "PassDetail.h"
#include "mlir/Dialect/Bufferization/Transforms/BufferUtils.h"
#include "mlir/Dialect/Bufferization/Transforms/Passes.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/IR/Operation.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/BufferUtils.h"
#include "mlir/Transforms/Passes.h"

using namespace mlir;
using namespace mlir::bufferization;

/// Returns true if the given operation implements a known high-level region-
/// based control-flow interface.
Expand Down Expand Up @@ -422,23 +423,22 @@ class PromoteBuffersToStackPass

} // namespace

std::unique_ptr<Pass> mlir::createBufferHoistingPass() {
std::unique_ptr<Pass> mlir::bufferization::createBufferHoistingPass() {
return std::make_unique<BufferHoistingPass>();
}

std::unique_ptr<Pass> mlir::createBufferLoopHoistingPass() {
std::unique_ptr<Pass> mlir::bufferization::createBufferLoopHoistingPass() {
return std::make_unique<BufferLoopHoistingPass>();
}

std::unique_ptr<Pass>
mlir::createPromoteBuffersToStackPass(unsigned maxAllocSizeInBytes,
unsigned bitwidthOfIndexType,
unsigned maxRankOfAllocatedMemRef) {
std::unique_ptr<Pass> mlir::bufferization::createPromoteBuffersToStackPass(
unsigned maxAllocSizeInBytes, unsigned bitwidthOfIndexType,
unsigned maxRankOfAllocatedMemRef) {
return std::make_unique<PromoteBuffersToStackPass>(
maxAllocSizeInBytes, bitwidthOfIndexType, maxRankOfAllocatedMemRef);
}

std::unique_ptr<Pass>
mlir::createPromoteBuffersToStackPass(std::function<bool(Value)> isSmallAlloc) {
std::unique_ptr<Pass> mlir::bufferization::createPromoteBuffersToStackPass(
std::function<bool(Value)> isSmallAlloc) {
return std::make_unique<PromoteBuffersToStackPass>(std::move(isSmallAlloc));
}
5 changes: 3 additions & 2 deletions ...b/Transforms/BufferResultsToOutParams.cpp → ...n/Transforms/BufferResultsToOutParams.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
//===----------------------------------------------------------------------===//

#include "PassDetail.h"
#include "mlir/Dialect/Bufferization/Transforms/Passes.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Operation.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/Passes.h"

using namespace mlir;

Expand Down Expand Up @@ -139,6 +139,7 @@ struct BufferResultsToOutParamsPass
};
} // namespace

std::unique_ptr<Pass> mlir::createBufferResultsToOutParamsPass() {
std::unique_ptr<Pass>
mlir::bufferization::createBufferResultsToOutParamsPass() {
return std::make_unique<BufferResultsToOutParamsPass>();
}
54 changes: 50 additions & 4 deletions mlir/lib/Transforms/BufferUtils.cpp → .../Bufferization/Transforms/BufferUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,18 @@
//
//===----------------------------------------------------------------------===//

#include "mlir/Transforms/BufferUtils.h"
#include "PassDetail.h"
#include "mlir/Dialect/Bufferization/Transforms/BufferUtils.h"
#include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Operation.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/Passes.h"
#include "llvm/ADT/SetOperations.h"

using namespace mlir;
using namespace mlir::bufferization;

//===----------------------------------------------------------------------===//
// BufferPlacementAllocs
Expand Down Expand Up @@ -139,3 +139,49 @@ bool BufferPlacementTransformationBase::isLoop(Operation *op) {

return false;
}

//===----------------------------------------------------------------------===//
// BufferPlacementTransformationBase
//===----------------------------------------------------------------------===//

memref::GlobalOp GlobalCreator::getGlobalFor(arith::ConstantOp constantOp) {
auto type = constantOp.getType().cast<RankedTensorType>();

BufferizeTypeConverter typeConverter;

// If we already have a global for this constant value, no need to do
// anything else.
auto it = globals.find(constantOp.getValue());
if (it != globals.end())
return cast<memref::GlobalOp>(it->second);

// Create a builder without an insertion point. We will insert using the
// symbol table to guarantee unique names.
OpBuilder globalBuilder(moduleOp.getContext());
SymbolTable symbolTable(moduleOp);

// Create a pretty name.
SmallString<64> buf;
llvm::raw_svector_ostream os(buf);
interleave(type.getShape(), os, "x");
os << "x" << type.getElementType();

// Add an optional alignment to the global memref.
IntegerAttr memrefAlignment =
alignment > 0 ? IntegerAttr::get(globalBuilder.getI64Type(), alignment)
: IntegerAttr();

auto global = globalBuilder.create<memref::GlobalOp>(
constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
/*sym_visibility=*/globalBuilder.getStringAttr("private"),
/*type=*/typeConverter.convertType(type).cast<MemRefType>(),
/*initial_value=*/constantOp.getValue().cast<ElementsAttr>(),
/*constant=*/true,
/*alignment=*/memrefAlignment);
symbolTable.insert(global);
// The symbol table inserts at the end of the module, but globals are a bit
// nicer if they are at the beginning.
global->moveBefore(&moduleOp.front());
globals[constantOp.getValue()] = global;
return global;
}
3 changes: 3 additions & 0 deletions mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
add_mlir_dialect_library(MLIRBufferizationTransforms
Bufferize.cpp
BufferDeallocation.cpp
BufferOptimizations.cpp
BufferResultsToOutParams.cpp
BufferUtils.cpp
OneShotAnalysis.cpp

ADDITIONAL_HEADER_DIRS
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
#include "mlir/Dialect/GPU/GPUDialect.h"
#include "mlir/Dialect/GPU/Passes.h"
#include "mlir/Dialect/GPU/Utils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/PatternMatch.h"
Expand Down
3 changes: 1 addition & 2 deletions mlir/lib/Dialect/GPU/Transforms/MemoryPromotion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,13 @@
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/GPU/MemoryPromotion.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/GPU/GPUDialect.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/LoopUtils.h"

using namespace mlir;
using namespace mlir::gpu;
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Linalg/ComprehensiveBufferize/ArithInterfaceImpl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@

#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
#include "mlir/Dialect/Bufferization/Transforms/BufferUtils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/IR/Dialect.h"
#include "mlir/IR/Operation.h"
#include "mlir/Transforms/BufferUtils.h"

using namespace mlir::bufferization;

Expand Down
2 changes: 0 additions & 2 deletions mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,13 @@
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/Linalg/Transforms/CodegenStrategy.h"

#include "mlir/Dialect/Linalg/Passes.h"
#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
#include "mlir/Dialect/SCF/Transforms.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Passes.h"

using namespace mlir;
Expand Down
2 changes: 0 additions & 2 deletions mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

#include "mlir/Dialect/Linalg/Transforms/HoistPadding.h"
#include "mlir/Analysis/SliceAnalysis.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/SCF/SCF.h"
Expand All @@ -24,7 +23,6 @@
#include "mlir/IR/AsmState.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Debug.h"

Expand Down
2 changes: 0 additions & 2 deletions mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
#include "mlir/Analysis/SliceAnalysis.h"
#include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/SCF/SCF.h"
Expand All @@ -27,7 +26,6 @@
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Debug.h"

Expand Down
12 changes: 9 additions & 3 deletions mlir/lib/Dialect/Linalg/Transforms/LinalgStrategyPasses.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
#include "PassDetail.h"
#include "mlir/Analysis/SliceAnalysis.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Passes.h"
#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
Expand All @@ -29,9 +31,7 @@
#include "mlir/Pass/PassManager.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"

using namespace mlir;
using namespace mlir::vector;
Expand Down Expand Up @@ -348,7 +348,13 @@ struct LinalgStrategyEnablePass
return signalPassFailure();
}

promoteSingleIterationLoops(funcOp);
// Gathers all innermost loops through a post order pruned walk.
funcOp.walk([](Operation *op) {
if (auto forOp = dyn_cast<AffineForOp>(op))
(void)promoteIfSingleIteration(forOp);
else if (auto forOp = dyn_cast<scf::ForOp>(op))
(void)promoteIfSingleIteration(forOp);
});
if (options.hoistRedundantVectorTransfers)
hoistRedundantVectorTransfers(funcOp);

Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Dialect/Linalg/Utils/Utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
Expand All @@ -31,7 +32,6 @@
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/LoopUtils.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Debug.h"

Expand Down
1 change: 1 addition & 0 deletions mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
add_mlir_dialect_library(MLIRMemRefTransforms
FoldSubViewOps.cpp
NormalizeMemRefs.cpp
ResolveShapedTypeResultDims.cpp

ADDITIONAL_HEADER_DIRS
Expand Down
7 changes: 4 additions & 3 deletions mlir/lib/Transforms/NormalizeMemRefs.cpp → ...ct/MemRef/Transforms/NormalizeMemRefs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@

#include "PassDetail.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"
#include "mlir/Dialect/MemRef/Transforms/Passes.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Support/Debug.h"

Expand Down Expand Up @@ -43,7 +43,8 @@ struct NormalizeMemRefs : public NormalizeMemRefsBase<NormalizeMemRefs> {

} // namespace

std::unique_ptr<OperationPass<ModuleOp>> mlir::createNormalizeMemRefsPass() {
std::unique_ptr<OperationPass<ModuleOp>>
mlir::memref::createNormalizeMemRefsPass() {
return std::make_unique<NormalizeMemRefs>();
}

Expand Down
43 changes: 43 additions & 0 deletions mlir/lib/Dialect/MemRef/Transforms/PassDetail.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
//===- PassDetail.h - MemRef Pass class details -----------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef DIALECT_MEMREF_TRANSFORMS_PASSDETAIL_H_
#define DIALECT_MEMREF_TRANSFORMS_PASSDETAIL_H_

#include "mlir/Pass/Pass.h"

namespace mlir {

class AffineDialect;

// Forward declaration from Dialect.h
template <typename ConcreteDialect>
void registerDialect(DialectRegistry &registry);

namespace arith {
class ArithmeticDialect;
} // namespace arith

namespace memref {
class MemRefDialect;
} // namespace memref

namespace tensor {
class TensorDialect;
} // namespace tensor

namespace vector {
class VectorDialect;
} // namespace vector

#define GEN_PASS_CLASSES
#include "mlir/Dialect/MemRef/Transforms/Passes.h.inc"

} // namespace mlir

#endif // DIALECT_MEMREF_TRANSFORMS_PASSDETAIL_H_
4 changes: 1 addition & 3 deletions mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
//
//===----------------------------------------------------------------------===//

#include "PassDetail.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
Expand Down Expand Up @@ -107,9 +108,6 @@ struct DimOfReifyRankedShapedTypeOpInterface : public OpRewritePattern<OpTy> {
//===----------------------------------------------------------------------===//

namespace {
#define GEN_PASS_CLASSES
#include "mlir/Dialect/MemRef/Transforms/Passes.h.inc"

struct ResolveRankedShapeTypeResultDimsPass final
: public ResolveRankedShapeTypeResultDimsBase<
ResolveRankedShapeTypeResultDimsPass> {
Expand Down
2 changes: 0 additions & 2 deletions mlir/lib/Dialect/OpenACC/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ add_mlir_dialect_library(MLIROpenACC
MLIROpenACCOpsIncGen

LINK_LIBS PUBLIC
MLIRArithmetic
MLIRIR
MLIRStandard
)

18 changes: 10 additions & 8 deletions mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@
// =============================================================================

#include "mlir/Dialect/OpenACC/OpenACC.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/OpenACC/OpenACCOpsEnums.cpp.inc"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/Transforms/DialectConversion.h"
#include "llvm/ADT/TypeSwitch.h"
Expand Down Expand Up @@ -175,14 +174,17 @@ struct RemoveConstantIfCondition : public OpRewritePattern<OpTy> {
LogicalResult matchAndRewrite(OpTy op,
PatternRewriter &rewriter) const override {
// Early return if there is no condition.
if (!op.ifCond())
Value ifCond = op.ifCond();
if (!ifCond)
return success();

auto constOp = op.ifCond().template getDefiningOp<arith::ConstantOp>();
if (constOp && constOp.getValue().template cast<IntegerAttr>().getInt())
rewriter.updateRootInPlace(op, [&]() { op.ifCondMutable().erase(0); });
else if (constOp)
rewriter.eraseOp(op);
IntegerAttr constAttr;
if (matchPattern(ifCond, m_Constant(&constAttr))) {
if (constAttr.getInt())
rewriter.updateRootInPlace(op, [&]() { op.ifCondMutable().erase(0); });
else
rewriter.eraseOp(op);
}

return success();
}
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/OpImplementation.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Quant/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,5 @@ add_mlir_dialect_library(MLIRQuant
MLIRPass
MLIRSideEffectInterfaces
MLIRSupport
MLIRStandard
MLIRTransformUtils
)
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
#include "mlir/Dialect/Quant/QuantOps.h"
#include "mlir/Dialect/Quant/QuantizeUtils.h"
#include "mlir/Dialect/Quant/UniformSupport.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/SCF/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ add_mlir_dialect_library(MLIRSCF
MLIRBufferization
MLIRIR
MLIRLoopLikeInterface
MLIRMemRef
MLIRSideEffectInterfaces
MLIRStandard
)
Expand Down
3 changes: 1 addition & 2 deletions mlir/lib/Dialect/SCF/SCF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Support/MathExtras.h"
#include "mlir/Transforms/InliningUtils.h"

using namespace mlir;
using namespace mlir::scf;

Expand Down
1 change: 1 addition & 0 deletions mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ add_mlir_dialect_library(MLIRSCFTransforms
LoopPipelining.cpp
LoopRangeFolding.cpp
LoopSpecialization.cpp
ParallelLoopCollapsing.cpp
ParallelLoopFusion.cpp
ParallelLoopTiling.cpp
StructuralTypeConversions.cpp
Expand Down
6 changes: 3 additions & 3 deletions ...lib/Transforms/ParallelLoopCollapsing.cpp → ...SCF/Transforms/ParallelLoopCollapsing.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
//===----------------------------------------------------------------------===//

#include "PassDetail.h"
#include "mlir/Dialect/SCF/Passes.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Dialect/SCF/Utils.h"
#include "mlir/Transforms/RegionUtils.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
Expand All @@ -20,7 +20,7 @@ using namespace mlir;

namespace {
struct ParallelLoopCollapsing
: public ParallelLoopCollapsingBase<ParallelLoopCollapsing> {
: public SCFParallelLoopCollapsingBase<ParallelLoopCollapsing> {
void runOnOperation() override {
Operation *module = getOperation();

Expand Down
694 changes: 692 additions & 2 deletions mlir/lib/Dialect/SCF/Transforms/Utils.cpp
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion mlir/lib/Dialect/Shape/IR/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,5 @@ add_mlir_dialect_library(MLIRShape
MLIRInferTypeOpInterface
MLIRIR
MLIRSideEffectInterfaces
MLIRStandard
MLIRTensor
)
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Shape/IR/Shape.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/CommonFolders.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Traits.h"
#include "mlir/IR/Builders.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
include "mlir/Dialect/Shape/IR/ShapeOps.td"
include "mlir/Dialect/StandardOps/IR/Ops.td"
include "mlir/Dialect/Tensor/IR/TensorOps.td"

def AllInputShapesEq : Constraint<CPred< [{
Expand Down
2 changes: 0 additions & 2 deletions mlir/lib/Dialect/SparseTensor/IR/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@ add_mlir_dialect_library(MLIRSparseTensor
MLIRSparseTensorOpsIncGen

LINK_LIBS PUBLIC
MLIRArithmetic
MLIRDialect
MLIRIR
MLIRStandard
MLIRSupport
)
15 changes: 8 additions & 7 deletions mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@
//
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpImplementation.h"
#include "llvm/ADT/TypeSwitch.h"

Expand Down Expand Up @@ -194,8 +193,9 @@ mlir::sparse_tensor::getSparseTensorEncoding(Type type) {
//===----------------------------------------------------------------------===//

static LogicalResult isInBounds(Value dim, Value tensor) {
if (auto constantOp = dim.getDefiningOp<arith::ConstantOp>()) {
unsigned d = constantOp.getValue().cast<IntegerAttr>().getInt();
IntegerAttr constantAttr;
if (matchPattern(dim, m_Constant(&constantAttr))) {
unsigned d = constantAttr.getInt();
if (d >= tensor.getType().cast<RankedTensorType>().getRank())
return failure();
}
Expand Down Expand Up @@ -227,11 +227,12 @@ static LogicalResult verify(InitOp op) {
for (unsigned i = 0; i < rank; i++) {
if (shape[i] == ShapedType::kDynamicSize)
continue;
auto constantOp = op.sizes()[i].getDefiningOp<arith::ConstantOp>();
if (!constantOp ||
constantOp.getValue().cast<IntegerAttr>().getInt() != shape[i])
IntegerAttr constantAttr;
if (!matchPattern(op.sizes()[i], m_Constant(&constantAttr)) ||
constantAttr.getInt() != shape[i]) {
return op.emitError("unexpected mismatch with static dimension size ")
<< shape[i];
}
}
return success();
}
Expand Down
45 changes: 2 additions & 43 deletions mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,57 +12,16 @@

#include "PassDetail.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
#include "mlir/Dialect/Bufferization/Transforms/BufferUtils.h"
#include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/StandardOps/Transforms/Passes.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/Transforms/BufferUtils.h"
#include "mlir/Transforms/DialectConversion.h"

using namespace mlir;

memref::GlobalOp GlobalCreator::getGlobalFor(arith::ConstantOp constantOp) {
auto type = constantOp.getType().cast<RankedTensorType>();

bufferization::BufferizeTypeConverter typeConverter;

// If we already have a global for this constant value, no need to do
// anything else.
auto it = globals.find(constantOp.getValue());
if (it != globals.end())
return cast<memref::GlobalOp>(it->second);

// Create a builder without an insertion point. We will insert using the
// symbol table to guarantee unique names.
OpBuilder globalBuilder(moduleOp.getContext());
SymbolTable symbolTable(moduleOp);

// Create a pretty name.
SmallString<64> buf;
llvm::raw_svector_ostream os(buf);
interleave(type.getShape(), os, "x");
os << "x" << type.getElementType();

// Add an optional alignment to the global memref.
IntegerAttr memrefAlignment =
alignment > 0 ? IntegerAttr::get(globalBuilder.getI64Type(), alignment)
: IntegerAttr();

auto global = globalBuilder.create<memref::GlobalOp>(
constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
/*sym_visibility=*/globalBuilder.getStringAttr("private"),
/*type=*/typeConverter.convertType(type).cast<MemRefType>(),
/*initial_value=*/constantOp.getValue().cast<ElementsAttr>(),
/*constant=*/true,
/*alignment=*/memrefAlignment);
symbolTable.insert(global);
// The symbol table inserts at the end of the module, but globals are a bit
// nicer if they are at the beginning.
global->moveBefore(&moduleOp.front());
globals[constantOp.getValue()] = global;
return global;
}
using namespace mlir::bufferization;

namespace {
class BufferizeTensorConstantOp
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Vector/VectorTransferSplitRewritePatterns.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include <type_traits>

#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Vector/VectorTransforms.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#include <type_traits>

#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Dialect/Vector/VectorUnrollDistribute.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/Interfaces/VectorInterfaces.h"
Expand Down
83 changes: 83 additions & 0 deletions mlir/lib/Interfaces/LoopLikeInterface.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,95 @@
//===----------------------------------------------------------------------===//

#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Support/Debug.h"

using namespace mlir;

#define DEBUG_TYPE "loop-like"

//===----------------------------------------------------------------------===//
// LoopLike Interfaces
//===----------------------------------------------------------------------===//

/// Include the definitions of the loop-like interfaces.
#include "mlir/Interfaces/LoopLikeInterface.cpp.inc"

//===----------------------------------------------------------------------===//
// LoopLike Utilities
//===----------------------------------------------------------------------===//

// Checks whether the given op can be hoisted by checking that
// - the op and any of its contained operations do not depend on SSA values
// defined inside of the loop (by means of calling definedOutside).
// - the op has no side-effects. If sideEffecting is Never, sideeffects of this
// op and its nested ops are ignored.
static bool canBeHoisted(Operation *op,
function_ref<bool(Value)> definedOutside) {
// Check that dependencies are defined outside of loop.
if (!llvm::all_of(op->getOperands(), definedOutside))
return false;
// Check whether this op is side-effect free. If we already know that there
// can be no side-effects because the surrounding op has claimed so, we can
// (and have to) skip this step.
if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
if (!memInterface.hasNoEffect())
return false;
// If the operation doesn't have side effects and it doesn't recursively
// have side effects, it can always be hoisted.
if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
return true;

// Otherwise, if the operation doesn't provide the memory effect interface
// and it doesn't have recursive side effects we treat it conservatively as
// side-effecting.
} else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
return false;
}

// Recurse into the regions for this op and check whether the contained ops
// can be hoisted.
for (auto &region : op->getRegions()) {
for (auto &block : region) {
for (auto &innerOp : block)
if (!canBeHoisted(&innerOp, definedOutside))
return false;
}
}
return true;
}

LogicalResult mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
auto &loopBody = looplike.getLoopBody();

// We use two collections here as we need to preserve the order for insertion
// and this is easiest.
SmallPtrSet<Operation *, 8> willBeMovedSet;
SmallVector<Operation *, 8> opsToMove;

// Helper to check whether an operation is loop invariant wrt. SSA properties.
auto isDefinedOutsideOfBody = [&](Value value) {
auto *definingOp = value.getDefiningOp();
return (definingOp && !!willBeMovedSet.count(definingOp)) ||
looplike.isDefinedOutsideOfLoop(value);
};

// Do not use walk here, as we do not want to go into nested regions and hoist
// operations from there. These regions might have semantics unknown to this
// rewriting. If the nested regions are loops, they will have been processed.
for (auto &block : loopBody) {
for (auto &op : block.without_terminator()) {
if (canBeHoisted(&op, isDefinedOutsideOfBody)) {
opsToMove.push_back(&op);
willBeMovedSet.insert(&op);
}
}
}

// For all instructions that we found to be invariant, move outside of the
// loop.
LogicalResult result = looplike.moveOutOfLoop(opsToMove);
LLVM_DEBUG(looplike.print(llvm::dbgs() << "\n\nModified loop:\n"));
return result;
}
14 changes: 0 additions & 14 deletions mlir/lib/Transforms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,13 @@
add_subdirectory(Utils)

add_mlir_library(MLIRTransforms
BufferOptimizations.cpp
BufferResultsToOutParams.cpp
BufferUtils.cpp
Canonicalizer.cpp
ControlFlowSink.cpp
CSE.cpp
Inliner.cpp
LocationSnapshot.cpp
LoopCoalescing.cpp
LoopFusion.cpp
LoopInvariantCodeMotion.cpp
NormalizeMemRefs.cpp
OpStats.cpp
ParallelLoopCollapsing.cpp
PipelineDataTransfer.cpp
SCCP.cpp
StripDebugInfo.cpp
SymbolDCE.cpp
Expand All @@ -25,19 +17,13 @@ add_mlir_library(MLIRTransforms
${MLIR_MAIN_INCLUDE_DIR}/mlir/Transforms

DEPENDS
MLIRStandardOpsIncGen
MLIRTransformsPassIncGen

LINK_LIBS PUBLIC
MLIRAffine
MLIRAnalysis
MLIRBufferization
MLIRCopyOpInterface
MLIRLoopLikeInterface
MLIRMemRef
MLIRSCF
MLIRPass
MLIRSupport
MLIRTransformUtils
MLIRVector
)
1 change: 0 additions & 1 deletion mlir/lib/Transforms/CSE.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
#include "mlir/IR/Dominance.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/ScopedHashTable.h"
Expand Down
2 changes: 1 addition & 1 deletion mlir/lib/Transforms/ControlFlowSink.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
#include "PassDetail.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Transforms/ControlFlowSinkUtils.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/Utils.h"

using namespace mlir;

Expand Down
79 changes: 1 addition & 78 deletions mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,10 @@
//===----------------------------------------------------------------------===//

#include "PassDetail.h"
#include "mlir/Transforms/Passes.h"

#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/LoopUtils.h"
#include "mlir/Transforms/Passes.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
Expand All @@ -34,80 +31,6 @@ struct LoopInvariantCodeMotion
};
} // namespace

// Checks whether the given op can be hoisted by checking that
// - the op and any of its contained operations do not depend on SSA values
// defined inside of the loop (by means of calling definedOutside).
// - the op has no side-effects. If sideEffecting is Never, sideeffects of this
// op and its nested ops are ignored.
static bool canBeHoisted(Operation *op,
function_ref<bool(Value)> definedOutside) {
// Check that dependencies are defined outside of loop.
if (!llvm::all_of(op->getOperands(), definedOutside))
return false;
// Check whether this op is side-effect free. If we already know that there
// can be no side-effects because the surrounding op has claimed so, we can
// (and have to) skip this step.
if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
if (!memInterface.hasNoEffect())
return false;
// If the operation doesn't have side effects and it doesn't recursively
// have side effects, it can always be hoisted.
if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
return true;

// Otherwise, if the operation doesn't provide the memory effect interface
// and it doesn't have recursive side effects we treat it conservatively as
// side-effecting.
} else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
return false;
}

// Recurse into the regions for this op and check whether the contained ops
// can be hoisted.
for (auto &region : op->getRegions()) {
for (auto &block : region) {
for (auto &innerOp : block)
if (!canBeHoisted(&innerOp, definedOutside))
return false;
}
}
return true;
}

LogicalResult mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
auto &loopBody = looplike.getLoopBody();

// We use two collections here as we need to preserve the order for insertion
// and this is easiest.
SmallPtrSet<Operation *, 8> willBeMovedSet;
SmallVector<Operation *, 8> opsToMove;

// Helper to check whether an operation is loop invariant wrt. SSA properties.
auto isDefinedOutsideOfBody = [&](Value value) {
auto *definingOp = value.getDefiningOp();
return (definingOp && !!willBeMovedSet.count(definingOp)) ||
looplike.isDefinedOutsideOfLoop(value);
};

// Do not use walk here, as we do not want to go into nested regions and hoist
// operations from there. These regions might have semantics unknown to this
// rewriting. If the nested regions are loops, they will have been processed.
for (auto &block : loopBody) {
for (auto &op : block.without_terminator()) {
if (canBeHoisted(&op, isDefinedOutsideOfBody)) {
opsToMove.push_back(&op);
willBeMovedSet.insert(&op);
}
}
}

// For all instructions that we found to be invariant, move outside of the
// loop.
auto result = looplike.moveOutOfLoop(opsToMove);
LLVM_DEBUG(looplike.print(llvm::dbgs() << "\n\nModified loop:\n"));
return result;
}

void LoopInvariantCodeMotion::runOnOperation() {
// Walk through all loops in a function in innermost-loop-first order. This
// way, we first LICM from the inner loop, and place the ops in
Expand Down
19 changes: 0 additions & 19 deletions mlir/lib/Transforms/PassDetail.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,8 @@
#include "mlir/Transforms/Passes.h"

namespace mlir {
class AffineDialect;

// Forward declaration from Dialect.h
template <typename ConcreteDialect>
void registerDialect(DialectRegistry &registry);

namespace arith {
class ArithmeticDialect;
} // namespace arith

namespace memref {
class MemRefDialect;
} // namespace memref

namespace bufferization {
class BufferizationDialect;
} // namespace bufferization

#define GEN_PASS_CLASSES
#include "mlir/Transforms/Passes.h.inc"

} // namespace mlir

#endif // TRANSFORMS_PASSDETAIL_H_
13 changes: 0 additions & 13 deletions mlir/lib/Transforms/Utils/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,12 @@ add_mlir_library(MLIRTransformUtils
FoldUtils.cpp
GreedyPatternRewriteDriver.cpp
InliningUtils.cpp
LoopFusionUtils.cpp
LoopUtils.cpp
RegionUtils.cpp
Utils.cpp

ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Transforms

DEPENDS
MLIRStandardOpsIncGen

LINK_LIBS PUBLIC
MLIRAffine
MLIRArithmetic
MLIRAnalysis
MLIRAffineAnalysis
MLIRMemRef
MLIRSCF
MLIRPass
MLIRRewrite
MLIRStandard
)
2 changes: 1 addition & 1 deletion mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
//
//===----------------------------------------------------------------------===//

#include "mlir/Transforms/ControlFlowSinkUtils.h"
#include "mlir/IR/Dominance.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Transforms/Utils.h"
#include <vector>

#define DEBUG_TYPE "cf-sink"
Expand Down
1 change: 0 additions & 1 deletion mlir/lib/Transforms/Utils/DialectConversion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/FunctionInterfaces.h"
#include "mlir/Rewrite/PatternApplicator.h"
#include "mlir/Transforms/Utils.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
Expand Down
Loading