diff --git a/include/NeuraDialect/CMakeLists.txt b/include/NeuraDialect/CMakeLists.txt index 1c9b30b5..b829e584 100644 --- a/include/NeuraDialect/CMakeLists.txt +++ b/include/NeuraDialect/CMakeLists.txt @@ -1,11 +1,3 @@ -# Set TableGen include paths -set(MLIR_TABLEGEN_INCLUDES - ${PROJECT_SOURCE_DIR}/include - ${PROJECT_SOURCE_DIR}/include/NeuraDialect - ${CMAKE_CURRENT_BINARY_DIR}/include/NeuraDialect - ${MLIR_MAIN_INCLUDE_DIR} - ${MLIR_INCLUDE_DIR}) - add_mlir_dialect(Neura neura) set(LLVM_TARGET_DEFINITIONS NeuraPasses.td) diff --git a/include/NeuraDialect/NeuraOps.td b/include/NeuraDialect/NeuraOps.td index 2c2a8758..25028b57 100644 --- a/include/NeuraDialect/NeuraOps.td +++ b/include/NeuraDialect/NeuraOps.td @@ -125,7 +125,7 @@ def Neura_LoadIndexedOp: Op:$base, Variadic:$indices, Optional:$predicate); + let arguments = (ins AnyType:$base, Variadic:$indices, Optional:$predicate); let results = (outs AnyType:$result); let assemblyFormat = "$base `[` $indices `:` type($indices) `]` type($base) ($predicate^ `:` type($predicate))? attr-dict `:` type($result)"; } @@ -139,7 +139,7 @@ def Neura_StoreIndexedOp: Op:$base, Variadic:$indices, Optional:$predicate); + let arguments = (ins AnyType:$value, AnyType:$base, Variadic:$indices, Optional:$predicate); let results = (outs); let assemblyFormat = "$value `to` $base `[` $indices `:` type($indices) `]` type($base) ($predicate^ `:` type($predicate))? attr-dict `:` type($value)"; } diff --git a/lib/Conversion/CMakeLists.txt b/lib/Conversion/CMakeLists.txt index ee851744..98f5dac2 100644 --- a/lib/Conversion/CMakeLists.txt +++ b/lib/Conversion/CMakeLists.txt @@ -5,22 +5,6 @@ add_subdirectory(LlvmToNeura) add_subdirectory(MemRefToNeura) add_subdirectory(BuiltinToNeura) -# add_mlir_library( -# MLIRNeuraConversion - -# DEPENDS -# MLIRNeuraTransformsIncGen - -# LINK_LIBS PUBLIC -# MLIRIR -# MLIRPass -# MLIRSupport -# MLIRTransforms -# MLIRNeura -# MLIRNeuraArithToNeuraPass -# MLIRNeuraLlvmToNeuraPass -# ${dialect_libs} -# ) add_library(MLIRConversion INTERFACE) add_dependencies(MLIRConversion MLIRConversionIncGen) diff --git a/lib/NeuraDialect/CMakeLists.txt b/lib/NeuraDialect/CMakeLists.txt index 1147c216..e61b0b15 100644 --- a/lib/NeuraDialect/CMakeLists.txt +++ b/lib/NeuraDialect/CMakeLists.txt @@ -1,19 +1,3 @@ -# Set include paths for TableGen -set(MLIR_TABLEGEN_INCLUDES - "-I${PROJECT_SOURCE_DIR}/include" - "-I${PROJECT_SOURCE_DIR}/include/NeuraDialect" - "-I${CMAKE_CURRENT_BINARY_DIR}/include/NeuraDialect") - -# Generate TableGen files -set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/include/NeuraDialect/Neura.td) -mlir_tablegen(Neura.h.inc -gen-op-decls ${MLIR_TABLEGEN_INCLUDES}) -mlir_tablegen(Neura.cpp.inc -gen-op-defs ${MLIR_TABLEGEN_INCLUDES}) -mlir_tablegen(NeuraDialect.h.inc -gen-dialect-decls ${MLIR_TABLEGEN_INCLUDES}) -mlir_tablegen(NeuraDialect.cpp.inc -gen-dialect-defs ${MLIR_TABLEGEN_INCLUDES}) -mlir_tablegen(NeuraTypes.h.inc -gen-typedef-decls ${MLIR_TABLEGEN_INCLUDES}) -mlir_tablegen(NeuraTypes.cpp.inc -gen-typedef-defs ${MLIR_TABLEGEN_INCLUDES}) -add_public_tablegen_target(MLIRNeuraDialectIncGen) - # Add the dialect library add_mlir_dialect_library(MLIRNeura Neura.cpp @@ -27,7 +11,6 @@ add_mlir_dialect_library(MLIRNeura ${PROJECT_SOURCE_DIR}/include/NeuraDialect DEPENDS - MLIRNeuraDialectIncGen MLIRNeuraTransformsIncGen MLIRConversionIncGen diff --git a/lib/NeuraDialect/Transforms/LeveragePredicatedValuePass.cpp b/lib/NeuraDialect/Transforms/LeveragePredicatedValuePass.cpp index e894c2d0..48038f77 100644 --- a/lib/NeuraDialect/Transforms/LeveragePredicatedValuePass.cpp +++ b/lib/NeuraDialect/Transforms/LeveragePredicatedValuePass.cpp @@ -6,6 +6,7 @@ #include "mlir/IR/PatternMatch.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/GreedyPatternRewriteDriver.h" +#include "llvm/Support/raw_ostream.h" using namespace mlir; @@ -38,6 +39,7 @@ struct LeveragePredicatedValuePass if (block == &block->getParent()->front()) { return; } + for (BlockArgument arg : block->getArguments()) { Type origType = arg.getType(); diff --git a/lib/NeuraDialect/Transforms/TransformCtrlToDataFlowPass.cpp b/lib/NeuraDialect/Transforms/TransformCtrlToDataFlowPass.cpp index 095aedec..465c44a2 100644 --- a/lib/NeuraDialect/Transforms/TransformCtrlToDataFlowPass.cpp +++ b/lib/NeuraDialect/Transforms/TransformCtrlToDataFlowPass.cpp @@ -1,17 +1,33 @@ #include "Common/AcceleratorAttrs.h" #include "NeuraDialect/NeuraDialect.h" #include "NeuraDialect/NeuraOps.h" -#include "NeuraDialect/NeuraTypes.h" #include "NeuraDialect/NeuraPasses.h" +#include "NeuraDialect/NeuraTypes.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Block.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/Dominance.h" +#include "mlir/IR/Location.h" +#include "mlir/IR/OpDefinition.h" #include "mlir/IR/PatternMatch.h" +#include "mlir/IR/Value.h" #include "mlir/Pass/Pass.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Support/LLVM.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include +#include using namespace mlir; #define GEN_PASS_DEF_TransformCtrlToDataFlow #include "NeuraDialect/NeuraPasses.h.inc" +// TODO: Needs to enbale a deterministic ctrol to data flow transformation +// https://github.com/coredac/dataflow/issues/64 + // Inserts `grant_once` for every predicated value defined in the entry block // that is used outside of the block (i.e., a live-out). void GrantPredicateInEntryBlock(Block *entry_block, OpBuilder &builder) { @@ -32,7 +48,7 @@ void GrantPredicateInEntryBlock(Block *entry_block, OpBuilder &builder) { // Case 1: Operand of a branch/cond_br → grant_once if (isa(user)) { - used_in_branch = true; + used_in_branch = true; } // Case 2: Used directly in other blocks → grant_always @@ -56,7 +72,8 @@ void GrantPredicateInEntryBlock(Block *entry_block, OpBuilder &builder) { continue; builder.setInsertionPointAfter(def_op); - auto granted = builder.create(def_op->getLoc(), val.getType(), val); + auto granted = builder.create(def_op->getLoc(), + val.getType(), val); // Replaces uses in branch ops. for (OpOperand &use : llvm::make_early_inc_range(val.getUses())) { @@ -74,7 +91,8 @@ void GrantPredicateInEntryBlock(Block *entry_block, OpBuilder &builder) { continue; builder.setInsertionPointAfter(def_op); - auto granted = builder.create(def_op->getLoc(), val.getType(), val); + auto granted = builder.create(def_op->getLoc(), + val.getType(), val); // Replaces direct external uses (not in entry block, not in branch ops). for (OpOperand &use : llvm::make_early_inc_range(val.getUses())) { @@ -87,307 +105,549 @@ void GrantPredicateInEntryBlock(Block *entry_block, OpBuilder &builder) { } } -// Returns blocks in post-order traversal order. -void getBlocksInPostOrder(Block *startBlock, SmallVectorImpl &postOrder, - DenseSet &visited) { - if (!visited.insert(startBlock).second) - return; +// Control flow struct. +struct ControlFlowInfo { + struct Edge { + Block *source; + Block *target; + Value condition; // Optional condition for the edge. + bool is_not_condition; + SmallVector passed_values; // Values passed to the target block. + bool is_back_edge; + }; + std::vector> all_edges; // All edges in the function. + DenseMap> + incoming_edges; // Incoming edges for each block. + DenseMap> + outgoing_edges; // Outgoing edges for each block. + + DenseMap> back_edges; + DenseMap> forward_edges; + DenseSet blocks_with_back_edges; // Blocks with backward edges. + + Edge *createEdge() { + all_edges.push_back(std::make_unique()); + return all_edges.back().get(); + } +}; - // Visits successors first. - for (Block *succ : startBlock->getSuccessors()) - getBlocksInPostOrder(succ, postOrder, visited); +// Checks if all the live-out values in a block are dominated by the block's +// arguments. +void assertLiveOutValuesDominatedByBlockArgs(func::FuncOp &func) { + llvm::errs() + << "[ctrl2data] Asserting live-out values dominated by block arguments\n"; + for (Block &block : func.getBlocks()) { + if (&block == &func.getBody().front()) + continue; - // Adds current block to post-order sequence. - postOrder.push_back(startBlock); -} + DenseSet live_out_values; + for (Operation &op : block) { + for (Value result : op.getResults()) { + for (OpOperand &use : result.getUses()) { + if (use.getOwner()->getBlock() != &block) { + live_out_values.insert(result); + break; + } + } + } + } -// Creates phi nodes for all live-in values in the given block. -void createPhiNodesForBlock( - Block *block, OpBuilder &builder, - SmallVectorImpl> &deferred_ctrl_movs) { - if (block->hasNoPredecessors()) { - // Skips phi insertion for entry block. - return; - } + // Skips blocks with no live-out values. + if (live_out_values.empty()) + continue; - // Collects all live-in values. - std::vector live_ins; - for (Operation &op : *block) { - for (Value operand : op.getOperands()) { - // Identifies operands defined in other blocks. - if (operand.getDefiningOp() && - operand.getDefiningOp()->getBlock() != block) { - live_ins.push_back(operand); - continue; - } - // Collects all block arguments. - if (auto blockArg = llvm::dyn_cast(operand)) { - live_ins.push_back(operand); - } + DenseSet dominated_by_block_args; + + for (BlockArgument arg : block.getArguments()) { + dominated_by_block_args.insert(arg); } - } - builder.setInsertionPointToStart(block); - for (Value live_in : live_ins) { - // Creates predicated type for phi node. - Type live_in_type = live_in.getType(); - Type predicated_type = isa(live_in_type) - ? live_in_type - : neura::PredicatedValue::get(builder.getContext(), live_in_type, builder.getI1Type()); - - // Uses the location from the first operation in the block or block's parent operation. - Location loc = block->empty() ? - block->getParent()->getLoc() : - block->front().getLoc(); - SmallVector phi_operands; - llvm::SmallDenseSet just_created_consumer_ops; - BlockArgument arg = dyn_cast(live_in); - // TODO: Following logic needs to be refactored. - for (Block *pred : block->getPredecessors()) { - Value incoming; - Value branch_pred; - Operation *term = pred->getTerminator(); - // If it's a branch or cond_br, get the value passed into this block argument - if (auto br = dyn_cast(term)) { - auto args = br.getArgs(); - if (arg) { - unsigned arg_index = arg.getArgNumber(); - assert(arg_index < args.size()); - incoming = args[arg_index]; - } else if (live_in.getDefiningOp()->getBlock() == pred) { - // Handles the case where live_in is not a block argument. - incoming = live_in; - } else { - // If live_in is not a block argument and not defined in the block, skips. - continue; - } - } else if (auto condBr = dyn_cast(term)) { - Value cond = condBr.getCondition(); - branch_pred = cond; // by default - OpBuilder pred_builder(condBr); - Location pred_loc = condBr.getLoc(); - - if (condBr.getTrueDest() == block) { - if (arg) { - auto trueArgs = condBr.getTrueArgs(); - unsigned arg_index = arg.getArgNumber(); - assert(arg_index < trueArgs.size()); - incoming = trueArgs[arg_index]; - } else if (live_in.getDefiningOp()->getBlock() == pred) { - // Handles the case where live_in is not a block argument. - incoming = live_in; - } else { - // If live_in is not a block argument and not defined in the block, skips. - continue; - } - // Applies grant_predicate. - incoming = pred_builder.create( - pred_loc, incoming.getType(), incoming, cond); - just_created_consumer_ops.insert(incoming.getDefiningOp()); - // Keep branch_pred = cond - } else if (condBr.getFalseDest() == block) { - if (arg) { - auto falseArgs = condBr.getFalseArgs(); - unsigned arg_index = arg.getArgNumber(); - assert(arg_index < falseArgs.size()); - incoming = falseArgs[arg_index]; - } else if (live_in.getDefiningOp()->getBlock() == pred) { - // Handles the case where live_in is not a block argument. - incoming = live_in; - } else { - // If live_in is not a block argument and not defined in the block, skips. + if (block.getNumArguments() == 0 && !live_out_values.empty()) { + assert(false && "Block without arguments has live-out values"); + } + + bool changed = true; + while (changed) { + changed = false; + for (Operation &op : block) { + for (Value result : op.getResults()) { + if (dominated_by_block_args.count(result)) continue; + + for (Value operand : op.getOperands()) { + if (dominated_by_block_args.count(operand)) { + dominated_by_block_args.insert(result); + changed = true; + break; + } } - // Negates cond for false edge. - branch_pred = pred_builder.create(pred_loc, cond.getType(), cond); - // Applies grant_predicate. - incoming = pred_builder.create( - pred_loc, incoming.getType(), incoming, branch_pred); - just_created_consumer_ops.insert(incoming.getDefiningOp()); - } else { - llvm::errs() << "cond_br does not target block:\n" << *block << "\n"; - assert(false); } - } else { - llvm::errs() << "Unknown branch terminator in block: " << *pred << "\n"; - continue; } + } + for (Value live_out : live_out_values) { + if (!dominated_by_block_args.count(live_out)) + assert(false && "Live-out value not dominated by block arguments"); + } + } - // If the incoming value is defined in the same block, inserts a `neura.reserve` - // and defer a backward ctrl move. - if (incoming.getDefiningOp() && incoming.getDefiningOp()->getBlock() == block) { - builder.setInsertionPointToStart(block); - auto placeholder = builder.create(loc, incoming.getType()); - phi_operands.push_back(placeholder.getResult()); - // Defers the backward ctrl move operation to be inserted after all phi operands - // are defined. Inserted: - // (real_defined_value, just_created_reserve, branch_pred, current_block). - deferred_ctrl_movs.emplace_back( - incoming, placeholder.getResult(), branch_pred, block); - } else { - phi_operands.push_back(incoming); + llvm::errs() + << "[ctrl2data] All live-out values are dominated by block arguments.\n"; +} + +// Builds control flow info for the given function. +void buildControlFlowInfo(func::FuncOp &func, ControlFlowInfo &ctrl_info, + DominanceInfo &dom_info) { + for (Block &block : func.getBlocks()) { + Operation *terminator = block.getTerminator(); + + if (auto cond_br = dyn_cast(terminator)) { + Block *true_dest = cond_br.getTrueDest(); + Block *false_dest = cond_br.getFalseDest(); + + // Creates an edge for true destination. + ControlFlowInfo::Edge *true_edge = ctrl_info.createEdge(); + true_edge->source = █ + true_edge->target = true_dest; + true_edge->condition = cond_br.getCondition(); + true_edge->is_not_condition = false; + true_edge->is_back_edge = dom_info.dominates(true_dest, &block); + for (Value passed_value : cond_br.getTrueArgs()) { + true_edge->passed_values.push_back(passed_value); } - // If live_in is not a block argument, we don't need to check for uniqueness. - if (!arg) { - continue; + + // Creates an edge for false destination. + ControlFlowInfo::Edge *false_edge = ctrl_info.createEdge(); + false_edge->source = █ + false_edge->target = false_dest; + false_edge->condition = cond_br.getCondition(); + false_edge->is_not_condition = true; + false_edge->is_back_edge = dom_info.dominates(false_dest, &block); + for (Value passed_value : cond_br.getFalseArgs()) { + false_edge->passed_values.push_back(passed_value); } - } - assert(!phi_operands.empty()); - - // Puts all operands into a set to ensure uniqueness. Specifically, following - // case is handled: - // --------------------------------------------------------- - // ^bb1: - // "neura.br"(%a)[^bb3] : (!neura.data) -> () - // - // ^bb2: - // "neura.br"(%a)[^bb3] : (!neura.data) -> () - // - // ^bb3(%x: !neura.data): - // ... - // --------------------------------------------------------- - // In above case, %a is used in both branches of the control flow, so we - // don't need a phi node, but we still need to replace its uses with the - // result of the phi node. - // This ensures that we only create a phi node if there are multiple unique - // operands. - llvm::SmallDenseSet unique_operands(phi_operands.begin(), phi_operands.end()); - - if (unique_operands.size() == 1) { - // No phi needed, but still replace - Value single = *unique_operands.begin(); - SmallVector uses; - for (OpOperand &use : live_in.getUses()) { - // Skip uses that were just created by the grant_predicate. - if (!just_created_consumer_ops.contains(use.getOwner())) { - uses.push_back(&use); - } + // Creates the blocks to edges mapping. + ctrl_info.outgoing_edges[&block].push_back(true_edge); + ctrl_info.outgoing_edges[&block].push_back(false_edge); + ctrl_info.incoming_edges[true_dest].push_back(true_edge); + ctrl_info.incoming_edges[false_dest].push_back(false_edge); + + // Handles back edges. + if (true_edge->is_back_edge) { + ctrl_info.back_edges[&block].push_back(true_edge); + ctrl_info.blocks_with_back_edges.insert(&block); + } else { + ctrl_info.forward_edges[&block].push_back(true_edge); } - for (OpOperand *use : uses) { - use->set(single); + if (false_edge->is_back_edge) { + ctrl_info.back_edges[&block].push_back(false_edge); + ctrl_info.blocks_with_back_edges.insert(&block); + } else { + ctrl_info.forward_edges[&block].push_back(false_edge); } - // No need to proceed further to create a phi node, as we have a single unique operand. - continue; - } - // Creates the phi node with dynamic number of operands. - auto phi_op = builder.create(loc, predicated_type, phi_operands); + } else if (auto br = dyn_cast(terminator)) { + Block *dest = br.getDest(); + + // Creates unconditional edge to the destination block. + ControlFlowInfo::Edge *edge = ctrl_info.createEdge(); + edge->source = █ + edge->target = dest; + edge->condition = nullptr; // No condition for Br. + edge->is_not_condition = false; + edge->is_back_edge = dom_info.dominates(dest, &block); + for (Value passed_value : br.getArgs()) { + edge->passed_values.push_back(passed_value); + } + + // Updates the blocks to edges mapping. + ctrl_info.outgoing_edges[&block].push_back(edge); + ctrl_info.incoming_edges[dest].push_back(edge); - // Saves users to be replaced *after* phi is constructed. - SmallVector uses_to_be_replaced; - for (OpOperand &use : live_in.getUses()) { - if (use.getOwner() != phi_op) { - uses_to_be_replaced.push_back(&use); + // Handles back edges. + if (edge->is_back_edge) { + ctrl_info.back_edges[&block].push_back(edge); + ctrl_info.blocks_with_back_edges.insert(&block); + } else { + ctrl_info.forward_edges[&block].push_back(edge); } - } - // Replaces live-in uses with the phi result. - for (OpOperand *use : uses_to_be_replaced) { - use->set(phi_op.getResult()); + + } else if (auto rt = dyn_cast(terminator)) { + llvm::errs() << "[ctrl2data] ReturnOp found: " << *rt << "\n"; + } else { + llvm::errs() << "[ctrl2data] Unknown terminator: " << *terminator << "\n"; + assert(false); } } } -namespace { -struct TransformCtrlToDataFlowPass - : public PassWrapper> { - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TransformCtrlToDataFlowPass) +Value getPrecessedCondition(Value condition, bool is_not_condition, + DenseMap &condition_cache, + OpBuilder &builder) { + if (!is_not_condition) { + return condition; + } - StringRef getArgument() const override { return "transform-ctrl-to-data-flow"; } - StringRef getDescription() const override { - return "Transforms control flow into data flow using predicated execution"; + auto it = condition_cache.find(condition); + if (it != condition_cache.end()) { + return it->second; } - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); + Block *source = condition.getDefiningOp()->getBlock(); + builder.setInsertionPoint(source->getTerminator()); + Value not_condition = builder.create( + condition.getLoc(), condition.getType(), condition); + condition_cache[condition] = not_condition; + return not_condition; +} + +void createReserveAndPhiOps(func::FuncOp &func, ControlFlowInfo &ctrl_info, + DenseMap &arg_to_reserve, + DenseMap &arg_to_phi_result, + OpBuilder &builder) { + DominanceInfo dom_info(func); + + // ================================================ + // Step 1: Categorizes edges into six types. + // ================================================ + // Type 1: Backward cond_br edges with values. + // Type 2: Backward br edges with values. + // Type 3: Forward cond_br edges with values. + // Type 4: Forward br edges with values. + // Type 5: Forward cond_br edges without values. + // Type 6: Forward br edges without values. + // For Backward edges without values, they can be transformed into type 1 or 2 + + DenseMap> + backward_value_edges; + DenseMap> + forward_value_edges; + DenseMap> + block_conditional_edges; + + DenseMap condition_cache; + + DenseMap> arg_to_phi_operands; + + for (auto &edge : ctrl_info.all_edges) { + Block *target = edge->target; + + // Type 1 & 2: Backward cond_br/br edges with values. + if (edge->is_back_edge && !edge->passed_values.empty()) { + if (edge->passed_values.size() != target->getNumArguments()) { + llvm::errs() + << "[ctrl2data] Error: Number of passed values does not match " + "target block arguments.\n"; + assert(false); + } + for (BlockArgument arg : target->getArguments()) { + backward_value_edges[arg].push_back(edge.get()); + } + } + // Type 3 & 4: Forward cond_br/br edges with values. + else if (!edge->is_back_edge && !edge->passed_values.empty()) { + ; + if (edge->passed_values.size() != target->getNumArguments()) { + llvm::errs() + << "[ctrl2data] Error: Number of passed values does not match " + "target block arguments.\n"; + assert(false); + } + for (BlockArgument arg : target->getArguments()) { + forward_value_edges[arg].push_back(edge.get()); + } + } + // Type 5 & 6: Forward cond_br/br edges without values. + else if (!edge->is_back_edge && edge->passed_values.empty()) { + if (edge->condition) { + // Only Type 5 needs to be processed here. + // If the edge has a condition, store it for later use. + block_conditional_edges[target].push_back(edge.get()); + } + } } - void runOnOperation() override { - ModuleOp module = getOperation(); + // ================================================ + // Step 2: Creates reserve and ctrl_mov operations for needed blockarguments. + // ================================================ + // Handles Type 1 & 2 edges. + for (auto &backward_pair : backward_value_edges) { + BlockArgument arg = backward_pair.first; + auto &edges = backward_pair.second; + Block *block = arg.getOwner(); + builder.setInsertionPointToStart(block); + neura::ReserveOp reserve_op = + builder.create(arg.getLoc(), arg.getType()); + arg_to_reserve[arg] = reserve_op.getResult(); + arg_to_phi_operands[arg].push_back(reserve_op.getResult()); + + // Creates ctrl_mov operations for reserved values. + for (ControlFlowInfo::Edge *edge : edges) { + Value val = edge->passed_values[arg.getArgNumber()]; + builder.setInsertionPoint(edge->source->getTerminator()); + + Value predicated_val = val; + if (edge->condition) { + Value processed_condition = getPrecessedCondition( + edge->condition, edge->is_not_condition, condition_cache, builder); + predicated_val = builder.create( + edge->condition.getLoc(), val.getType(), predicated_val, + processed_condition); + } - // Declares a vector to hold deferred backward ctrl move operations. - // This is useful when a live-in value is defined within the same block. - // The tuple contains: - // - real value (the one that is defined in the same block, after the placeholder) - // - placeholder value (the one that will be used in the phi node) - // - branch predicate (if any, for cond_br) - // - block where the backward ctrl move should be inserted - SmallVector, 4> deferred_ctrl_movs; - module.walk([&](func::FuncOp func) { + // Creates ctrl_mov operation. + builder.create(val.getLoc(), predicated_val, + reserve_op); + } + } - OpBuilder builder(func.getContext()); - GrantPredicateInEntryBlock(&func.getBody().front(), builder); + // ================================================ + // Step 3: Prepares for creating phi operations. + // ================================================ + // Handles Type 3 & 4 edges. + + for (auto &forward_pair : forward_value_edges) { + BlockArgument arg = forward_pair.first; + auto &edges = forward_pair.second; - // Get blocks in post-order - SmallVector postOrder; - DenseSet visited; - getBlocksInPostOrder(&func.getBody().front(), postOrder, visited); + for (ControlFlowInfo::Edge *edge : edges) { + Value val = edge->passed_values[arg.getArgNumber()]; - // Process blocks bottom-up - for (Block *block : postOrder) { - // Creates phi nodes for live-ins. - createPhiNodesForBlock(block, builder, deferred_ctrl_movs); + Value predicated_val = val; + if (edge->condition) { + builder.setInsertionPoint(edge->source->getTerminator()); + Value processed_condition = getPrecessedCondition( + edge->condition, edge->is_not_condition, condition_cache, builder); + + predicated_val = builder.create( + edge->condition.getLoc(), predicated_val.getType(), predicated_val, + processed_condition); } - // Flattens blocks into the entry block. - Block *entryBlock = &func.getBody().front(); - SmallVector blocks_to_flatten; - for (Block &block : func.getBody()) { - if (&block != entryBlock) - blocks_to_flatten.push_back(&block); + arg_to_phi_operands[arg].push_back(predicated_val); + } + } + + // ================================================ + // Step 4: Creates phi operations for each block argument. + // ================================================ + DenseSet args_needing_phi; + + for (auto &arg_to_phi_pair : arg_to_phi_operands) { + BlockArgument arg = arg_to_phi_pair.first; + auto &phi_operands = arg_to_phi_pair.second; + + if (phi_operands.size() <= 1) { + // No need to create a phi operation if there's only one operand. + + if (phi_operands.size() == 1) { + arg_to_phi_result[arg] = phi_operands[0]; + arg.replaceAllUsesWith(phi_operands[0]); } + continue; + } + + // Handles the blcockargument with/without reserve seperately (different + // insertion points). + if (arg_to_reserve.count(arg)) { + Value reserve_value = arg_to_reserve[arg]; + builder.setInsertionPointAfter(reserve_value.getDefiningOp()); + + // Creates phi operation for reserved values. + auto phi = builder.create(arg.getLoc(), arg.getType(), + phi_operands); + arg_to_phi_result[arg] = phi; + } else { + Block *placement = arg.getParentBlock(); + + builder.setInsertionPointToStart(placement); + + // Creates phi operation for block argument without reserve. + auto phi = builder.create(arg.getLoc(), arg.getType(), + phi_operands); + arg.replaceAllUsesWith(phi); + arg_to_phi_result[arg] = phi; + } + } + + // ================================================ + // Step 5: Handles Forward cond_br edges without values. + // ================================================ + // Handles Type 5 edges. + for (auto &condition_pair : block_conditional_edges) { + Block *target = condition_pair.first; + auto &edges = condition_pair.second; - // Erases terminators before moving ops into entry block. - for (Block *block : blocks_to_flatten) { - for (Operation &op : llvm::make_early_inc_range(*block)) { - if (isa(op) || isa(op)) { - op.erase(); + if (edges.empty()) { + continue; + } + + // Collects all conditions for the target block. + SmallVector conditions; + for (ControlFlowInfo::Edge *edge : edges) { + Value condition = getPrecessedCondition( + edge->condition, edge->is_not_condition, condition_cache, builder); + conditions.push_back(condition); + } + + // Unsupported case: multiple conditions for a single block. + // TODO: Adds support if needed. + if (conditions.size() > 1) { + llvm::errs() << "[ctrl2data] Unsupported case: multiple conditions for a " + "single block: " + << *target << "\n"; + assert(false); + } + + if (target->getArguments().empty()) { + // Grants predicate for all the live-in values in the target block. + DenseSet live_in_values; + for (Operation &op : target->getOperations()) { + for (Value operand : op.getOperands()) { + if (operand.getDefiningOp() && + operand.getDefiningOp()->getBlock() != target && + !isa(operand.getDefiningOp())) { + live_in_values.insert(operand); } } } - // Moves all operations from blocks to the entry block before the terminator. - for (Block *block : blocks_to_flatten) { - auto &ops = block->getOperations(); - while (!ops.empty()) { - Operation &op = ops.front(); - op.moveBefore(&entryBlock->back()); + // Applies grant_predicate for each live-in value. + for (Value live_in_value : live_in_values) { + // Finds the earliest use of the live-in value. + Operation *earliest_use = nullptr; + for (Operation &op : target->getOperations()) { + for (Value operand : op.getOperands()) { + if (operand == live_in_value) { + earliest_use = &op; + break; + } + } + if (earliest_use) { + break; + } } - } - // Erases any remaining br/cond_br that were moved into the entry block. - for (Operation &op : llvm::make_early_inc_range(*entryBlock)) { - if (isa(op) || isa(op)) { - op.erase(); + if (earliest_use) { + builder.setInsertionPoint(earliest_use); + } else { + builder.setInsertionPointToStart(target); } - } - for (Block *block : blocks_to_flatten) { - block->erase(); + // Creates predicated version of the live-in value + Value predicated_value = builder.create( + live_in_value.getLoc(), live_in_value.getType(), live_in_value, + conditions[0]); + + // Replace uses of the live-in value within this block only. + for (OpOperand &use : + llvm::make_early_inc_range(live_in_value.getUses())) { + if (use.getOwner()->getBlock() == target && + use.getOwner() != predicated_value.getDefiningOp()) { + use.set(predicated_value); + } + } } - }); + } + } +} - // Inserts the deferred backward ctrl move operations after phi operands - // are defined. - for (auto &[real_dependent, placeholder, branch_pred, block] : deferred_ctrl_movs) { - Operation *def_op = real_dependent.getDefiningOp(); - assert(def_op && "Backward ctrl move's source must be an op result"); - - // Find the correct insertion point: after both real_dependent and branch_pred - Operation *insert_after = def_op; - if (Operation *pred_def = branch_pred.getDefiningOp()) { - if (insert_after->isBeforeInBlock(pred_def)) - insert_after = pred_def; - } +// Transforms control flow into data flow. +void transformControlFlowToDataFlow(func::FuncOp &func, + ControlFlowInfo &ctrl_info, + DominanceInfo &dom_info, + OpBuilder &builder) { + + // Asserts that all live-out values are dominated by block arguments. + assertLiveOutValuesDominatedByBlockArgs(func); + + // Creates reserve and phi operations for each block argument. + DenseMap arg_to_reserve; + DenseMap arg_to_phi_result; + createReserveAndPhiOps(func, ctrl_info, arg_to_reserve, arg_to_phi_result, + builder); + + // Replaces blockarguments with phi results + for (auto &arg_to_phi_pair : arg_to_phi_result) { + BlockArgument arg = arg_to_phi_pair.first; + Value phi_result = arg_to_phi_pair.second; + arg.replaceAllUsesWith(phi_result); + } - OpBuilder mov_builder(insert_after->getBlock(), ++Block::iterator(insert_after)); - Location insert_loc = insert_after->getLoc(); + // Flattens blocks into the entry block. + Block *entryBlock = &func.getBody().front(); + SmallVector blocks_to_flatten; + for (Block &block : func.getBody()) { + if (&block != entryBlock) + blocks_to_flatten.push_back(&block); + } - Value guarded_val = real_dependent; + // Erases terminators before moving ops into entry block. + for (Block *block : blocks_to_flatten) { + for (Operation &op : llvm::make_early_inc_range(*block)) { + if (isa(op) || isa(op)) { + op.erase(); + } + } + } + + // Moves all operations from blocks to the entry block before the + // terminator. + for (Block *block : blocks_to_flatten) { + auto &ops = block->getOperations(); + while (!ops.empty()) { + Operation &op = ops.front(); + op.moveBefore(&entryBlock->back()); + } + } - mov_builder.create(insert_loc, guarded_val, placeholder); + // Erases any remaining br/cond_br that were moved into the entry block. + for (Operation &op : llvm::make_early_inc_range(*entryBlock)) { + if (isa(op) || isa(op)) { + op.erase(); } } + + // Erases now-empty blocks + for (Block *block : blocks_to_flatten) { + block->erase(); + } +} + +namespace { +struct TransformCtrlToDataFlowPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TransformCtrlToDataFlowPass) + + StringRef getArgument() const override { + return "transform-ctrl-to-data-flow"; + } + StringRef getDescription() const override { + return "Transforms control flow into data flow using predicated " + "execution"; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + + void runOnOperation() override { + ModuleOp module = getOperation(); + module.walk([&](func::FuncOp func) { + OpBuilder builder(func.getContext()); + GrantPredicateInEntryBlock(&func.getBody().front(), builder); + + DominanceInfo dom_info(func); + + // Step 1: Analyzes the control flow and creates control flow info + // struct. + ControlFlowInfo ctrl_info; + buildControlFlowInfo(func, ctrl_info, dom_info); + + // Step 2: Transforms control flow into data flow. + transformControlFlowToDataFlow(func, ctrl_info, dom_info, builder); + }); + } }; } // namespace diff --git a/test/affine2neura/bert/bert_node0/bert_node0.mlir b/test/affine2neura/bert/bert_node0/bert_node0.mlir index ba82071e..12f98cfd 100644 --- a/test/affine2neura/bert/bert_node0/bert_node0.mlir +++ b/test/affine2neura/bert/bert_node0/bert_node0.mlir @@ -35,3 +35,4 @@ module attributes {} { // CHECK-NEXT: neura.br %12 : i64 to ^bb1 // CHECK-NEXT: ^bb3: // pred: ^bb1 // CHECK-NEXT: "neura.return"() : () -> () +// CHECK-NEXT: } \ No newline at end of file diff --git a/test/affine2neura/bert/bert_node1/bert_node1.mlir b/test/affine2neura/bert/bert_node1/bert_node1.mlir index f79959a2..ebaca758 100644 --- a/test/affine2neura/bert/bert_node1/bert_node1.mlir +++ b/test/affine2neura/bert/bert_node1/bert_node1.mlir @@ -1,5 +1,6 @@ // RUN: mlir-opt %s --lower-affine --convert-scf-to-cf --convert-cf-to-llvm -o %t-llvm.mlir // RUN: mlir-neura-opt %t-llvm.mlir --assign-accelerator --lower-arith-to-neura --lower-memref-to-neura --lower-builtin-to-neura --lower-llvm-to-neura | FileCheck %s +// RUN: mlir-neura-opt %t-llvm.mlir --assign-accelerator --lower-arith-to-neura --lower-memref-to-neura --lower-builtin-to-neura --lower-llvm-to-neura --leverage-predicated-value --transform-ctrl-to-data-flow | FileCheck %s -check-prefix=CTRL2DATA module attributes {} { func.func @_Z10bert_node1PA1_A1_A1_A1_A128_bPA1_A128_S1_(%arg0: memref, %arg1: memref) attributes {} { affine.for %arg2 = 0 to 128 { @@ -42,3 +43,42 @@ module attributes {} { // CHECK-NEXT: ^bb6: // pred: ^bb1 // CHECK-NEXT: "neura.return"() : () -> () // CHECK-NEXT: } + + +// CTRL2DATA: func.func @_Z10bert_node1PA1_A1_A1_A1_A128_bPA1_A128_S1_(%arg0: memref, %arg1: memref) attributes {accelerator = "neura"} { +// CTRL2DATA-NEXT: %0 = "neura.constant"() <{value = 1 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %1 = "neura.grant_always"(%0) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %2 = "neura.constant"() <{value = 128 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %3 = "neura.grant_always"(%2) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %4 = "neura.constant"() <{value = 0 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %5 = "neura.grant_always"(%4) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %6 = "neura.cast"(%4) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %7 = "neura.grant_once"(%6) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %8 = neura.reserve : !neura.data +// CTRL2DATA-NEXT: %9 = "neura.phi"(%8, %7) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %10 = "neura.cast"(%9) <{cast_type = "int_to_index"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %11 = "neura.icmp"(%10, %3) <{cmpType = "slt"}> : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %12 = "neura.not"(%11) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %13 = neura.grant_predicate %5, %11 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %14 = "neura.cast"(%13) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %15 = neura.reserve : !neura.data +// CTRL2DATA-NEXT: %16 = "neura.phi"(%15, %14) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %17 = "neura.cast"(%16) <{cast_type = "int_to_index"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %18 = "neura.icmp"(%17, %3) <{cmpType = "slt"}> : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %19 = "neura.not"(%18) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[VAL1:.*]] = neura.grant_predicate %5, %18 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL2:.*]] = neura.grant_predicate %17, %18 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %22 = neura.load_indexed %arg0[%[[VAL1:.*]], %[[VAL1:.*]], %[[VAL1:.*]], %[[VAL1:.*]], %[[VAL1:.*]], %[[VAL2:.*]] : !neura.data, !neura.data, !neura.data, !neura.data, !neura.data, !neura.data] memref : !neura.data +// CTRL2DATA-NEXT: %23 = neura.grant_predicate %10, %18 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: neura.store_indexed %22 to %arg1[%[[VAL1:.*]], %[[VAL1:.*]], %23, %[[VAL1:.*]], %[[VAL1:.*]], %[[VAL2:.*]] : !neura.data, !neura.data, !neura.data, !neura.data, !neura.data, !neura.data] memref : !neura.data +// CTRL2DATA-NEXT: %24 = neura.grant_predicate %1, %18 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %25 = "neura.add"(%[[VAL2:.*]], %24) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %26 = "neura.cast"(%25) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: neura.ctrl_mov %26 -> %15 : !neura.data !neura.data +// CTRL2DATA-DAG: %[[VAL3:.*]] = neura.grant_predicate %10, %19 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL4:.*]] = neura.grant_predicate %1, %19 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %29 = "neura.add"(%[[VAL3:.*]], %[[VAL4:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %30 = "neura.cast"(%29) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: neura.ctrl_mov %30 -> %8 : !neura.data !neura.data +// CTRL2DATA-NEXT: "neura.return"() : () -> () +// CTRL2DATA-NEXT: } \ No newline at end of file diff --git a/test/affine2neura/bert/bert_node28/bert_node28.mlir b/test/affine2neura/bert/bert_node28/bert_node28.mlir index e93de764..d73589b7 100644 --- a/test/affine2neura/bert/bert_node28/bert_node28.mlir +++ b/test/affine2neura/bert/bert_node28/bert_node28.mlir @@ -1,5 +1,6 @@ // RUN: mlir-opt %s --lower-affine --convert-scf-to-cf --convert-cf-to-llvm -o %t-llvm.mlir // RUN: mlir-neura-opt %t-llvm.mlir --assign-accelerator --lower-arith-to-neura --lower-memref-to-neura --lower-builtin-to-neura --lower-llvm-to-neura | FileCheck %s +// RUN: mlir-neura-opt %t-llvm.mlir --assign-accelerator --lower-arith-to-neura --lower-memref-to-neura --lower-builtin-to-neura --lower-llvm-to-neura --leverage-predicated-value --transform-ctrl-to-data-flow | FileCheck %s -check-prefix=CTRL2DATA module attributes {} { func.func @_Z11bert_node28PA128_A768_KfPA768_S0_PA128_A768_f(%arg0: memref, %arg1: memref, %arg2: memref) attributes {} { affine.for %arg3 = 0 to 128 { @@ -62,3 +63,61 @@ module attributes {} { // CHECK-NEXT: neura.br %26 : i64 to ^bb1 // CHECK-NEXT: ^bb9: // pred: ^bb1 // CHECK-NEXT: "neura.return"() : () -> () + + +// CTRL2DATA: func.func @_Z11bert_node28PA128_A768_KfPA768_S0_PA128_A768_f(%arg0: memref, %arg1: memref, %arg2: memref) attributes {accelerator = "neura"} { +// CTRL2DATA-NEXT: %0 = "neura.constant"() <{value = 768 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %1 = "neura.grant_always"(%0) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %2 = "neura.constant"() <{value = 1 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %3 = "neura.grant_always"(%2) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %4 = "neura.constant"() <{value = 128 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %5 = "neura.grant_always"(%4) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %6 = "neura.constant"() <{value = 0 : index}> : () -> !neura.data +// CTRL2DATA-NEXT: %7 = "neura.grant_always"(%6) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %8 = "neura.cast"(%6) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %9 = "neura.grant_once"(%8) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %10 = neura.reserve : !neura.data +// CTRL2DATA-NEXT: %11 = "neura.phi"(%10, %9) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %12 = "neura.cast"(%11) <{cast_type = "int_to_index"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %13 = "neura.icmp"(%12, %5) <{cmpType = "slt"}> : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %14 = "neura.not"(%13) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %15 = neura.grant_predicate %7, %13 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %16 = "neura.cast"(%15) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %17 = neura.reserve : !neura.data +// CTRL2DATA-NEXT: %18 = "neura.phi"(%17, %16) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %19 = "neura.cast"(%18) <{cast_type = "int_to_index"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %20 = "neura.icmp"(%19, %1) <{cmpType = "slt"}> : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %21 = "neura.not"(%20) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %22 = neura.grant_predicate %7, %20 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %23 = "neura.cast"(%22) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %24 = neura.reserve : !neura.data +// CTRL2DATA-NEXT: %25 = "neura.phi"(%24, %23) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %26 = "neura.cast"(%25) <{cast_type = "int_to_index"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %27 = "neura.icmp"(%26, %1) <{cmpType = "slt"}> : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %28 = "neura.not"(%27) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[VAL1:.*]] = neura.grant_predicate %26, %27 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL2:.*]] = neura.grant_predicate %12, %27 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL3:.*]] = neura.grant_predicate %7, %27 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %32 = neura.load_indexed %arg0[%[[VAL3:.*]], %[[VAL2:.*]], %[[VAL1:.*]] : !neura.data, !neura.data, !neura.data] memref : !neura.data +// CTRL2DATA-NEXT: %33 = neura.grant_predicate %19, %27 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %34 = neura.load_indexed %arg1[%[[VAL3:.*]], %[[VAL1:.*]], %33 : !neura.data, !neura.data, !neura.data] memref : !neura.data +// CTRL2DATA-NEXT: %35 = neura.load_indexed %arg2[%[[VAL3:.*]], %[[VAL2:.*]], %33 : !neura.data, !neura.data, !neura.data] memref : !neura.data +// CTRL2DATA-NEXT: %36 = "neura.fmul"(%32, %34) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %37 = "neura.fadd"(%35, %36) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: neura.store_indexed %37 to %arg2[%[[VAL3:.*]], %[[VAL2:.*]], %33 : !neura.data, !neura.data, !neura.data] memref : !neura.data +// CTRL2DATA-NEXT: %38 = neura.grant_predicate %3, %27 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %39 = "neura.add"(%[[VAL1:.*]], %38) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %40 = "neura.cast"(%39) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: neura.ctrl_mov %40 -> %24 : !neura.data !neura.data +// CTRL2DATA-DAG: %[[VAL4:.*]] = neura.grant_predicate %19, %28 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL5:.*]] = neura.grant_predicate %3, %28 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %43 = "neura.add"(%[[VAL4:.*]], %[[VAL5:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %44 = "neura.cast"(%43) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: neura.ctrl_mov %44 -> %17 : !neura.data !neura.data +// CTRL2DATA-DAG: %[[VAL6:.*]] = neura.grant_predicate %3, %21 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL7:.*]] = neura.grant_predicate %12, %21 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %47 = "neura.add"(%[[VAL7:.*]], %[[VAL6:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %48 = "neura.cast"(%47) <{cast_type = "index_to_int"}> : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: neura.ctrl_mov %48 -> %10 : !neura.data !neura.data +// CTRL2DATA-NEXT: "neura.return"() : () -> () +// CTRL2DATA-NEXT: } \ No newline at end of file diff --git a/test/neura/ctrl/branch.mlir b/test/neura/ctrl/branch.mlir index 572738b6..cff2666d 100644 --- a/test/neura/ctrl/branch.mlir +++ b/test/neura/ctrl/branch.mlir @@ -63,14 +63,13 @@ func.func @test(%in: i64) -> f32 { // CTRL2DATA-NEXT: %8 = "neura.grant_once"(%7) : (!neura.data) -> !neura.data // CTRL2DATA-NEXT: %9 = "neura.icmp"(%arg0, %0) <{cmpType = "eq"}> : (i64, !neura.data) -> !neura.data // CTRL2DATA-NEXT: %10 = "neura.grant_once"(%9) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %11 = neura.grant_predicate %6, %10 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %12 = neura.grant_predicate %8, %10 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %13 = "neura.not"(%10) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %14 = neura.grant_predicate %2, %13 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %15 = "neura.not"(%10) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %16 = neura.grant_predicate %4, %15 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %17 = "neura.fadd"(%14, %16) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %18 = "neura.fmul"(%11, %12) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %19 = "neura.phi"(%17, %18) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: "neura.return"(%19) : (!neura.data) -> () +// CTRL2DATA-DAG: %[[TRUE_VAL1:.*]] = neura.grant_predicate %6, %10 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[TRUE_VAL2:.*]] = neura.grant_predicate %8, %10 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[NOT:.*]] = "neura.not"(%10) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[FALSE_VAL1:.*]] = neura.grant_predicate %2, %[[NOT:.*]] : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[FALSE_VAL2:.*]] = neura.grant_predicate %4, %[[NOT:.*]] : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %16 = "neura.fadd"(%[[FALSE_VAL1:.*]], %[[FALSE_VAL2:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %17 = "neura.fmul"(%[[TRUE_VAL1:.*]], %[[TRUE_VAL2:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %18 = "neura.phi"(%16, %17) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: "neura.return"(%18) : (!neura.data) -> () // CTRL2DATA-NEXT: } \ No newline at end of file diff --git a/test/neura/ctrl/branch_for.mlir b/test/neura/ctrl/branch_for.mlir index e5a57b8b..ce20d515 100644 --- a/test/neura/ctrl/branch_for.mlir +++ b/test/neura/ctrl/branch_for.mlir @@ -11,33 +11,34 @@ // RUN: --transform-ctrl-to-data-flow \ // RUN: | FileCheck %s -check-prefix=CTRL2DATA -// RUN: mlir-neura-opt %s \ -// RUN: --assign-accelerator \ -// RUN: --lower-llvm-to-neura \ -// RUN: --leverage-predicated-value \ -// RUN: --transform-ctrl-to-data-flow \ -// RUN: --insert-data-mov \ -// RUN: | FileCheck %s -check-prefix=MOV - -// RUN: mlir-neura-opt %s \ -// RUN: --assign-accelerator \ -// RUN: --lower-llvm-to-neura \ -// RUN: --leverage-predicated-value \ -// RUN: --transform-ctrl-to-data-flow \ -// RUN: --insert-data-mov \ -// RUN: --map-to-accelerator \ -// RUN: | FileCheck %s -check-prefix=MAPPING +// TODO: Enable the following tests once the ctrl2data is refactored. +// RU: mlir-neura-opt %s \ +// RU: --assign-accelerator \ +// RU: --lower-llvm-to-neura \ +// RU: --leverage-predicated-value \ +// RU: --transform-ctrl-to-data-flow \ +// RU: --insert-data-mov \ +// RU: | FileCheck %s -check-prefix=MOV -// RUN: mlir-neura-opt %s \ -// RUN: --assign-accelerator \ -// RUN: --lower-llvm-to-neura \ -// RUN: --leverage-predicated-value \ -// RUN: --transform-ctrl-to-data-flow \ -// RUN: --insert-data-mov \ -// RUN: --map-to-accelerator \ -// RUN: --generate-code +// RU: mlir-neura-opt %s \ +// RU: --assign-accelerator \ +// RU: --lower-llvm-to-neura \ +// RU: --leverage-predicated-value \ +// RU: --transform-ctrl-to-data-flow \ +// RU: --insert-data-mov \ +// RU: --map-to-accelerator \ +// RU: | FileCheck %s -check-prefix=MAPPING -// RUN: FileCheck %s --input-file=generated-instructions.json -check-prefix=INST +// RU: mlir-neura-opt %s \ +// RU: --assign-accelerator \ +// RU: --lower-llvm-to-neura \ +// RU: --leverage-predicated-value \ +// RU: --transform-ctrl-to-data-flow \ +// RU: --insert-data-mov \ +// RU: --map-to-accelerator \ +// RU: --generate-code + +// RU: FileCheck %s --input-file=generated-instructions.json -check-prefix=INST func.func @loop_test() -> f32 { %n = llvm.mlir.constant(10 : i64) : i64 @@ -78,27 +79,27 @@ func.func @loop_test() -> f32 { // CTRL2DATA-NEXT: %0 = "neura.constant"() <{predicate = true, value = 10 : i64}> : () -> !neura.data // CTRL2DATA-NEXT: %1 = "neura.grant_always"(%0) : (!neura.data) -> !neura.data // CTRL2DATA-NEXT: %2 = "neura.constant"() <{predicate = true, value = 0 : i64}> : () -> !neura.data -// CTRL2DATA-NEXT: %3 = "neura.grant_once"(%2) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %[[INT1:.*]] = "neura.grant_once"(%2) : (!neura.data) -> !neura.data // CTRL2DATA-NEXT: %4 = "neura.constant"() <{predicate = true, value = 1 : i64}> : () -> !neura.data // CTRL2DATA-NEXT: %5 = "neura.grant_always"(%4) : (!neura.data) -> !neura.data // CTRL2DATA-NEXT: %6 = "neura.constant"() <{predicate = true, value = 3.000000e+00 : f32}> : () -> !neura.data // CTRL2DATA-NEXT: %7 = "neura.grant_always"(%6) : (!neura.data) -> !neura.data // CTRL2DATA-NEXT: %8 = "neura.constant"() <{predicate = true, value = 0.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %9 = "neura.grant_once"(%8) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %10 = neura.reserve : !neura.data -// CTRL2DATA-NEXT: %11 = "neura.phi"(%3, %10) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %12 = neura.reserve : !neura.data -// CTRL2DATA-NEXT: %13 = "neura.phi"(%9, %12) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %14 = "neura.fadd"(%13, %7) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %15 = "neura.add"(%11, %5) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %[[FLOAT1:.*]] = "neura.grant_once"(%8) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[RESERVEINT:.*]] = neura.reserve : !neura.data +// CTRL2DATA-DAG: %[[PHIINT:.*]] = "neura.phi"(%[[RESERVEINT:.*]], %[[INT1:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[RESERVEFLOAT:.*]] = neura.reserve : !neura.data +// CTRL2DATA-DAG: %[[PHIFLOAT:.*]] = "neura.phi"(%[[RESERVEFLOAT:.*]], %[[FLOAT1:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %[[FLOAT2:.*]] = "neura.fadd"(%[[PHIFLOAT:.*]], %7) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %[[INT2:.*]] = "neura.add"(%[[PHIINT:.*]], %5) : (!neura.data, !neura.data) -> !neura.data // CTRL2DATA-NEXT: %16 = "neura.icmp"(%15, %1) <{cmpType = "slt"}> : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %17 = "neura.not"(%16) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %18 = neura.grant_predicate %14, %17 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %19 = neura.grant_predicate %14, %16 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: neura.ctrl_mov %19 -> %12 : !neura.data !neura.data -// CTRL2DATA-NEXT: %20 = neura.grant_predicate %15, %16 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: neura.ctrl_mov %20 -> %10 : !neura.data !neura.data -// CTRL2DATA-NEXT: "neura.return"(%18) : (!neura.data) -> () +// CTRL2DATA-DAG: %[[GRANTFLOAT:.*]] = neura.grant_predicate %[[FLOAT2:.*]], %16 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: neura.ctrl_mov %[[GRANTFLOAT:.*]] -> %[[RESERVEFLOAT:.*]] : !neura.data !neura.data +// CTRL2DATA-DAG: %[[GRANTINT:.*]] = neura.grant_predicate %[[INT2:.*]], %16 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: neura.ctrl_mov %[[GRANTINT:.*]] -> %[[RESERVEINT:.*]] : !neura.data !neura.data +// CTRL2DATA-NEXT: %19 = "neura.not"(%16) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %20 = neura.grant_predicate %14, %19 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: "neura.return"(%20) : (!neura.data) -> () // CTRL2DATA-NEXT: } // MOV: func.func @loop_test() -> f32 attributes {accelerator = "neura"} { diff --git a/test/neura/ctrl/branch_with_and_without_arg.mlir b/test/neura/ctrl/branch_with_and_without_arg.mlir index 04bc1751..a3293386 100644 --- a/test/neura/ctrl/branch_with_and_without_arg.mlir +++ b/test/neura/ctrl/branch_with_and_without_arg.mlir @@ -49,24 +49,22 @@ func.func @test(%in: i64) -> f32 { // CHECK-NEXT: } // CTRL2DATA: func.func @test(%arg0: i64) -> f32 attributes {accelerator = "neura"} { -// CTRL2DATA-NEXT: %0 = "neura.constant"() <{predicate = true, value = 0 : i64}> : () -> !neura.data -// CTRL2DATA-NEXT: %1 = "neura.constant"() <{predicate = true, value = 1.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %2 = "neura.grant_once"(%1) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %3 = "neura.constant"() <{predicate = true, value = 2.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %4 = "neura.grant_always"(%3) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %5 = "neura.grant_once"(%3) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %6 = "neura.constant"() <{predicate = true, value = 3.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %7 = "neura.grant_once"(%6) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %8 = "neura.icmp"(%arg0, %0) <{cmpType = "eq"}> : (i64, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %9 = "neura.grant_once"(%8) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %10 = neura.grant_predicate %7, %9 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %11 = neura.grant_predicate %4, %9 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %12 = "neura.not"(%9) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %13 = neura.grant_predicate %2, %12 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %14 = "neura.not"(%9) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %15 = neura.grant_predicate %5, %14 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %16 = "neura.fadd"(%13, %15) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %17 = "neura.fmul"(%10, %11) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %18 = "neura.phi"(%16, %17) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: "neura.return"(%18) : (!neura.data) -> () -// CTRL2DATA-NEXT: } +// CTRL2DATA-NEXT: %0 = "neura.constant"() <{predicate = true, value = 0 : i64}> : () -> !neura.data +// CTRL2DATA-NEXT: %1 = "neura.constant"() <{predicate = true, value = 1.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %2 = "neura.grant_once"(%1) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %3 = "neura.constant"() <{predicate = true, value = 2.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %4 = "neura.grant_always"(%3) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %5 = "neura.grant_once"(%3) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %6 = "neura.constant"() <{predicate = true, value = 3.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %7 = "neura.grant_once"(%6) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %8 = "neura.icmp"(%arg0, %0) <{cmpType = "eq"}> : (i64, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %9 = "neura.grant_once"(%8) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[VAL1:.*]] = neura.grant_predicate %7, %9 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[NOT:.*]] = "neura.not"(%9) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[VAL2:.*]] = neura.grant_predicate %2, %[[NOT:.*]] : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL3:.*]] = neura.grant_predicate %5, %[[NOT:.*]] : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %14 = "neura.fadd"(%[[VAL2:.*]], %[[VAL3:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %15 = "neura.fmul"(%[[VAL1:.*]], %4) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %16 = "neura.phi"(%14, %15) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: "neura.return"(%16) : (!neura.data) -> () +// CTRL2DATA-NEXT: } \ No newline at end of file diff --git a/test/neura/ctrl/branch_without_arg.mlir b/test/neura/ctrl/branch_without_arg.mlir index 131753d4..08f458c1 100644 --- a/test/neura/ctrl/branch_without_arg.mlir +++ b/test/neura/ctrl/branch_without_arg.mlir @@ -51,25 +51,24 @@ func.func @test(%in: i64) -> f32 { // CHECK-NEXT: } // CTRL2DATA: func.func @test(%arg0: i64) -> f32 attributes {accelerator = "neura"} { -// CTRL2DATA-NEXT: %0 = "neura.constant"() <{predicate = true, value = 0 : i64}> : () -> !neura.data -// CTRL2DATA-NEXT: %1 = "neura.constant"() <{predicate = true, value = 1.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %2 = "neura.grant_always"(%1) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %3 = "neura.constant"() <{predicate = true, value = 2.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %4 = "neura.grant_always"(%3) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %5 = "neura.constant"() <{predicate = true, value = 3.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %6 = "neura.grant_once"(%5) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %7 = "neura.constant"() <{predicate = true, value = 4.000000e+00 : f32}> : () -> !neura.data -// CTRL2DATA-NEXT: %8 = "neura.grant_once"(%7) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %9 = "neura.icmp"(%arg0, %0) <{cmpType = "eq"}> : (i64, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %10 = "neura.grant_once"(%9) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %11 = neura.grant_predicate %6, %10 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %12 = neura.grant_predicate %8, %10 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %13 = "neura.not"(%10) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %14 = neura.grant_predicate %2, %13 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %15 = "neura.not"(%10) : (!neura.data) -> !neura.data -// CTRL2DATA-NEXT: %16 = neura.grant_predicate %4, %15 : !neura.data, !neura.data -> !neura.data -// CTRL2DATA-NEXT: %17 = "neura.fadd"(%14, %16) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %18 = "neura.fmul"(%11, %12) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: %19 = "neura.phi"(%17, %18) : (!neura.data, !neura.data) -> !neura.data -// CTRL2DATA-NEXT: "neura.return"(%19) : (!neura.data) -> () -// CTRL2DATA-NEXT: } +// CTRL2DATA-NEXT: %0 = "neura.constant"() <{predicate = true, value = 0 : i64}> : () -> !neura.data +// CTRL2DATA-NEXT: %1 = "neura.constant"() <{predicate = true, value = 1.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %2 = "neura.grant_always"(%1) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %3 = "neura.constant"() <{predicate = true, value = 2.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %4 = "neura.grant_always"(%3) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %5 = "neura.constant"() <{predicate = true, value = 3.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %6 = "neura.grant_once"(%5) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %7 = "neura.constant"() <{predicate = true, value = 4.000000e+00 : f32}> : () -> !neura.data +// CTRL2DATA-NEXT: %8 = "neura.grant_once"(%7) : (!neura.data) -> !neura.data +// CTRL2DATA-NEXT: %9 = "neura.icmp"(%arg0, %0) <{cmpType = "eq"}> : (i64, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %10 = "neura.grant_once"(%9) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[VAL1:.*]] = neura.grant_predicate %6, %10 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL2:.*]] = neura.grant_predicate %8, %10 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %13 = "neura.not"(%10) : (!neura.data) -> !neura.data +// CTRL2DATA-DAG: %[[VAL3:.*]] = neura.grant_predicate %2, %13 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-DAG: %[[VAL4:.*]] = neura.grant_predicate %4, %13 : !neura.data, !neura.data -> !neura.data +// CTRL2DATA-NEXT: %16 = "neura.fadd"(%[[VAL3:.*]], %[[VAL4:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %17 = "neura.fmul"(%[[VAL1:.*]], %[[VAL2:.*]]) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: %18 = "neura.phi"(%16, %17) : (!neura.data, !neura.data) -> !neura.data +// CTRL2DATA-NEXT: "neura.return"(%18) : (!neura.data) -> () +// CTRL2DATA-NEXT: } \ No newline at end of file