diff --git a/graph_net/tools/init_input_tensor_constraints.sh b/graph_net/tools/init_input_tensor_constraints.sh index 736d5f21bb..42036e694a 100755 --- a/graph_net/tools/init_input_tensor_constraints.sh +++ b/graph_net/tools/init_input_tensor_constraints.sh @@ -30,6 +30,7 @@ python3 -m graph_net.model_path_handler \ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", diff --git a/graph_net/torch/dim_gen_passes/non_batch_call_method_view_pass.py b/graph_net/torch/dim_gen_passes/non_batch_call_method_view_pass.py new file mode 100644 index 0000000000..5e71c15029 --- /dev/null +++ b/graph_net/torch/dim_gen_passes/non_batch_call_method_view_pass.py @@ -0,0 +1,114 @@ +import torch.fx as fx +from graph_net.torch.dim_gen_passes import DimensionGeneralizationPass +from collections import namedtuple +import os + + +class ConcretePass(DimensionGeneralizationPass): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def get_pass_name(cls) -> str: + return os.path.basename(__file__)[:-3] + + def need_rewrite(self, traced_module: fx.GraphModule) -> bool: + if 0 in self.axes: + return False + if self.dim <= 1: + return False + return any(self._node_need_rewrite(node) for node in traced_module.graph.nodes) + + def _node_need_rewrite(self, node) -> bool: + if not (node.op == "call_method"): + return False + if not (node.target == "view"): + return False + if not (len(node.args) >= 2): + return False + view_args = node.args[1:] + if not any(arg == self.dim for arg in view_args): + return False + if -1 in view_args: + if len(view_args) == 2: + return True + return False + return True + + def rewrite(self, traced_module: fx.GraphModule) -> fx.GraphModule: + """ + Fx Pass: Dynamic the hard-coded non batch size in view ops. + """ + # Create a new graph to hold the rewritten nodes + new_graph = fx.Graph() + + # Create a map to link nodes from the old graph to nodes in the new graph + val_map = {} + + NodeAxis = namedtuple("NodeAxis", ["node", "shape_axis"]) + last_node_axis = None + + def try_reset_last_node_axis(node, new_node): + nonlocal last_node_axis + axis = _find_matching_axis(node.meta.get("tensor_meta")) + if axis is not None: + last_node_axis = NodeAxis(node=new_node, shape_axis=axis) + + def _find_matching_axis(node_meta): + if node_meta is None or not hasattr(node_meta, "shape"): + return None + for axis, dim in enumerate(node_meta.shape): + if dim == self.dim and axis > 0: + return axis + return None + + def _get_target_axis_info(node): + input_node = node.args[0] + axis = _find_matching_axis(input_node.meta.get("tensor_meta")) + if axis is not None: + new_input_node = val_map.get(input_node, input_node) + return NodeAxis(node=new_input_node, shape_axis=axis) + return None + + def create_new_node(node): + # Try to find the dimension from the input tensor itself first + target_axis_info = None + if self._node_need_rewrite(node): + target_axis_info = _get_target_axis_info(node) + + # Fallback to last_node_axis if not found in input tensor + if target_axis_info is None: + target_axis_info = last_node_axis + + if not (self._node_need_rewrite(node) and target_axis_info is not None): + # Copy other nodes to the new graph + new_node = new_graph.node_copy(node, lambda x: val_map[x]) + try_reset_last_node_axis(node=node, new_node=new_node) + return new_node + + def get_new_node_arg(arg): + if not (isinstance(arg, int) and arg == self.dim): + return val_map[arg] if arg in val_map else arg + + assert arg == self.dim + + # Use the size() method to retrieve the dynamic dimension + size_node = new_graph.call_method( + "size", + args=(target_axis_info.node, target_axis_info.shape_axis), + ) + return size_node + + new_view_args = tuple(get_new_node_arg(arg) for arg in node.args) + + # --- Rebuild the node --- + new_node = new_graph.call_method("view", args=new_view_args) + + return new_node + + for node in traced_module.graph.nodes: + val_map[node] = create_new_node(node) + + # Replace the old graph with the new graph and return + traced_module.graph = new_graph + traced_module.recompile() + return traced_module diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh16/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-base-nh16/graph_net.json index f0056fba10..e44d75f8ca 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh16/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh16/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh16/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-base-nh16/input_tensor_constraints.py index dc912e5297..785c83bcfd 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh16/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh16/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], @@ -45,360 +51,355 @@ ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1024, 768], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1024, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), ([768], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh24/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-base-nh24/graph_net.json index a5cd7c80dc..b7ea953501 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh24/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh24/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh24/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-base-nh24/input_tensor_constraints.py index 55a07b2cc4..ade7405325 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh24/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh24/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1536, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 24], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], @@ -45,360 +51,355 @@ ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 1536], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 768], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 1536], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), ([768], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh32/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-base-nh32/graph_net.json index b30e11dfe5..d397947943 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh32/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh32/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh32/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-base-nh32/input_tensor_constraints.py index df94d3817a..47be59f6de 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh32/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh32/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [2048, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 32], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], @@ -45,360 +51,355 @@ ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 2048], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 768], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 2048], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), ([768], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh8/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-base-nh8/graph_net.json index b2e57c07f5..9b428e48bb 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh8/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh8/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-base-nh8/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-base-nh8/input_tensor_constraints.py index 762590f459..0d25b5a477 100644 --- a/samples/transformers-auto-model/google/t5-efficient-base-nh8/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-base-nh8/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [512, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 8], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], @@ -45,360 +51,355 @@ ), ( [768], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [768, 512], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 768], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [768, 512], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [768], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [3072, 768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [768, 3072], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [768], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 768], "L_self_modules_embed_tokens_parameters_weight_"), ([768], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-dl12/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-dl12/graph_net.json index 838a1daa37..56ca4cd4b6 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-dl12/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-dl12/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-dl12/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-dl12/input_tensor_constraints.py index 69fc976bbf..16740d2815 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-dl12/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-dl12/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-el2/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-el2/graph_net.json index 0bc8a331b3..589faace4d 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-el2/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-el2/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-el2/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-el2/input_tensor_constraints.py index 4c5bd09f87..29200348f3 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-el2/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-el2/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,27 +51,27 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -75,10 +81,5 @@ [1024, 4096], "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ( - [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", - ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-el4/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-el4/graph_net.json index 5f66d18cc9..8be1490dd3 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-el4/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-el4/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-el4/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-el4/input_tensor_constraints.py index 92a19c75a9..3d84d9e061 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-el4/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-el4/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,27 +51,27 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -77,27 +83,27 @@ ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -109,27 +115,27 @@ ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -139,10 +145,5 @@ [1024, 4096], "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ( - [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", - ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-el6/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-el6/graph_net.json index 9b87302c14..f5799eb1af 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-el6/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-el6/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-el6/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-el6/input_tensor_constraints.py index 6735488d6d..49ce3efba9 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-el6/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-el6/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,27 +51,27 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -77,27 +83,27 @@ ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -109,27 +115,27 @@ ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -141,27 +147,27 @@ ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -173,27 +179,27 @@ ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -203,10 +209,5 @@ [1024, 4096], "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ( - [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", - ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv128/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-kv128/graph_net.json index a0d39e18ff..b6451a6b4a 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv128/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv128/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv128/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-kv128/input_tensor_constraints.py index 426a419618..cce2cfb8ae 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv128/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv128/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [2048, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv16/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-kv16/graph_net.json index 203f56dcea..7d2457032f 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv16/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv16/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv16/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-kv16/input_tensor_constraints.py index 0d2731741e..0e7b21405d 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv16/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv16/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [256, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv256/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-kv256/graph_net.json index be941f2e75..45992f4cea 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv256/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv256/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv256/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-kv256/input_tensor_constraints.py index 1cc63f0af3..eb81152bd6 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv256/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv256/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [4096, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [4096, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 4096], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv32/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-kv32/graph_net.json index 97eb1d85ee..90b3174e29 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv32/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv32/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-kv32/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-kv32/input_tensor_constraints.py index d65e52a3bd..d782a2b0c7 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-kv32/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-kv32/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh2/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nh2/graph_net.json index d30ed36530..bdd08898b1 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh2/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh2/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh2/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nh2/input_tensor_constraints.py index 476d85c1c9..00444a6e09 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh2/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh2/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [128, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 2], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 128], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [128, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [128, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [128, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 128], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh24/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nh24/graph_net.json index 390dc16e0f..9fb6bec671 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh24/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh24/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh24/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nh24/input_tensor_constraints.py index 4a4921c1b5..e355ce8304 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh24/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh24/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1536, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 24], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 1536], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [1536, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1536, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [1536, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 1536], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh32/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nh32/graph_net.json index 75f950de7f..24c0d8a147 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh32/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh32/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh32/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nh32/input_tensor_constraints.py index 5f07cd8bff..cdf172e9e9 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh32/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh32/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [2048, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 32], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 2048], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [2048, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [2048, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [2048, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 2048], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh4/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nh4/graph_net.json index ae711d67c2..6836aced93 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh4/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh4/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh4/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nh4/input_tensor_constraints.py index 6c4b0d7031..981277a28d 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh4/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh4/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [256, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 4], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 256], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [256, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [256, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [256, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 256], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/graph_net.json index d0e1f8259c..184603f511 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/input_tensor_constraints.py index b79b955588..32442be1d3 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh8-nl32/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 8], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,1000 +51,995 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_24_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_25_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_26_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_27_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_28_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_29_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_30_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_31_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh8/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nh8/graph_net.json index cc86b84165..ac497078b2 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh8/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh8/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nh8/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nh8/input_tensor_constraints.py index a94da07f58..d89386cbea 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nh8/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nh8/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 8], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,744 +51,739 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [1024, 512], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [512, 1024], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( - [512, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [1024, 512], + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nl10/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nl10/graph_net.json index 94026e7b7c..fd029085d3 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nl10/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nl10/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nl10/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nl10/input_tensor_constraints.py index 6aecbe4880..f2bb113c99 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nl10/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nl10/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,27 +51,27 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -77,27 +83,27 @@ ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -109,27 +115,27 @@ ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -141,27 +147,27 @@ ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -173,27 +179,27 @@ ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -205,27 +211,27 @@ ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -237,27 +243,27 @@ ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -269,27 +275,27 @@ ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -301,27 +307,27 @@ ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -331,10 +337,5 @@ [1024, 4096], "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", - ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nl20/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nl20/graph_net.json index 9060d19b98..07733a2efe 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nl20/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nl20/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nl20/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nl20/input_tensor_constraints.py index c627845522..14f0868dd0 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nl20/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nl20/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,616 +51,611 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nl32/graph_net.json b/samples/transformers-auto-model/google/t5-efficient-large-nl32/graph_net.json index 62c4533b40..44cf423f01 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nl32/graph_net.json +++ b/samples/transformers-auto-model/google/t5-efficient-large-nl32/graph_net.json @@ -27,11 +27,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-efficient-large-nl32/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-efficient-large-nl32/input_tensor_constraints.py index 519132cbe0..19709ca92a 100644 --- a/samples/transformers-auto-model/google/t5-efficient-large-nl32/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-efficient-large-nl32/input_tensor_constraints.py @@ -1,27 +1,33 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [1024], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 16], @@ -29,11 +35,11 @@ ), ( [1024, 1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], @@ -45,1000 +51,995 @@ ), ( [1024], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_10_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_11_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_12_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_13_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_14_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_15_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_24_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_16_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_24_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_25_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_17_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_25_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_26_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_18_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_26_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_27_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_19_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_27_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_28_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_20_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_28_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_29_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_21_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_29_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_22_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_30_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_23_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_30_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_31_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_24_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_31_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_25_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_26_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_27_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_28_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_29_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", + "L_self_modules_block_modules_30_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_8_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [1024, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [1024], - "L_self_modules_block_modules_9_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [4096, 1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wi_parameters_weight_", ), ( [1024, 4096], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", - ), - ( - [1024], - "L_self_modules_block_modules_9_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_31_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ([32128, 1024], "L_self_modules_embed_tokens_parameters_weight_"), ([1024], "L_self_modules_final_layer_norm_parameters_weight_"), ] diff --git a/samples/transformers-auto-model/google/t5-v1_1-small/graph_net.json b/samples/transformers-auto-model/google/t5-v1_1-small/graph_net.json index 8de4d70c68..dc3dd1cd4d 100644 --- a/samples/transformers-auto-model/google/t5-v1_1-small/graph_net.json +++ b/samples/transformers-auto-model/google/t5-v1_1-small/graph_net.json @@ -28,11 +28,13 @@ "naive_call_method_reshape_pass", "naive_call_method_expand_pass", "non_batch_call_method_expand_pass", + "non_batch_call_method_view_pass", "non_batch_call_function_arange_pass", "non_batch_call_function_getitem_slice_pass", "non_batch_call_function_full_pass", "non_batch_call_function_full_plus_one_pass", "non_batch_call_function_zeros_pass", "non_batch_call_function_arange_plus_one_pass" - ] + ], + "symbolic_dimension_reifier": "naive_nlp_sym_dim_reifier" } \ No newline at end of file diff --git a/samples/transformers-auto-model/google/t5-v1_1-small/input_tensor_constraints.py b/samples/transformers-auto-model/google/t5-v1_1-small/input_tensor_constraints.py index a341bb18f7..ee05fac4a3 100644 --- a/samples/transformers-auto-model/google/t5-v1_1-small/input_tensor_constraints.py +++ b/samples/transformers-auto-model/google/t5-v1_1-small/input_tensor_constraints.py @@ -1,39 +1,45 @@ from sympy import Symbol, Expr, Rel, Eq S0 = Symbol("S0") +S1 = Symbol("S1") -dynamic_dim_constraint_symbols = [S0] +dynamic_dim_constraint_symbols = [S0, S1] -dynamic_dim_constraint_symbol2example_value = {S0: 1} +dynamic_dim_constraint_symbol2example_value = {S0: 1, S1: 12} dynamic_dim_constraint_relations = [] dynamic_dim_constraint_input_shapes = [ - ([S0, 12], "L_attention_mask_"), - ([S0, 12], "L_input_ids_"), + ([S0, S1], "L_input_ids_"), + ([32128, 512], "L_self_modules_embed_tokens_parameters_weight_"), + ([S0, S1], "L_attention_mask_"), + ( + [512], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + ), ( [384, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( - [512, 384], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + [384, 512], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), ( [32, 6], "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_", ), ( - [384, 512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", + [512, 384], + "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", ), ( [512], - "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -49,27 +55,27 @@ ), ( [512], - "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -85,27 +91,27 @@ ), ( [512], - "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -121,27 +127,27 @@ ), ( [512], - "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -157,27 +163,27 @@ ), ( [512], - "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -193,27 +199,27 @@ ), ( [512], - "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -229,27 +235,27 @@ ), ( [512], - "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_6_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_6_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -265,27 +271,27 @@ ), ( [512], - "L_self_modules_block_modules_6_modules_layer_modules_1_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", - ), - ( - [512, 384], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", ), ( [384, 512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_", ), ( [384, 512], "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_", ), + ( + [512, 384], + "L_self_modules_block_modules_7_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_", + ), ( [512], - "L_self_modules_block_modules_7_modules_layer_modules_0_modules_layer_norm_parameters_weight_", + "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", ), ( [1024, 512], @@ -299,10 +305,5 @@ [512, 1024], "L_self_modules_block_modules_7_modules_layer_modules_1_modules_DenseReluDense_modules_wo_parameters_weight_", ), - ( - [512], - "L_self_modules_block_modules_7_modules_layer_modules_1_modules_layer_norm_parameters_weight_", - ), - ([32128, 512], "L_self_modules_embed_tokens_parameters_weight_"), ([512], "L_self_modules_final_layer_norm_parameters_weight_"), ]