diff --git a/.gitignore b/.gitignore index f1d02f4dd25c6..890aca6fa96ad 100644 --- a/.gitignore +++ b/.gitignore @@ -73,8 +73,8 @@ tools/nvcc_lazy # these files (directories) are generated before build system generation paddle/fluid/operators/generated_op.cc paddle/fluid/operators/generated_sparse_op.cc -paddle/phi/ops/compat/generated_sig.cc -paddle/phi/ops/compat/generated_sparse_sig.cc +paddle/fluid/operators/generated_static_op.cc +paddle/phi/ops/compat/generated_*.cc paddle/phi/api/yaml/parsed_apis/ paddle/fluid/operators/generator/parsed_ops/ paddle/fluid/pybind/tmp_eager_op_function_impl.h diff --git a/paddle/fluid/operators/generator/CMakeLists.txt b/paddle/fluid/operators/generator/CMakeLists.txt index 53cbb990dc1ee..62c11faadaf20 100644 --- a/paddle/fluid/operators/generator/CMakeLists.txt +++ b/paddle/fluid/operators/generator/CMakeLists.txt @@ -5,6 +5,7 @@ include(phi) set(op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/ops.yaml) set(legacy_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_ops.yaml) set(bw_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/backward.yaml) +set(static_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/static_ops.yaml) set(legacy_bw_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_backward.yaml) set(sparse_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_ops.yaml) @@ -29,10 +30,14 @@ set(parsed_op_dir ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator/parsed_ops) set(generated_op_path ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_op.cc) +set(generated_static_op_path + ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_static_op.cc) set(generated_sparse_ops_path ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_sparse_op.cc) set(generated_argument_mapping_path ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sig.cc) +set(generated_static_argument_mapping_path + ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_static_sig.cc) set(generated_sparse_argument_mapping_path ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sparse_sig.cc) @@ -54,6 +59,8 @@ execute_process( COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${legacy_bw_op_yaml_file} --output_path ./parsed_ops/legacy_backward_ops.parsed.yaml --backward + COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${static_op_yaml_file} + --output_path ./parsed_ops/static_ops.parsed.yaml COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${sparse_op_yaml_file} --output_path ./parsed_ops/sparse_ops.parsed.yaml COMMAND @@ -75,7 +82,8 @@ execute_process( COMMAND ${PYTHON_EXECUTABLE} cross_validate.py --forward_yaml_paths ./parsed_ops/ops.parsed.yaml ./parsed_ops/legacy_ops.parsed.yaml - --backward_yaml_paths ./parsed_ops/backward_ops.parsed.yaml + ./parsed_ops/static_ops.parsed.yaml --backward_yaml_paths + ./parsed_ops/backward_ops.parsed.yaml ./parsed_ops/legacy_backward_ops.parsed.yaml RESULT_VARIABLE _result) if(${_result}) @@ -113,6 +121,20 @@ if(${_result}) message(FATAL_ERROR "operator codegen failed, exiting.") endif() +execute_process( + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator + COMMAND + ${PYTHON_EXECUTABLE} generate_static_op.py --ops_yaml_path + ./parsed_ops/static_ops.parsed.yaml --op_version_yaml_path + ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/op_version.yaml + --op_compat_yaml_path ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/op_compat.yaml + --output_op_path "${generated_static_op_path}.tmp" --output_arg_map_path + "${generated_static_argument_mapping_path}.tmp" + RESULT_VARIABLE _result) +if(${_result}) + message(FATAL_ERROR "operator codegen failed, exiting.") +endif() + execute_process( WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator COMMAND @@ -126,84 +148,33 @@ if(${_result}) message(FATAL_ERROR "sparse operator codegen failed, exiting.") endif() -if(EXISTS "${generated_op_path}.tmp" AND EXISTS "${generated_op_path}") - execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${generated_op_path}.tmp" "${generated_op_path}") - message("copy if different ${generated_op_path}.tmp ${generated_op_path}") -elseif(EXISTS "${generated_op_path}.tmp") - execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_op_path}.tmp" - "${generated_op_path}") - message("copy ${generated_op_path}.tmp ${generated_op_path}") -else() - execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f "${generated_op_path}") - message("remove ${generated_op_path}") -endif() - -if(EXISTS "${generated_sparse_ops_path}.tmp" AND EXISTS - "${generated_sparse_ops_path}") - execute_process( - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${generated_sparse_ops_path}.tmp" "${generated_sparse_ops_path}") - message( - "copy if different ${generated_sparse_ops_path}.tmp ${generated_sparse_ops_path}" - ) -elseif(EXISTS "${generated_sparse_ops_path}.tmp") - execute_process( - COMMAND ${CMAKE_COMMAND} -E copy "${generated_sparse_ops_path}.tmp" - "${generated_sparse_ops_path}") - message("copy ${generated_sparse_ops_path}.tmp ${generated_sparse_ops_path}") -else() - execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f - "${generated_sparse_ops_path}") - message("remove ${generated_sparse_ops_path}") -endif() - -if(EXISTS "${generated_argument_mapping_path}.tmp" - AND EXISTS "${generated_argument_mapping_path}") - execute_process( - COMMAND - ${CMAKE_COMMAND} -E copy_if_different - "${generated_argument_mapping_path}.tmp" - "${generated_argument_mapping_path}") - message( - "copy if different ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}" - ) -elseif(EXISTS "${generated_argument_mapping_path}.tmp") - execute_process( - COMMAND ${CMAKE_COMMAND} -E copy "${generated_argument_mapping_path}.tmp" - "${generated_argument_mapping_path}") - message( - "copy ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}" - ) -else() - execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f - "${generated_argument_mapping_path}") - message("remove ${generated_argument_mapping_path}") -endif() - -if(EXISTS "${generated_sparse_argument_mapping_path}.tmp" - AND EXISTS "${generated_sparse_argument_mapping_path}") - execute_process( - COMMAND - ${CMAKE_COMMAND} -E copy_if_different - "${generated_sparse_argument_mapping_path}.tmp" - "${generated_sparse_argument_mapping_path}") - message( - "copy if different ${generated_sparse_argument_mapping_path}.tmp ${generated_sparse_argument_mapping_path}" - ) -elseif(EXISTS "${generated_sparse_argument_mapping_path}.tmp") - execute_process( - COMMAND - ${CMAKE_COMMAND} -E copy "${generated_sparse_argument_mapping_path}.tmp" - "${generated_sparse_argument_mapping_path}") - message( - "copy ${generated_sparse_argument_mapping_path}.tmp ${generated_sparse_argument_mapping_path}" - ) -else() - execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f - "${generated_sparse_argument_mapping_path}") - message("remove ${generated_sparse_argument_mapping_path}") -endif() +set(generated_static_files + "${generated_op_path}" + "${generated_static_op_path}" + "${generated_sparse_ops_path}" + "${generated_argument_mapping_path}" + "${generated_static_argument_mapping_path}" + "${generated_sparse_argument_mapping_path}") + +foreach(generated_static_file ${generated_static_files}) + if(EXISTS "${generated_static_file}.tmp" AND EXISTS + "${generated_static_file}") + execute_process( + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${generated_static_file}.tmp" "${generated_static_file}") + message( + "copy if different ${generated_static_file}.tmp ${generated_static_file}") + elseif(EXISTS "${generated_static_file}.tmp") + execute_process( + COMMAND ${CMAKE_COMMAND} -E copy "${generated_static_file}.tmp" + "${generated_static_file}") + message("copy ${generated_static_file}.tmp ${generated_static_file}") + else() + execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f + "${generated_static_file}") + message("remove ${generated_static_file}") + endif() +endforeach() # op extra info file set(ops_extra_info_gen_file diff --git a/paddle/fluid/operators/generator/generate_static_op.py b/paddle/fluid/operators/generator/generate_static_op.py new file mode 100644 index 0000000000000..b24e60dc4da1a --- /dev/null +++ b/paddle/fluid/operators/generator/generate_static_op.py @@ -0,0 +1,153 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from pathlib import Path + +import yaml +from filters import ( + cartesian_prod_mapping, + to_input_name, + to_int_array_tensor_name, + to_int_array_tensors_name, + to_op_attr_type, + to_opmaker_name, + to_opmaker_name_cstr, + to_pascal_case, + to_scalar_tensor_name, +) +from generate_op import replace_compat_name +from jinja2 import Environment, FileSystemLoader, StrictUndefined +from parse_utils import to_named_dict +from tests import ( + is_base_op, + is_initializer_list, + is_scalar, + is_vec, + supports_inplace, + supports_no_need_buffer, +) + +file_loader = FileSystemLoader(Path(__file__).parent / "templates") +env = Environment( + loader=file_loader, + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + extensions=['jinja2.ext.do'], +) +env.filters["to_op_attr_type"] = to_op_attr_type +env.filters["to_opmaker_name"] = to_opmaker_name +env.filters["to_pascal_case"] = to_pascal_case +env.filters["to_scalar_tensor_name"] = to_scalar_tensor_name +env.filters["to_int_array_tensor_name"] = to_int_array_tensor_name +env.filters["to_int_array_tensors_name"] = to_int_array_tensors_name +env.filters["to_input_name"] = to_input_name +env.filters["to_opmaker_name_cstr"] = to_opmaker_name_cstr +env.filters["cartesian_prod_mapping"] = cartesian_prod_mapping +env.tests["base_op"] = is_base_op +env.tests["vec"] = is_vec +env.tests["scalar"] = is_scalar +env.tests["initializer_list"] = is_initializer_list +env.tests["supports_inplace"] = supports_inplace +env.tests["supports_no_need_buffer"] = supports_no_need_buffer + + +def restruct_io(op): + op["input_dict"] = to_named_dict(op["inputs"]) + op["attr_dict"] = to_named_dict(op["attrs"]) + op["output_dict"] = to_named_dict(op["outputs"]) + return op + + +def main( + ops_yaml_path, + op_compat_yaml_path, + op_version_yaml_path, + output_op_path, + output_arg_map_path, +): + with open(ops_yaml_path, "rt") as f: + ops = yaml.safe_load(f) + ops = [restruct_io(op) for op in ops] + forward_op_dict = to_named_dict(ops) + + with open(op_version_yaml_path, "rt") as f: + op_versions = yaml.safe_load(f) + + # add op version info into op + for op_version in op_versions: + if op_version['op'] in forward_op_dict: + forward_op_dict[op_version['op']]['version'] = op_version['version'] + + with open(op_compat_yaml_path, "rt") as f: + op_op_map = yaml.safe_load(f) + + for op in ops: + op['op_name'] = op['name'] + + replace_compat_name(op_op_map, forward_op_dict, {}) + + if len(ops) == 0: + if os.path.isfile(output_op_path): + os.remove(output_op_path) + if os.path.isfile(output_arg_map_path): + os.remove(output_arg_map_path) + return + + op_template = env.get_template('op.c.j2') + with open(output_op_path, "wt") as f: + msg = op_template.render( + ops=ops, backward_ops=[], op_dict=forward_op_dict + ) + f.write(msg) + + ks_template = env.get_template('ks.c.j2') + with open(output_arg_map_path, 'wt') as f: + msg = ks_template.render(ops=ops, backward_ops=[]) + f.write(msg) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate operator file from op yaml." + ) + parser.add_argument( + '--ops_yaml_path', type=str, help="parsed static ops yaml file." + ) + parser.add_argument( + '--op_compat_yaml_path', type=str, help="ops args compat yaml file." + ) + parser.add_argument( + '--op_version_yaml_path', type=str, help="ops version yaml file." + ) + parser.add_argument( + "--output_op_path", type=str, help="path to save generated operators." + ) + parser.add_argument( + "--output_arg_map_path", + type=str, + help="path to save generated argument mapping functions.", + ) + + args = parser.parse_args() + main( + args.ops_yaml_path, + args.op_compat_yaml_path, + args.op_version_yaml_path, + args.output_op_path, + args.output_arg_map_path, + ) diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 0a991761e0fdc..32fe25624fe7b 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -1031,11 +1031,3 @@ kernel : func : unfold backward : unfold_grad - -- op: share_buffer - args : (Tensor[] x, bool[] share_dims_and_dtype={}) - output : Tensor[](out){x.size()}, Tensor[](xout){x.size()} - infer_meta : - func : ShareBufferInferMeta - kernel : - func : share_buffer diff --git a/paddle/phi/api/yaml/static_ops.yaml b/paddle/phi/api/yaml/static_ops.yaml new file mode 100644 index 0000000000000..1849b9f6c1e48 --- /dev/null +++ b/paddle/phi/api/yaml/static_ops.yaml @@ -0,0 +1,7 @@ +- op : share_buffer + args : (Tensor[] x, bool[] share_dims_and_dtype={}) + output : Tensor[](out){x.size()}, Tensor[](xout){x.size()} + infer_meta : + func : ShareBufferInferMeta + kernel : + func : share_buffer