From b6635f6de2cb476715b57d620222d23be216acf5 Mon Sep 17 00:00:00 2001 From: hjchen2 Date: Mon, 23 Aug 2021 21:43:32 +0800 Subject: [PATCH 1/4] Remove python function wrapper. --- oneflow/api/python/framework/doc.cpp | 44 ++++++++++++++++++ python/oneflow/F/__init__.py | 16 +++++++ python/oneflow/__init__.py | 6 +-- python/oneflow/framework/docstr/utils.py | 9 +--- python/oneflow/framework/functional.py | 45 ------------------- .../framework/register_class_method_util.py | 2 - tools/generate_functional_api.py | 4 ++ 7 files changed, 68 insertions(+), 58 deletions(-) create mode 100644 oneflow/api/python/framework/doc.cpp delete mode 100644 python/oneflow/framework/functional.py diff --git a/oneflow/api/python/framework/doc.cpp b/oneflow/api/python/framework/doc.cpp new file mode 100644 index 00000000000..ea82372823a --- /dev/null +++ b/oneflow/api/python/framework/doc.cpp @@ -0,0 +1,44 @@ +/* +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include + +#include "oneflow/api/python/of_api_registry.h" +#include "oneflow/api/python/framework/throw.h" + +namespace py = pybind11; + +namespace oneflow { + +py::object AddFunctionDoc(const py::object& f, const std::string& doc_string) { + static std::vector all_doc_strings; + all_doc_strings.emplace_back(doc_string); + const char* doc_str = all_doc_strings.back().c_str(); + PyObject* fptr = f.ptr(); + if (PyCFunction_Check(fptr)) { + auto* f = (PyCFunctionObject*)fptr; + if (f->m_ml->ml_doc) { + THROW(RuntimeError) << "function " << f->m_ml->ml_name << " already has a docstring."; + } + f->m_ml->ml_doc = doc_str; + } else { + THROW(RuntimeError) << "function is " << Py_TYPE(fptr)->tp_name << ", not a valid PyCFunction."; + } + return f; +} + +} // namespace oneflow + +ONEFLOW_API_PYBIND11_MODULE("", m) { m.def("add_doc", &oneflow::AddFunctionDoc); } diff --git a/python/oneflow/F/__init__.py b/python/oneflow/F/__init__.py index e69de29bb2d..419121d74e9 100644 --- a/python/oneflow/F/__init__.py +++ b/python/oneflow/F/__init__.py @@ -0,0 +1,16 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from oneflow._oneflow_internal.F import * diff --git a/python/oneflow/__init__.py b/python/oneflow/__init__.py index f001b0e32ef..775bef25fbd 100644 --- a/python/oneflow/__init__.py +++ b/python/oneflow/__init__.py @@ -107,12 +107,10 @@ def _SyncOnMasterFn(): atexit.register(_SyncOnMasterFn) del atexit del oneflow + +import oneflow.F import oneflow.framework.docstr as docstr -from oneflow.framework.docstr.utils import register_docstr -register_docstr() -del register_docstr -del docstr from oneflow.autograd import grad_enable, no_grad, inference_mode, is_grad_enabled import oneflow.nn.image import oneflow.nn.modules.acosh diff --git a/python/oneflow/framework/docstr/utils.py b/python/oneflow/framework/docstr/utils.py index 8cfc557af15..ba7b46f9e56 100644 --- a/python/oneflow/framework/docstr/utils.py +++ b/python/oneflow/framework/docstr/utils.py @@ -13,13 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. """ -_function_docstr = {} +import oneflow._oneflow_internal def add_docstr(fun, docstr: str): - _function_docstr[fun] = docstr - - -def register_docstr(): - for (fun, docstr) in _function_docstr.items(): - setattr(fun, "__doc__", docstr) + oneflow._oneflow_internal.add_doc(fun, docstr) diff --git a/python/oneflow/framework/functional.py b/python/oneflow/framework/functional.py deleted file mode 100644 index 95e7cdc1e26..00000000000 --- a/python/oneflow/framework/functional.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Copyright 2020 The OneFlow Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import oneflow as flow -import oneflow._oneflow_internal - - -class Function: - def __init__(self, func_name, handle): - self.func_name = func_name - self.handle = handle - - def __call__(self, *args, **kwargs): - return self.handle(*args, **kwargs) - - -def RegisterFunctionalApis(): - import inspect - - import oneflow.F - - for s in dir(oneflow._oneflow_internal.F): - f = getattr(oneflow._oneflow_internal.F, s) - if inspect.isbuiltin(f): - func_name = s - if s in _function_name_aliases: - func_name = _function_name_aliases[s] - setattr(oneflow.F, func_name, Function(func_name, f)) - setattr(oneflow.F, s, Function(func_name, f)) - del inspect - - -_function_name_aliases = {"add_scalar": "scalar_add"} diff --git a/python/oneflow/framework/register_class_method_util.py b/python/oneflow/framework/register_class_method_util.py index dd53ba0f87e..fce74ed0ca5 100644 --- a/python/oneflow/framework/register_class_method_util.py +++ b/python/oneflow/framework/register_class_method_util.py @@ -14,7 +14,6 @@ limitations under the License. """ import oneflow._oneflow_internal -import oneflow.framework.functional as functional import oneflow.framework.generator as generator import oneflow.framework.op_expr_util as op_expr_util import oneflow.framework.tensor as tensor_util @@ -23,4 +22,3 @@ def RegisterMethod4Class(): tensor_util.RegisterMethods() op_expr_util.RegisterMethod4UserOpExpr() - functional.RegisterFunctionalApis() diff --git a/tools/generate_functional_api.py b/tools/generate_functional_api.py index a9195c0ba0b..0796473404d 100644 --- a/tools/generate_functional_api.py +++ b/tools/generate_functional_api.py @@ -121,7 +121,11 @@ namespace functional = one::functional; ONEFLOW_API_PYBIND11_MODULE("F", m) {{ + py::options options; + options.disable_function_signatures(); + {1} + options.enable_function_signatures(); }} }} // namespace oneflow From 5391b82873e0b73eeca777e3efbdc82798a3c00d Mon Sep 17 00:00:00 2001 From: hjchen2 Date: Mon, 23 Aug 2021 22:36:05 +0800 Subject: [PATCH 2/4] Increase ref --- oneflow/api/python/framework/doc.cpp | 11 ++++++----- python/oneflow/framework/docstr/utils.py | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/oneflow/api/python/framework/doc.cpp b/oneflow/api/python/framework/doc.cpp index ea82372823a..9ee66b4f3de 100644 --- a/oneflow/api/python/framework/doc.cpp +++ b/oneflow/api/python/framework/doc.cpp @@ -22,20 +22,21 @@ namespace py = pybind11; namespace oneflow { -py::object AddFunctionDoc(const py::object& f, const std::string& doc_string) { +py::object AddFunctionDoc(py::object f, const std::string& doc_string) { static std::vector all_doc_strings; all_doc_strings.emplace_back(doc_string); const char* doc_str = all_doc_strings.back().c_str(); - PyObject* fptr = f.ptr(); - if (PyCFunction_Check(fptr)) { - auto* f = (PyCFunctionObject*)fptr; + PyObject* obj = f.ptr(); + if (PyCFunction_Check(obj)) { + auto* f = (PyCFunctionObject*)obj; if (f->m_ml->ml_doc) { THROW(RuntimeError) << "function " << f->m_ml->ml_name << " already has a docstring."; } f->m_ml->ml_doc = doc_str; } else { - THROW(RuntimeError) << "function is " << Py_TYPE(fptr)->tp_name << ", not a valid PyCFunction."; + THROW(RuntimeError) << "function is " << Py_TYPE(obj)->tp_name << ", not a valid PyCFunction."; } + f.inc_ref(); return f; } diff --git a/python/oneflow/framework/docstr/utils.py b/python/oneflow/framework/docstr/utils.py index ba7b46f9e56..41931071e8a 100644 --- a/python/oneflow/framework/docstr/utils.py +++ b/python/oneflow/framework/docstr/utils.py @@ -16,5 +16,6 @@ import oneflow._oneflow_internal + def add_docstr(fun, docstr: str): oneflow._oneflow_internal.add_doc(fun, docstr) From e7fd28846df550cd00b0abe77c5f753b88e4f5c1 Mon Sep 17 00:00:00 2001 From: hjchen2 Date: Tue, 24 Aug 2021 13:40:59 +0800 Subject: [PATCH 3/4] Fix doc test --- python/oneflow/test/modules/test_functional_docstr.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/oneflow/test/modules/test_functional_docstr.py b/python/oneflow/test/modules/test_functional_docstr.py index 0634db793ca..fdb4a4961ca 100644 --- a/python/oneflow/test/modules/test_functional_docstr.py +++ b/python/oneflow/test/modules/test_functional_docstr.py @@ -22,11 +22,6 @@ import oneflow as flow import oneflow.unittest -from oneflow.framework.functional import Function - - -def _is_oneflow_functional(object): - return isinstance(object, Function) def _run_functional_doctest( @@ -44,7 +39,7 @@ def _run_functional_doctest( runner = doctest.DebugRunner(verbose=verbose, optionflags=optionflags) else: runner = doctest.DocTestRunner(verbose=verbose, optionflags=optionflags) - r = inspect.getmembers(flow.F, _is_oneflow_functional) + r = inspect.getmembers(flow.F) for (name, fun) in r: if fun.__doc__ is not None: print("test on docstr of: ", ".".join([module.__name__, name])) From 48c98065fdd792bc25e58cdb835ad3c13910da0c Mon Sep 17 00:00:00 2001 From: Houjiang Chen Date: Tue, 24 Aug 2021 16:55:48 +0800 Subject: [PATCH 4/4] Update python/oneflow/framework/docstr/utils.py Co-authored-by: Yao Chi --- python/oneflow/framework/docstr/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/oneflow/framework/docstr/utils.py b/python/oneflow/framework/docstr/utils.py index 41931071e8a..301610c8d1e 100644 --- a/python/oneflow/framework/docstr/utils.py +++ b/python/oneflow/framework/docstr/utils.py @@ -18,4 +18,4 @@ def add_docstr(fun, docstr: str): - oneflow._oneflow_internal.add_doc(fun, docstr) + return oneflow._oneflow_internal.add_doc(fun, docstr)