migrate job_build_and_infer api to pybind11 (#3940)

* migrate job_build_and_infer api to pybind11

* add a blank line

* import JobBuildAndInferCfgError in c_api_util

* import JobBuildAndInferCfgError in c_api_util

* fix code format

* add macro switch

* default show cpp error stack frame

Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>

oneflow/api/python/job_build/job_build_and_infer.cpp

@@ -0,0 +1,79 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include <pybind11/pybind11.h>
+#include <string>
+#include "oneflow/api/python/of_api_registry.h"
+#include "oneflow/api/python/job_build/job_build_and_infer.h"
+
+namespace py = pybind11;
+
+ONEFLOW_API_PYBIND11_MODULE("", m) {
+  m.def("JobBuildAndInferCtx_Open", &JobBuildAndInferCtx_Open);
+  m.def("JobBuildAndInferCtx_GetCurrentJobName", &JobBuildAndInferCtx_GetCurrentJobName);
+  m.def("JobBuildAndInferCtx_Close", &JobBuildAndInferCtx_Close);
+
+  m.def("CurJobBuildAndInferCtx_CheckJob", &CurJobBuildAndInferCtx_CheckJob);
+  m.def("CurJobBuildAndInferCtx_SetJobConf", &CurJobBuildAndInferCtx_SetJobConf);
+  m.def("CurJobBuildAndInferCtx_SetTrainConf", &CurJobBuildAndInferCtx_SetTrainConf);
+
+  m.def("CurJobBuildAndInferCtx_Complete", &CurJobBuildAndInferCtx_Complete,
+        py::call_guard<py::gil_scoped_release>());
+  m.def("CurJobBuildAndInferCtx_HasJobConf", &CurJobBuildAndInferCtx_HasJobConf);
+  m.def("CurJobBuildAndInferCtx_AddAndInferMirroredOp",
+        &CurJobBuildAndInferCtx_AddAndInferMirroredOp, py::call_guard<py::gil_scoped_release>());
+
+  m.def("CurJobBuildAndInferCtx_AddAndInferConsistentOp",
+        &CurJobBuildAndInferCtx_AddAndInferConsistentOp);
+  m.def("JobBuildAndInferCtx_MirroredBlobGetSerializedParallelConfFromProducerView",
+        &JobBuildAndInferCtx_MirroredBlobGetSerializedParallelConfFromProducerView);
+  m.def("CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair",
+        &CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair);
+
+  m.def("JobBuildAndInferCtx_GetSerializedIdListAsStaticShape",
+        &JobBuildAndInferCtx_GetSerializedIdListAsStaticShape);
+  m.def("JobBuildAndInferCtx_GetDataType", &JobBuildAndInferCtx_GetDataType);
+  m.def("JobBuildAndInferCtx_IsDynamic", &JobBuildAndInferCtx_IsDynamic);
+
+  m.def("JobBuildAndInferCtx_DisableBoxing", &JobBuildAndInferCtx_DisableBoxing);
+  m.def("JobBuildAndInferCtx_IsTensorList", &JobBuildAndInferCtx_IsTensorList);
+  m.def("JobBuildAndInferCtx_GetBatchAxis", &JobBuildAndInferCtx_GetBatchAxis);
+
+  m.def("JobBuildAndInferCtx_GetSplitAxisFromProducerView",
+        &JobBuildAndInferCtx_GetSplitAxisFromProducerView);
+  m.def("JobBuildAndInferCtx_GetSerializedParallelConfFromProducerView",
+        &JobBuildAndInferCtx_GetSerializedParallelConfFromProducerView);
+  m.def("CurJobBuildAndInferCtx_AddLossLogicalBlobName",
+        &CurJobBuildAndInferCtx_AddLossLogicalBlobName);
+
+  m.def("JobBuildAndInferCtx_IsMirroredBlob", &JobBuildAndInferCtx_IsMirroredBlob);
+  m.def("JobBuildAndInferCtx_MirroredBlobGetNumSubLbi",
+        &JobBuildAndInferCtx_MirroredBlobGetNumSubLbi);
+  m.def("JobBuildAndInferCtx_MirroredBlobGetSerializedSubLbi",
+        &JobBuildAndInferCtx_MirroredBlobGetSerializedSubLbi);
+
+  m.def("JobBuildAndInferCtx_MirroredBlobGetSerializedIdListAsStaticShape",
+        &JobBuildAndInferCtx_MirroredBlobGetSerializedIdListAsStaticShape);
+  m.def("JobBuildAndInferCtx_MirroredBlobGetDataType",
+        &JobBuildAndInferCtx_MirroredBlobGetDataType);
+  m.def("JobBuildAndInferCtx_MirroredBlobIsDynamic", &JobBuildAndInferCtx_MirroredBlobIsDynamic);
+
+  m.def("JobBuildAndInferCtx_MirroredBlobIsTensorList",
+        &JobBuildAndInferCtx_MirroredBlobIsTensorList);
+  m.def("JobBuildAndInferCtx_MirroredBlobGetBatchAxis",
+        &JobBuildAndInferCtx_MirroredBlobGetBatchAxis);
+  m.def("JobBuildAndInferCtx_MirroredBlobGetSplitAxisFromProducerView",
+        &JobBuildAndInferCtx_MirroredBlobGetSplitAxisFromProducerView);
+}

oneflow/api/python/job_build/job_build_and_infer.h

@@ -0,0 +1,197 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_API_PYTHON_JOB_BUILD_JOB_BUILD_AND_INFER_H_
+#define ONEFLOW_API_PYTHON_JOB_BUILD_JOB_BUILD_AND_INFER_H_
+
+#include <utility>
+#include "oneflow/core/job/job_build_and_infer_ctx.h"
+#include "oneflow/core/record/record.pb.h"
+#include "oneflow/core/job/job_build_and_infer_ctx_mgr.h"
+#include "oneflow/core/job/job.pb.h"
+#include "oneflow/core/job/placement.pb.h"
+#include "oneflow/core/operator/op_conf.pb.h"
+#include "oneflow/api/python/job_build/job_build_and_infer_helper.h"
+
+std::shared_ptr<oneflow::cfg::ErrorProto> JobBuildAndInferCtx_Open(const std::string& job_name) {
+  return oneflow::JobBuildAndInferCtx_Open(job_name).GetDataAndErrorProto();
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_GetCurrentJobName() {
+  return oneflow::JobBuildAndInferCtx_GetCurrentJobName().GetDataAndErrorProto(std::string(""));
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> JobBuildAndInferCtx_Close() {
+  return oneflow::JobBuildAndInferCtx_Close().GetDataAndErrorProto();
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> CurJobBuildAndInferCtx_CheckJob() {
+  return oneflow::CurJobBuildAndInferCtx_CheckJob().GetDataAndErrorProto();
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> CurJobBuildAndInferCtx_SetJobConf(
+    const std::string& serialized_job_conf) {
+  return oneflow::CurJobBuildAndInferCtx_SetJobConf(serialized_job_conf).GetDataAndErrorProto();
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> CurJobBuildAndInferCtx_SetTrainConf(
+    const std::string& serialized_train_conf) {
+  return oneflow::CurJobBuildAndInferCtx_SetTrainConf(serialized_train_conf).GetDataAndErrorProto();
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> CurJobBuildAndInferCtx_Complete() {
+  return oneflow::CurJobBuildAndInferCtx_Complete().GetDataAndErrorProto();
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>> CurJobBuildAndInferCtx_HasJobConf() {
+  return oneflow::CurJobBuildAndInferCtx_HasJobConf().GetDataAndErrorProto(false);
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+CurJobBuildAndInferCtx_AddAndInferMirroredOp(const std::string& serialized_op_conf) {
+  return oneflow::CurJobBuildAndInferCtx_AddAndInferMirroredOp(serialized_op_conf)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+CurJobBuildAndInferCtx_AddAndInferConsistentOp(const std::string& serialized_op_conf) {
+  return oneflow::CurJobBuildAndInferCtx_AddAndInferConsistentOp(serialized_op_conf)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetSerializedParallelConfFromProducerView(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetSerializedParallelConfFromProducerView(
+             job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair(
+    const std::string& lbi_uuid_pair) {
+  return oneflow::CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair(lbi_uuid_pair)
+      .GetDataAndErrorProto();
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_GetSerializedIdListAsStaticShape(const std::string& job_name,
+                                                     const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_GetSerializedIdListAsStaticShape(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<long long, std::shared_ptr<oneflow::cfg::ErrorProto>> JobBuildAndInferCtx_GetDataType(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_GetDataType(job_name, lbn).GetDataAndErrorProto(0LL);
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>> JobBuildAndInferCtx_IsDynamic(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_IsDynamic(job_name, lbn).GetDataAndErrorProto(false);
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>> JobBuildAndInferCtx_DisableBoxing(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_DisableBoxing(job_name, lbn).GetDataAndErrorProto(false);
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>> JobBuildAndInferCtx_IsTensorList(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_IsTensorList(job_name, lbn).GetDataAndErrorProto(false);
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>> JobBuildAndInferCtx_GetBatchAxis(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_GetBatchAxis(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_GetSplitAxisFromProducerView(const std::string& job_name,
+                                                 const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_GetSplitAxisFromProducerView(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_GetSerializedParallelConfFromProducerView(const std::string& job_name,
+                                                              const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_GetSerializedParallelConfFromProducerView(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::shared_ptr<oneflow::cfg::ErrorProto> CurJobBuildAndInferCtx_AddLossLogicalBlobName(
+    const std::string& lbn) {
+  return oneflow::CurJobBuildAndInferCtx_AddLossLogicalBlobName(lbn).GetDataAndErrorProto();
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>> JobBuildAndInferCtx_IsMirroredBlob(
+    const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_IsMirroredBlob(job_name, lbn).GetDataAndErrorProto(false);
+}
+
+std::pair<int, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetNumSubLbi(const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetNumSubLbi(job_name, lbn)
+      .GetDataAndErrorProto(0);
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetSerializedSubLbi(const std::string& job_name,
+                                                    const std::string& lbn, int index) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetSubLbi(job_name, lbn, index)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetSerializedIdListAsStaticShape(const std::string& job_name,
+                                                                 const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetSerializedIdListAsStaticShape(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<long long, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetDataType(const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetDataType(job_name, lbn)
+      .GetDataAndErrorProto(0LL);
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobIsDynamic(const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobIsDynamic(job_name, lbn)
+      .GetDataAndErrorProto(false);
+}
+
+std::pair<bool, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobIsTensorList(const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobIsTensorList(job_name, lbn)
+      .GetDataAndErrorProto(0LL);
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetBatchAxis(const std::string& job_name, const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetBatchAxis(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+std::pair<std::string, std::shared_ptr<oneflow::cfg::ErrorProto>>
+JobBuildAndInferCtx_MirroredBlobGetSplitAxisFromProducerView(const std::string& job_name,
+                                                             const std::string& lbn) {
+  return oneflow::JobBuildAndInferCtx_MirroredBlobGetSplitAxisFromProducerView(job_name, lbn)
+      .GetDataAndErrorProto(std::string(""));
+}
+
+#endif  // ONEFLOW_API_PYTHON_JOB_BUILD_JOB_BUILD_AND_INFER_H_

oneflow/python/job_build_and_infer_helper.h → oneflow/api/python/job_build/job_build_and_infer_helper.h

@@ -13,9 +13,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-#ifndef ONEFLOW_PYTHON_JOB_BUILD_AND_INFER_HELPER_H_
-#define ONEFLOW_PYTHON_JOB_BUILD_AND_INFER_HELPER_H_
+#ifndef ONEFLOW_API_PYTHON_JOB_BUILD_JOB_BUILD_AND_INFER_HELPER_H_
+#define ONEFLOW_API_PYTHON_JOB_BUILD_JOB_BUILD_AND_INFER_HELPER_H_
 
+#include "oneflow/core/job/global_for.h"
 #include "oneflow/core/common/protobuf.h"
 #include "oneflow/core/job/job_build_and_infer_ctx.h"
 #include "oneflow/core/record/record.pb.h"
@@ -99,6 +100,13 @@ Maybe<std::string> CurJobBuildAndInferCtx_AddAndInferConsistentOp(const std::str
   return PbMessage2TxtString(*op_attribute);
 }
 
+Maybe<std::string> JobBuildAndInferCtx_MirroredBlobGetSerializedParallelConfFromProducerView(
+    const std::string& job_name, const std::string& lbn) {
+  auto* ctx = JUST(GetJobBuildAndInferCtx(job_name));
+  return PbMessage2TxtString(
+      JUST(ctx->MirroredBlobGetParallelDescFromProducerView(lbn))->parallel_conf());
+}
+
 Maybe<void> CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair(
     const std::string& lbi_uuid_pair_str) {
   auto* ctx = JUST(GetCurInferCtx());
@@ -217,13 +225,6 @@ Maybe<std::string> JobBuildAndInferCtx_MirroredBlobGetSplitAxisFromProducerView(
   return PbMessage2TxtString(*JUST(ctx->MirroredBlobGetSplitAxisFromProducerView(lbn)));
 }
 
-Maybe<std::string> JobBuildAndInferCtx_MirroredBlobGetSerializedParallelConfFromProducerView(
-    const std::string& job_name, const std::string& lbn) {
-  auto* ctx = JUST(GetJobBuildAndInferCtx(job_name));
-  return PbMessage2TxtString(
-      JUST(ctx->MirroredBlobGetParallelDescFromProducerView(lbn))->parallel_conf());
-}
-
 }  // namespace oneflow
 
-#endif  // ONEFLOW_PYTHON_JOB_BUILD_AND_INFER_HELPER_H_
+#endif  // ONEFLOW_API_PYTHON_JOB_BUILD_JOB_BUILD_AND_INFER_HELPER_H_