py: add support for namespaced function reflection

Register top-level function definitions in the namespace (register the parent namespace information) and implement automatic wrapping of these functions for the python API. Move base convenience APIs to the =SemBaseApi= files, to avoid adding too many things in the code logic definition.
haxscramper · Mar 16, 2024 · e512c4a · e512c4a
1 parent f1fd8d6
commit e512c4a
Show file tree

Hide file tree

Showing 14 changed files with 246 additions and 258 deletions.
diff --git a/scripts/cxx_codegen/clang_reflection_lib.cpp b/scripts/cxx_codegen/clang_reflection_lib.cpp
@@ -885,6 +885,12 @@ bool ReflASTVisitor::VisitFunctionDecl(c::FunctionDecl* Decl) {
             func->mutable_resultty(),
             Decl->getReturnType(),
             Decl->getLocation());
+
+        for (auto const& space :
+             getNamespaces(Decl, Decl->getLocation())) {
+            auto added = func->add_spaces();
+            *added     = space;
+        }
     }
     return true;
 }

diff --git a/scripts/cxx_codegen/reflection_defs.proto b/scripts/cxx_codegen/reflection_defs.proto
@@ -119,6 +119,7 @@ message Function {
     repeated Arg Arguments = 3;
     repeated TemplateSpec Templates = 4;
     string DbgOrigin = 5;
+    repeated QualType Spaces = 6;
 }
 
 message Typedef {

diff --git a/scripts/py_codegen/py_codegen/astbuilder_pybind11.py b/scripts/py_codegen/py_codegen/astbuilder_pybind11.py
@@ -128,6 +128,7 @@ class Py11Function:
     Body: Optional[List[BlockId]] = None
     Doc: GenTuDoc = field(default_factory=lambda: GenTuDoc(""))
     DefParams: Optional[List[BlockId]] = None
+    Spaces: List[QualType] = field(default_factory=list)
 
     def build_typedef(self, ast: pya.ASTBuilder) -> pya.FunctionDefParams:
         return pya.FunctionDefParams(
@@ -151,6 +152,7 @@ def FromGenTu(
             CxxName=meth.name,
             Doc=meth.doc,
             Args=meth.arguments,
+            Spaces=meth.spaces,
         )
 
     def build_argument_binder(self, Args: List[GenTuIdent],
@@ -217,14 +219,22 @@ def build_doc_comment(self, ast: ASTBuilder) -> list[BlockId]:
             return []
 
     def build_bind(self, ast: ASTBuilder) -> BlockId:
+        if self.Spaces:
+            full_name = ast.Scoped(
+                QualType(name=self.Spaces[0].name, Spaces=self.Spaces[1:]),
+                ast.string(self.CxxName))
+
+        else:
+            full_name = ast.string(self.CxxName)
+
         return ast.XCall(
             "m.def",
             [
                 ast.Literal(self.PyName),
                 self.build_call_pass(
                     ast,
                     self.Args,
-                    FunctionQualName=ast.string(self.CxxName),
+                    FunctionQualName=full_name,
                 ),
                 *self.build_argument_binder(self.Args, ast),
                 *self.build_doc_comment(ast),

diff --git a/scripts/py_codegen/py_codegen/codegen.py b/scripts/py_codegen/py_codegen/codegen.py
@@ -817,6 +817,7 @@ def gen_value(ast: ASTBuilder, pyast: pya.ASTBuilder, reflection_path: str) -> G
         yaml.safe_dump(to_base_types(tu), stream=file)
 
     with open("/tmp/reflection_data.json", "w") as file:
+        log(CAT).debug(f"Debug reflection data to {file.name}")
         file.write(open_proto_file(reflection_path).to_json(2))
 
     global org_type_names

diff --git a/scripts/py_codegen/py_codegen/gen_tu_cpp.py b/scripts/py_codegen/py_codegen/gen_tu_cpp.py
@@ -77,6 +77,7 @@ class GenTuFunction:
     isPureVirtual: bool = False
     parentClass: Optional['GenTuStruct'] = None
     original: Optional[Path] = None
+    spaces: List[QualType] = field(default_factory=list)
 
     def format(self) -> str:
         return "function %s %s(%s)" % (self.result.format(), self.name, ", ".join(

diff --git a/scripts/py_codegen/py_codegen/refl_read.py b/scripts/py_codegen/py_codegen/refl_read.py
@@ -107,8 +107,10 @@ def conv_proto_type(typ: pb.QualType, is_anon_name: bool = False) -> QualType:
 
 @beartype
 def conv_proto_record(record: pb.Record, original: Optional[Path]) -> GenTuStruct:
-    result = GenTuStruct(conv_proto_type(record.name, is_anon_name=not record.has_name),
-                         GenTuDoc(""),)
+    result = GenTuStruct(
+        conv_proto_type(record.name, is_anon_name=not record.has_name),
+        GenTuDoc(""),
+    )
 
     result.original = copy(original)
     result.IsForwardDecl = record.is_forward_decl
@@ -181,6 +183,7 @@ def conv_proto_function(rec: pb.Function, original: Optional[Path]) -> GenTuFunc
         arguments=[conv_proto_arg(arg) for arg in rec.arguments],
         doc=GenTuDoc(""),
         original=copy(original),
+        spaces=[conv_proto_type(T) for T in rec.spaces],
     )
 
 

diff --git a/scripts/py_haxorg/py_haxorg/pyhaxorg.pyi b/scripts/py_haxorg/py_haxorg/pyhaxorg.pyi
@@ -1098,8 +1098,6 @@ def parseStringOpts(text: str, opts: OrgParseParameters) -> Document: ...
 
 def formatToString(arg: Org) -> str: ...
 
-def eachSubnodeRec(node: Org, callback: function) -> None: ...
-
 def exportToYamlString(node: Org) -> str: ...
 
 def exportToYamlFile(node: Org, path: str) -> None: ...
@@ -1116,4 +1114,6 @@ def exportToTreeString(node: Org, opts: OrgTreeExportOpts) -> str: ...
 
 def exportToTreeFile(node: Org, path: str, opts: OrgTreeExportOpts) -> None: ...
 
+def eachSubnodeRec(node: Org, callback: function) -> None: ...
+
 /* clang-format on */
diff --git a/src/editor/gui_lib/org_qml_backend.hpp b/src/editor/gui_lib/org_qml_backend.hpp
@@ -15,7 +15,7 @@ struct OrgBackend : public QObject {
     OrgBackend() {}
 
     void parseString(CR<std::string> text) {
-        document = org::parseString(text);
+        document = sem::parseString(text);
         model    = std::make_shared<OrgDocumentModel>(document, this);
         filter   = std::make_shared<OrgDocumentSearchFilter>(
             model.get(), this);
@@ -59,7 +59,7 @@ struct OrgBackend : public QObject {
     }
 
     Q_INVOKABLE void saveDocumentToJson(CR<QString> path) {
-        writeFile(to_std(path), org::toJson(document.asOrg()).dump(2));
+        writeFile(to_std(path), sem::exportToJsonString(document.asOrg()));
     }
   signals:
     void hasDocumentChanged(bool status);

diff --git a/src/py_libs/pyhaxorg/pyhaxorg.cpp b/src/py_libs/pyhaxorg/pyhaxorg.cpp
@@ -1950,29 +1950,29 @@ example),)RAW")
          pybind11::arg("node"),
          R"RAW(\brief Recursively register all availble targets from the nodes.)RAW")
     ;
-  pybind11::class_<OrgParseParameters>(m, "OrgParseParameters")
-    .def(pybind11::init([](pybind11::kwargs const& kwargs) -> OrgParseParameters {
-                        OrgParseParameters result{};
+  pybind11::class_<sem::OrgParseParameters>(m, "OrgParseParameters")
+    .def(pybind11::init([](pybind11::kwargs const& kwargs) -> sem::OrgParseParameters {
+                        sem::OrgParseParameters result{};
                         init_fields_from_kwargs(result, kwargs);
                         return result;
                         }))
-    .def_readwrite("baseTokenTracePath", &OrgParseParameters::baseTokenTracePath)
-    .def_readwrite("tokenTracePath", &OrgParseParameters::tokenTracePath)
-    .def_readwrite("parseTracePath", &OrgParseParameters::parseTracePath)
-    .def_readwrite("semTracePath", &OrgParseParameters::semTracePath)
+    .def_readwrite("baseTokenTracePath", &sem::OrgParseParameters::baseTokenTracePath)
+    .def_readwrite("tokenTracePath", &sem::OrgParseParameters::tokenTracePath)
+    .def_readwrite("parseTracePath", &sem::OrgParseParameters::parseTracePath)
+    .def_readwrite("semTracePath", &sem::OrgParseParameters::semTracePath)
     ;
-  pybind11::class_<OrgTreeExportOpts>(m, "OrgTreeExportOpts")
-    .def(pybind11::init([](pybind11::kwargs const& kwargs) -> OrgTreeExportOpts {
-                        OrgTreeExportOpts result{};
+  pybind11::class_<sem::OrgTreeExportOpts>(m, "OrgTreeExportOpts")
+    .def(pybind11::init([](pybind11::kwargs const& kwargs) -> sem::OrgTreeExportOpts {
+                        sem::OrgTreeExportOpts result{};
                         init_fields_from_kwargs(result, kwargs);
                         return result;
                         }))
-    .def_readwrite("withLineCol", &OrgTreeExportOpts::withLineCol)
-    .def_readwrite("withOriginalId", &OrgTreeExportOpts::withOriginalId)
-    .def_readwrite("withSubnodeIdx", &OrgTreeExportOpts::withSubnodeIdx)
-    .def_readwrite("skipEmptyFields", &OrgTreeExportOpts::skipEmptyFields)
-    .def_readwrite("startLevel", &OrgTreeExportOpts::startLevel)
-    .def_readwrite("withColor", &OrgTreeExportOpts::withColor)
+    .def_readwrite("withLineCol", &sem::OrgTreeExportOpts::withLineCol)
+    .def_readwrite("withOriginalId", &sem::OrgTreeExportOpts::withOriginalId)
+    .def_readwrite("withSubnodeIdx", &sem::OrgTreeExportOpts::withSubnodeIdx)
+    .def_readwrite("skipEmptyFields", &sem::OrgTreeExportOpts::skipEmptyFields)
+    .def_readwrite("startLevel", &sem::OrgTreeExportOpts::startLevel)
+    .def_readwrite("withColor", &sem::OrgTreeExportOpts::withColor)
     ;
   pybind11::class_<ExporterPython>(m, "ExporterPython")
     .def(pybind11::init([](pybind11::kwargs const& kwargs) -> ExporterPython {
@@ -2107,52 +2107,52 @@ example),)RAW")
                      })
     ;
   m.def("parseFile",
-        static_cast<sem::SemId<sem::Document>(*)(std::string, OrgParseParameters const&)>(&parseFile),
+        static_cast<sem::SemId<sem::Document>(*)(std::string, sem::OrgParseParameters const&)>(&sem::parseFile),
         pybind11::arg("file"),
         pybind11::arg("opts"));
   m.def("parseString",
-        static_cast<sem::SemId<sem::Document>(*)(std::string const)>(&parseString),
+        static_cast<sem::SemId<sem::Document>(*)(std::string const)>(&sem::parseString),
         pybind11::arg("text"));
   m.def("parseStringOpts",
-        static_cast<sem::SemId<sem::Document>(*)(std::string const, OrgParseParameters const&)>(&parseStringOpts),
+        static_cast<sem::SemId<sem::Document>(*)(std::string const, sem::OrgParseParameters const&)>(&sem::parseStringOpts),
         pybind11::arg("text"),
         pybind11::arg("opts"));
   m.def("formatToString",
-        static_cast<std::string(*)(sem::SemId<sem::Org>)>(&formatToString),
+        static_cast<std::string(*)(sem::SemId<sem::Org>)>(&sem::formatToString),
         pybind11::arg("arg"));
-  m.def("eachSubnodeRec",
-        static_cast<void(*)(sem::SemId<sem::Org>, pybind11::function)>(&eachSubnodeRec),
-        pybind11::arg("node"),
-        pybind11::arg("callback"));
   m.def("exportToYamlString",
-        static_cast<std::string(*)(sem::SemId<sem::Org> const&)>(&exportToYamlString),
+        static_cast<std::string(*)(sem::SemId<sem::Org> const&)>(&sem::exportToYamlString),
         pybind11::arg("node"));
   m.def("exportToYamlFile",
-        static_cast<void(*)(sem::SemId<sem::Org> const&, std::string)>(&exportToYamlFile),
+        static_cast<void(*)(sem::SemId<sem::Org> const&, std::string)>(&sem::exportToYamlFile),
         pybind11::arg("node"),
         pybind11::arg("path"));
   m.def("exportToJsonString",
-        static_cast<std::string(*)(sem::SemId<sem::Org> const&)>(&exportToJsonString),
+        static_cast<std::string(*)(sem::SemId<sem::Org> const&)>(&sem::exportToJsonString),
         pybind11::arg("node"));
   m.def("exportToJsonFile",
-        static_cast<void(*)(sem::SemId<sem::Org> const&, std::string)>(&exportToJsonFile),
+        static_cast<void(*)(sem::SemId<sem::Org> const&, std::string)>(&sem::exportToJsonFile),
         pybind11::arg("node"),
         pybind11::arg("path"));
   m.def("readProtobufFile",
-        static_cast<sem::SemId<sem::Document>(*)(std::string const&)>(&readProtobufFile),
+        static_cast<sem::SemId<sem::Document>(*)(std::string const&)>(&sem::readProtobufFile),
         pybind11::arg("file"));
   m.def("exportToProtobufFile",
-        static_cast<void(*)(sem::SemId<sem::Document>, std::string const&)>(&exportToProtobufFile),
+        static_cast<void(*)(sem::SemId<sem::Document>, std::string const&)>(&sem::exportToProtobufFile),
         pybind11::arg("doc"),
         pybind11::arg("file"));
   m.def("exportToTreeString",
-        static_cast<std::string(*)(sem::SemId<sem::Org> const&, OrgTreeExportOpts const&)>(&exportToTreeString),
+        static_cast<std::string(*)(sem::SemId<sem::Org> const&, sem::OrgTreeExportOpts const&)>(&sem::exportToTreeString),
         pybind11::arg("node"),
         pybind11::arg("opts"));
   m.def("exportToTreeFile",
-        static_cast<void(*)(sem::SemId<sem::Org> const&, std::string, OrgTreeExportOpts const&)>(&exportToTreeFile),
+        static_cast<void(*)(sem::SemId<sem::Org> const&, std::string, sem::OrgTreeExportOpts const&)>(&sem::exportToTreeFile),
         pybind11::arg("node"),
         pybind11::arg("path"),
         pybind11::arg("opts"));
+  m.def("eachSubnodeRec",
+        static_cast<void(*)(sem::SemId<sem::Org>, pybind11::function)>(&eachSubnodeRec),
+        pybind11::arg("node"),
+        pybind11::arg("callback"));
 }
 /* clang-format on */
diff --git a/src/py_libs/pyhaxorg/pyhaxorg_manual_impl.cpp b/src/py_libs/pyhaxorg/pyhaxorg_manual_impl.cpp
@@ -18,123 +18,6 @@
 
 template class Exporter<ExporterPython, py::object>;
 
-std::string exportToJsonString(sem::SemId<sem::Org> const& node) {
-    return to_string(ExporterJson{}.evalTop(node));
-}
-
-void exportToJsonFile(sem::SemId<sem::Org> const& node, std::string path) {
-    writeFile(fs::path{path}, exportToJsonString(node));
-}
-
-std::string exportToYamlString(sem::SemId<sem::Org> const& node) {
-    std::stringstream os;
-    os << ExporterYaml{}.evalTop(node);
-    return os.str();
-}
-
-void exportToYamlFile(sem::SemId<sem::Org> const& node, std::string path) {
-    writeFile(fs::path{path}, exportToYamlString(node));
-}
-
-
-std::string exportToTreeString(
-    sem::SemId<sem::Org> const& node,
-    OrgTreeExportOpts const&    opts) {
-    ColStream    os{};
-    ExporterTree tree{os};
-
-    tree.conf.withLineCol     = opts.withLineCol;
-    tree.conf.withOriginalId  = opts.withOriginalId;
-    tree.conf.skipEmptyFields = opts.skipEmptyFields;
-    tree.conf.startLevel      = opts.startLevel;
-    tree.evalTop(node);
-
-    std::string result = os.toString(opts.withColor);
-    return result;
-}
-
-void exportToTreeFile(
-    sem::SemId<sem::Org> const& node,
-    std::string                 path,
-    OrgTreeExportOpts const&    opts) {
-
-    ColStream    os{};
-    ExporterTree tree{os};
-
-    tree.conf.withLineCol     = opts.withLineCol;
-    tree.conf.withOriginalId  = opts.withOriginalId;
-    tree.conf.skipEmptyFields = opts.skipEmptyFields;
-    tree.conf.startLevel      = opts.startLevel;
-    tree.evalTop(node);
-
-    std::ofstream file{path};
-    file << os.toString(opts.withColor);
-}
-
-sem::SemId<sem::Document> parseFile(
-    std::string               file,
-    const OrgParseParameters& opts) {
-    return parseStringOpts(readFile(fs::path{file}), opts);
-}
-
-sem::SemId<sem::Document> parseString(std::string text) {
-    return parseStringOpts(text, OrgParseParameters{});
-}
-
-sem::SemId<sem::Document> parseStringOpts(
-    const std::string         text,
-    OrgParseParameters const& opts) {
-    LexerParams         p;
-    SPtr<std::ofstream> fileTrace;
-    if (opts.baseTokenTracePath) {
-        fileTrace = std::make_shared<std::ofstream>(
-            *opts.baseTokenTracePath);
-    }
-    p.traceStream            = fileTrace.get();
-    OrgTokenGroup baseTokens = ::tokenize(text.data(), text.size(), p);
-    OrgTokenGroup tokens;
-    OrgTokenizer  tokenizer{&tokens};
-
-    if (opts.tokenTracePath) {
-        tokenizer.setTraceFile(*opts.tokenTracePath);
-    }
-
-    tokenizer.convert(baseTokens);
-    Lexer<OrgTokenKind, OrgFill> lex{&tokens};
-
-    OrgNodeGroup nodes{&tokens};
-    OrgParser    parser{&nodes};
-    if (opts.parseTracePath) { parser.setTraceFile(*opts.parseTracePath); }
-
-    (void)parser.parseFull(lex);
-
-    sem::OrgConverter converter{};
-    if (opts.semTracePath) { converter.setTraceFile(*opts.semTracePath); }
-
-    return converter.toDocument(OrgAdapter(&nodes, OrgId(0)));
-}
-
-sem::SemId<sem::Document> readProtobufFile(const std::string& file) {
-    sem::SemId        read_node = sem::SemId<sem::Org>::Nil();
-    std::ifstream     stream{file};
-    orgproto::AnyNode result;
-    result.ParseFromIstream(&stream);
-    proto_serde<orgproto::AnyNode, sem::SemId<sem::Org>>::read(
-        result,
-        proto_write_accessor<sem::SemId<sem::Org>>::for_ref(read_node));
-    return read_node.as<sem::Document>();
-}
-
-void exportToProtobufFile(
-    sem::SemId<sem::Document> doc,
-    const std::string&        file) {
-    std::ofstream     stream{file};
-    orgproto::AnyNode result;
-    proto_serde<orgproto::AnyNode, sem::SemId<sem::Org>>::write(
-        &result, doc.asOrg());
-    result.SerializeToOstream(&stream);
-}
-
 
 std::vector<sem::SemId<sem::Org>> getSubnodeRange(
     sem::SemId<sem::Org> id,
@@ -301,9 +184,7 @@ ExporterPython::Res ExporterPython::evalTop(sem::SemId<sem::Org> org) {
         return tmp;
     }
 }
-std::string formatToString(sem::SemId<sem::Org> arg) {
-    return sem::Formatter::format(arg);
-}
+
 
 void eachSubnodeRec(sem::SemId<sem::Org> node, py::function callback) {
     sem::eachSubnodeRec(