Replace m2r with new typescript-based method

.chronus/changes/feature-md2rst-2025-1-21-1-49-22.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/http-client-python"
+---
+
+Replace m2r with new md2rst() method

packages/http-client-python/emitter/src/emitter.ts

@@ -15,7 +15,7 @@ import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js";
 import { runPython3 } from "./run-python3.js";
 import { disableGenerationMap, simpleTypesMap, typesMap } from "./types.js";
-import { removeUnderscoresFromNamespace } from "./utils.js";
+import { md2Rst, removeUnderscoresFromNamespace } from "./utils.js";
 
 export function getModelsMode(context: SdkContext): "dpg" | "none" {
   const specifiedModelsMode = context.emitContext.options["models-mode"];
@@ -77,6 +77,44 @@ async function createPythonSdkContext<TServiceOperation extends SdkServiceOperat
   };
 }
 
+function walkThroughNodes(yamlMap: Record<string, any>): Record<string, any> {
+  const stack = [yamlMap];
+  const seen = new WeakSet();
+
+  while (stack.length > 0) {
+    const current = stack.pop();
+
+    if (seen.has(current!)) {
+      continue;
+    }
+    if (current !== undefined && current !== null) {
+      seen.add(current);
+    }
+
+    if (Array.isArray(current)) {
+      for (let i = 0; i < current.length; i++) {
+        if (current[i] !== undefined && typeof current[i] === "object") {
+          stack.push(current[i]);
+        }
+      }
+    } else {
+      for (const key in current) {
+        if (key === "description" || key === "summary") {
+          if (current[key] !== undefined) {
+            current[key] = md2Rst(current[key]);
+          }
+        } else if (Array.isArray(current[key])) {
+          stack.push(current[key]);
+        } else if (current[key] !== undefined && typeof current[key] === "object") {
+          stack.push(current[key]);
+        }
+      }
+    }
+  }
+
+  return yamlMap;
+}
+
 function cleanAllCache() {
   typesMap.clear();
   simpleTypesMap.clear();
@@ -100,7 +138,10 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
     });
     return;
   }
-  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap);
+
+  const parsedYamlMap = walkThroughNodes(yamlMap);
+
+  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap);
   const resolvedOptions = sdkContext.emitContext.options;
   const commandArgs: Record<string, string> = {};
   if (resolvedOptions["packaging-files-config"]) {
@@ -166,9 +207,7 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
         async def main():
           import warnings
           with warnings.catch_warnings():
-            warnings.simplefilter("ignore", SyntaxWarning) # bc of m2r2 dep issues
-            from pygen import m2r, preprocess, codegen, black
-          m2r.M2R(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+            from pygen import preprocess, codegen, black
           preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
           codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
           black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()

packages/http-client-python/emitter/src/utils.ts

@@ -10,6 +10,7 @@ import {
   SdkType,
 } from "@azure-tools/typespec-client-generator-core";
 import { getNamespaceFullName } from "@typespec/compiler";
+import { marked, Token } from "marked";
 import { PythonSdkContext } from "./lib.js";
 import { getSimpleTypeResult, getType } from "./types.js";
 
@@ -232,3 +233,84 @@ export function getClientNamespace<TServiceOperation extends SdkServiceOperation
     ? rootNamespace
     : removeUnderscoresFromNamespace(clientNamespace).toLowerCase();
 }
+
+function parseToken(token: Token): string {
+  let parsed = "";
+  switch (token.type) {
+    case "heading":
+      parsed += `${"=".repeat(token.text.length)}\n${token.text}\n${"=".repeat(
+        token.text.length,
+      )}\n\n`;
+      break;
+    case "paragraph":
+      parsed += `${token.text}\n\n`;
+      break;
+    case "strong":
+      parsed += `**${token.text}**`;
+      break;
+    case "em":
+      parsed += `*${token.text}*`;
+      break;
+    case "codespan":
+      parsed += `\`\`${token.text}\`\``;
+      break;
+    case "code":
+      let codeBlockStyle = token.codeBlockStyle;
+      if (codeBlockStyle === undefined) {
+        codeBlockStyle = token.raw.split("\n")[0].replace("```", "").trim();
+      }
+      parsed += `\n\n.. code-block:: ${codeBlockStyle ?? ""}\n\n   ${token.text.split("\n").join("\n   ")}`;
+      break;
+    case "link":
+      if (token.href !== undefined) {
+        parsed += `\`${token.text} <${token.href}>\`_`;
+        break;
+      }
+      parsed += `${token.text}`;
+      break;
+    case "list":
+      if (!token.ordered) {
+        parsed += `\n\n${token.items.map((item: any) => `* ${item.text}`).join("\n")}`;
+        break;
+      }
+      parsed += `\n\n${token.items.map((item: any, index: number) => `${index + 1}. ${item.text}`).join("\n")}`;
+      break;
+    default:
+      parsed += token.raw;
+  }
+  return parsed;
+}
+
+export function md2Rst(text?: string): string | undefined {
+  try {
+    if (!text || text === "") return text;
+    const tokens = marked.lexer(text);
+    let rst = "";
+
+    tokens.forEach((token: Token) => {
+      if (token.type === "heading") {
+        // Heading tokens are block level, so we should check if there are additional tokens inside
+        const parsedHeadingText = md2Rst(token.text);
+        rst += `${"=".repeat(
+          parsedHeadingText!.length,
+        )}\n${parsedHeadingText}\n${"=".repeat(parsedHeadingText!.length)}\n\n`;
+      } else if ("tokens" in token && token.tokens !== undefined && token.tokens.length > 0) {
+        token.tokens.forEach((element: any) => {
+          rst += parseToken(element);
+        });
+      } else {
+        rst += parseToken(token);
+      }
+    });
+
+    // Trim trailing whitespace or tabs
+    return rst.replace(/[ \t]+$/, "");
+  } catch (e) {
+    if (e instanceof RangeError) {
+      // The error is thrown by the tokenizer when the markdown is too long
+      // We can ignore it and return the original text
+      return text;
+    }
+  }
+  return text;
+}

packages/http-client-python/emitter/test/utils.test.ts

@@ -1,6 +1,6 @@
 import { strictEqual } from "assert";
 import { describe, it } from "vitest";
-import { camelToSnakeCase } from "../src/utils.js";
+import { camelToSnakeCase, md2Rst } from "../src/utils.js";
 
 describe("typespec-python: utils", () => {
   it("camelToSnakeCase", async () => {
@@ -19,4 +19,9 @@ describe("typespec-python: utils", () => {
       strictEqual(camelToSnakeCase(input), expected);
     }
   });
+
+  it("md2rst", async () => {
+    const des = "Format: <MajorVersion>.<MinorVersion>.<Patch>";
+    strictEqual(md2Rst(des), "Format: <MajorVersion>.<MinorVersion>.<Patch>");
+  });
 });

packages/http-client-python/eng/scripts/setup/run_tsp.py

@@ -7,7 +7,7 @@
 import venv
 import logging
 from pathlib import Path
-from pygen import m2r, preprocess, codegen, black
+from pygen import preprocess, codegen, black
 from pygen.utils import parse_args
 
 _ROOT_DIR = Path(__file__).parent.parent.parent.parent
@@ -34,9 +34,8 @@
         debugpy.wait_for_client()
         breakpoint()  # pylint: disable=undefined-variable
 
-    # run m2r
+    # pre-process and run black
     args, unknown_args = parse_args()
-    m2r.M2R(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()
     preprocess.PreProcessPlugin(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()
     codegen.CodeGenerator(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()
     black.BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process()

packages/http-client-python/generator/setup.py

@@ -51,7 +51,6 @@
         "black==24.8.0",
         "docutils>=0.20.1",
         "Jinja2==3.1.3",
-        "m2r2==0.3.3.post2",
         "PyYAML==6.0.1",
         "tomli==2.0.1",
         "setuptools==69.5.1",

packages/http-client-python/package.json

@@ -66,6 +66,7 @@
   },
   "dependencies": {
     "js-yaml": "~4.1.0",
+    "marked": "^15.0.6",
     "pyodide": "0.26.2",
     "semver": "~7.6.2",
     "tsx": "~4.19.1"