From 5ccb2d8e0cfd4da9e1f115b1af361e69ad22a8e1 Mon Sep 17 00:00:00 2001 From: Jonathan Schneider Date: Sun, 3 May 2026 21:37:55 -0400 Subject: [PATCH] RPC: continuous profiling for Python/JS/Go subprocesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each language's RPC server now starts the Pyroscope SDK when PYROSCOPE_SERVER_ADDRESS is set in the environment, with PYROSCOPE_TAGS forwarded verbatim and a runtime=python/node/go tag added so flame graphs in a shared `modcli` Pyroscope application can be sliced by which subprocess produced them. Env vars propagate from the parent JVM via ProcessBuilder.environment() inheritance, so the saas-side CliProcessEnvContributor that already drives the JVM agent transparently reaches all four runtimes once these inits land. Python: pyroscope-io is a new optional dep ([profiling] extra) so local dev and CI without the package don't break — _init_pyroscope() warns and no-ops on ImportError. JS: @pyroscope/nodejs is a new optionalDependency so npm install doesn't fail when the binding's native components aren't available; initPyroscope() catches the require and warns. Go: github.com/grafana/pyroscope-go is a regular dep (Go has no optional deps; the SDK is small and pure-Go via stdlib pprof). initPyroscope() short-circuits on missing env so non-profiled deployments pay only one os.Getenv at startup. C# requires no rewrite-side change: CoreCLR loads the Pyroscope native profiler purely from CORECLR_ENABLE_PROFILING / CORECLR_PROFILER / CORECLR_PROFILER_PATH env vars, which the same saas-side contributor sets alongside PYROSCOPE_*. --- rewrite-go/cmd/rpc/main.go | 28 +++ rewrite-go/go.mod | 7 +- rewrite-go/go.sum | 6 + rewrite-javascript/rewrite/package-lock.json | 180 ++++++++++++++++++ rewrite-javascript/rewrite/package.json | 1 + rewrite-javascript/rewrite/src/rpc/server.ts | 29 +++ rewrite-python/rewrite/pyproject.toml | 5 + .../rewrite/src/rewrite/rpc/server.py | 29 +++ 8 files changed, 284 insertions(+), 1 deletion(-) diff --git a/rewrite-go/cmd/rpc/main.go b/rewrite-go/cmd/rpc/main.go index 038cd3550ff..f3b115f1eae 100644 --- a/rewrite-go/cmd/rpc/main.go +++ b/rewrite-go/cmd/rpc/main.go @@ -36,6 +36,7 @@ import ( "time" "github.com/google/uuid" + "github.com/grafana/pyroscope-go" goparser "github.com/openrewrite/rewrite/rewrite-go/pkg/parser" "github.com/openrewrite/rewrite/rewrite-go/pkg/printer" @@ -257,7 +258,34 @@ func parseFlags() serverConfig { return cfg } +// initPyroscope starts continuous profiling when PYROSCOPE_SERVER_ADDRESS is +// set. Tags inherited via PYROSCOPE_TAGS (k=v,k=v) are forwarded verbatim; a +// runtime=go tag is added so flame graphs in the shared modcli application +// can be sliced by which RPC subprocess produced them. +func initPyroscope() { + server := os.Getenv("PYROSCOPE_SERVER_ADDRESS") + if server == "" { + return + } + appName := os.Getenv("PYROSCOPE_APPLICATION_NAME") + if appName == "" { + appName = "modcli" + } + tags := map[string]string{"runtime": "go"} + for _, pair := range strings.Split(os.Getenv("PYROSCOPE_TAGS"), ",") { + if i := strings.Index(pair, "="); i > 0 { + tags[strings.TrimSpace(pair[:i])] = strings.TrimSpace(pair[i+1:]) + } + } + _, _ = pyroscope.Start(pyroscope.Config{ + ApplicationName: appName, + ServerAddress: server, + Tags: tags, + }) +} + func main() { + initPyroscope() cfg := parseFlags() s := newServer(cfg) s.logger.Println("Go RPC server starting...") diff --git a/rewrite-go/go.mod b/rewrite-go/go.mod index 213ea00d7a2..0265ee06d71 100644 --- a/rewrite-go/go.mod +++ b/rewrite-go/go.mod @@ -4,4 +4,9 @@ go 1.25.0 require github.com/google/uuid v1.6.0 -require golang.org/x/mod v0.35.0 // indirect +require ( + github.com/grafana/pyroscope-go v1.2.8 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect + github.com/klauspost/compress v1.17.8 // indirect + golang.org/x/mod v0.35.0 // indirect +) diff --git a/rewrite-go/go.sum b/rewrite-go/go.sum index 4ae3bf38875..9fe15ebc7e5 100644 --- a/rewrite-go/go.sum +++ b/rewrite-go/go.sum @@ -1,4 +1,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/pyroscope-go v1.2.8 h1:UvCwIhlx9DeV7F6TW/z8q1Mi4PIm3vuUJ2ZlCEvmA4M= +github.com/grafana/pyroscope-go v1.2.8/go.mod h1:SSi59eQ1/zmKoY/BKwa5rSFsJaq+242Bcrr4wPix1g8= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= diff --git a/rewrite-javascript/rewrite/package-lock.json b/rewrite-javascript/rewrite/package-lock.json index 89dd92a6a26..11e0f68b53c 100644 --- a/rewrite-javascript/rewrite/package-lock.json +++ b/rewrite-javascript/rewrite/package-lock.json @@ -36,6 +36,7 @@ "vitest": "^4.0.18" }, "optionalDependencies": { + "@pyroscope/nodejs": "^0.4.0", "prettier": "^3.7.4" } }, @@ -45,6 +46,53 @@ "integrity": "sha512-9En2OKHBOO39vztHUxHUh/Xh6wTI1lEQ9c0ivr7QX3ozaKgs770TRJrgtBbQBeLLbMoLGG3fBk3otAlSY440pw==", "license": "MIT" }, + "node_modules/@datadog/pprof": { + "version": "5.13.3", + "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-5.13.3.tgz", + "integrity": "sha512-G25IicP7pc5CXmAfVz7nrIERsKK9hvPz6p7xsLTUwG4Qs+Zgd5KFedKCVsnvNasLc7l7OXQ6839ajowgQLWTyw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "delay": "^5.0.0", + "node-gyp-build": "<4.0", + "p-limit": "^3.1.0", + "pprof-format": "^2.2.1", + "source-map": "^0.7.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@datadog/pprof/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@datadog/pprof/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", @@ -648,6 +696,35 @@ "win32" ] }, + "node_modules/@pyroscope/nodejs": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@pyroscope/nodejs/-/nodejs-0.4.11.tgz", + "integrity": "sha512-hfLE72zc8toxC4UPgSPHxglHfFF9+62YqhqUC6LsK5pdaBQBKjqvIW9EFL6vp5c3lSs7ctjnLy2Rki9RCsqQ8g==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@datadog/pprof": "5.13.3", + "debug": "^4.4.3", + "p-limit": "^7.3.0", + "regenerator-runtime": "^0.14.1", + "source-map": "^0.7.6" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "express": "^4.0.0 || ^5.0.0", + "fastify": "^5.7.4" + }, + "peerDependenciesMeta": { + "express": { + "optional": true + }, + "fastify": { + "optional": true + } + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.59.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", @@ -1264,6 +1341,37 @@ "node": ">=20" } }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delay": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", + "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/diff": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.3.tgz", @@ -1398,6 +1506,13 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, "node_modules/mutative": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/mutative/-/mutative-1.3.0.tgz", @@ -1426,6 +1541,18 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/node-gyp-build": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-3.9.0.tgz", + "integrity": "sha512-zLcTg6P4AbcHPq465ZMFNXx7XpKKJh+7kkN699NiQWisR2uWYOWNWqRHAmbnmKiL4e9aLSlmy5U7rEMUXV59+A==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -1437,6 +1564,22 @@ ], "license": "MIT" }, + "node_modules/p-limit": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", + "license": "MIT", + "optional": true, + "dependencies": { + "yocto-queue": "^1.2.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -1500,6 +1643,13 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/pprof-format": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/pprof-format/-/pprof-format-2.2.1.tgz", + "integrity": "sha512-p4tVN7iK19ccDqQv8heyobzUmbHyds4N2FI6aBMcXz6y99MglTWDxIyhFkNaLeEXs6IFUEzT0zya0icbSLLY0g==", + "license": "MIT", + "optional": true + }, "node_modules/prettier": { "version": "3.7.4", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.7.4.tgz", @@ -1516,6 +1666,13 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT", + "optional": true + }, "node_modules/rollup": { "version": "4.59.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", @@ -1580,6 +1737,16 @@ "dev": true, "license": "ISC" }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -1877,6 +2044,19 @@ "engines": { "node": ">= 14.6" } + }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/rewrite-javascript/rewrite/package.json b/rewrite-javascript/rewrite/package.json index 63b3253d34f..fcae2df00c8 100644 --- a/rewrite-javascript/rewrite/package.json +++ b/rewrite-javascript/rewrite/package.json @@ -86,6 +86,7 @@ "vitest": "^4.0.18" }, "optionalDependencies": { + "@pyroscope/nodejs": "^0.4.0", "prettier": "^3.7.4" }, "bin": { diff --git a/rewrite-javascript/rewrite/src/rpc/server.ts b/rewrite-javascript/rewrite/src/rpc/server.ts index bff45ddcd8e..f06787297c8 100755 --- a/rewrite-javascript/rewrite/src/rpc/server.ts +++ b/rewrite-javascript/rewrite/src/rpc/server.ts @@ -31,6 +31,33 @@ import "../javascript"; // Not possible to set the stack size when executing from npx for security reasons require('v8').setFlagsFromString('--stack-size=8000'); +function initPyroscope(logger: rpc.Logger): void { + const server = process.env.PYROSCOPE_SERVER_ADDRESS; + if (!server) { + return; + } + let Pyroscope: any; + try { + Pyroscope = require('@pyroscope/nodejs'); + } catch { + logger.warn('PYROSCOPE_SERVER_ADDRESS set but @pyroscope/nodejs not installed; profiling disabled'); + return; + } + const tags: Record = {runtime: 'node'}; + for (const pair of (process.env.PYROSCOPE_TAGS || '').split(',')) { + const eq = pair.indexOf('='); + if (eq > 0) { + tags[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } + Pyroscope.init({ + appName: process.env.PYROSCOPE_APPLICATION_NAME || 'modcli', + serverAddress: server, + tags, + }); + Pyroscope.start(); +} + interface ProgramOptions { logFile?: string; metricsCsv?: string; @@ -97,6 +124,8 @@ async function main() { log: (msg: string) => log && options.traceRpcMessages && log.write(`[js trace] ${msg}\n`) }; + initPyroscope(logger); + // Create the connection with the custom logger const connection = rpc.createMessageConnection( new rpc.StreamMessageReader(process.stdin), diff --git a/rewrite-python/rewrite/pyproject.toml b/rewrite-python/rewrite/pyproject.toml index 52b7dd6be46..72eff56d3de 100644 --- a/rewrite-python/rewrite/pyproject.toml +++ b/rewrite-python/rewrite/pyproject.toml @@ -41,6 +41,11 @@ publish = [ "build>=1.0.0", "twine>=5.0.0", ] +# Continuous profiling for production recipe-worker subprocesses; the RPC +# server's _init_pyroscope() is a no-op when this isn't installed. +profiling = [ + "pyroscope-io>=0.8.0", +] [project.urls] Homepage = "https://github.com/openrewrite/rewrite" diff --git a/rewrite-python/rewrite/src/rewrite/rpc/server.py b/rewrite-python/rewrite/src/rewrite/rpc/server.py index 7d9993df2ca..661210d55eb 100644 --- a/rewrite-python/rewrite/src/rewrite/rpc/server.py +++ b/rewrite-python/rewrite/src/rewrite/rpc/server.py @@ -1694,6 +1694,33 @@ def write_message(response: dict): os.write(sys.stdout.fileno(), header + content_bytes) +def _init_pyroscope() -> None: + """Start continuous profiling when PYROSCOPE_SERVER_ADDRESS is set. + + Tags inherited via PYROSCOPE_TAGS (k=v,k=v) are forwarded verbatim; a + `runtime=python` tag is added so flame graphs in the shared `modcli` + application can be sliced by which RPC subprocess produced them. + """ + server = os.environ.get("PYROSCOPE_SERVER_ADDRESS") + if not server: + return + try: + import pyroscope # type: ignore[import-not-found] + except ImportError: + logger.warning("PYROSCOPE_SERVER_ADDRESS set but pyroscope-io not installed; profiling disabled") + return + tags: Dict[str, str] = {"runtime": "python"} + for pair in os.environ.get("PYROSCOPE_TAGS", "").split(","): + if "=" in pair: + k, v = pair.split("=", 1) + tags[k.strip()] = v.strip() + pyroscope.configure( + application_name=os.environ.get("PYROSCOPE_APPLICATION_NAME", "modcli"), + server_address=server, + tags=tags, + ) + + def main(): """Main entry point for the RPC server.""" global _trace_rpc @@ -1705,6 +1732,8 @@ def main(): parser.add_argument('--recipe-install-dir', help='Directory where recipe pip packages are installed') args = parser.parse_args() + _init_pyroscope() + if args.recipe_install_dir: global _recipe_install_dir _recipe_install_dir = Path(args.recipe_install_dir)