From 455d92916a9c22a3e2ae2f094e00b60d520efd44 Mon Sep 17 00:00:00 2001 From: Patrick Quist Date: Tue, 7 Nov 2023 23:59:40 +0100 Subject: [PATCH] Execution with heaptrack (#5644) --- docs/API.md | 13 +- .../compiler-explorer.amazon.properties | 2 + .../compiler-explorer.defaults.properties | 2 + lib/base-compiler.ts | 211 +++++++-- lib/compiler-finder.ts | 1 + lib/compilers/dotnet.ts | 17 +- lib/compilers/fake-for-test.ts | 2 +- lib/handlers/compile.interfaces.ts | 9 +- lib/handlers/compile.ts | 31 +- lib/runtime-tools/_all.ts | 25 ++ lib/runtime-tools/base-runtime-tool.ts | 52 +++ lib/runtime-tools/heaptrack-wrapper.ts | 191 ++++++++ lib/utils.ts | 117 +++-- static/compiler-shared.interfaces.ts | 2 + static/compiler-shared.ts | 28 ++ static/components.interfaces.ts | 2 + static/components.ts | 3 + static/panes/compiler.interfaces.ts | 2 + static/panes/compiler.ts | 41 +- static/panes/executor.interfaces.ts | 4 +- static/panes/executor.ts | 72 ++- static/styles/explorer.scss | 74 +++- static/widgets/runtime-tools.ts | 412 ++++++++++++++++++ test/handlers/compile-tests.js | 3 +- types/compilation/compilation.interfaces.ts | 7 +- types/compiler.interfaces.ts | 4 +- types/execution/execution.interfaces.ts | 34 +- types/tool.interfaces.ts | 1 + views/popups/_all.pug | 2 + views/popups/overrides-selection.pug | 3 +- views/popups/runtimetools-selection.pug | 31 ++ views/templates/panes/executor.pug | 3 + views/templates/templates.pug | 2 + .../widgets/possible-runtime-tool-tpl.pug | 16 + 34 files changed, 1293 insertions(+), 126 deletions(-) create mode 100644 lib/runtime-tools/_all.ts create mode 100644 lib/runtime-tools/base-runtime-tool.ts create mode 100644 lib/runtime-tools/heaptrack-wrapper.ts create mode 100644 static/widgets/runtime-tools.ts create mode 100644 views/popups/runtimetools-selection.pug create mode 100644 views/templates/widgets/possible-runtime-tool-tpl.pug diff --git a/docs/API.md b/docs/API.md index 8433bf13f06..9409646f8bd 100644 --- a/docs/API.md +++ b/docs/API.md @@ -99,7 +99,18 @@ Execution Only request example: "userArguments": "-O3", "executeParameters": { "args": ["arg1", "arg2"], - "stdin": "hello, world!" + "stdin": "hello, world!", + "runtimeTools": [ + { + "name": "env", + "options": [ + { + "name": "MYENV", + "value": "123" + } + ] + } + ] }, "compilerOptions": { "executorRequest": true diff --git a/etc/config/compiler-explorer.amazon.properties b/etc/config/compiler-explorer.amazon.properties index dbc3c97c7ed..acaf22aa195 100644 --- a/etc/config/compiler-explorer.amazon.properties +++ b/etc/config/compiler-explorer.amazon.properties @@ -40,6 +40,8 @@ cmake=/opt/compiler-explorer/cmake/bin/cmake useninja=false ld=/usr/bin/ld readelf=/usr/bin/readelf +mkfifo=/usr/bin/mkfifo +heaptrackPath=/opt/compiler-explorer/heaptrack-v1.3.0 formatters=clangformat:rustfmt:gofmt:dartformat:vfmt formatter.clangformat.name=clang-format diff --git a/etc/config/compiler-explorer.defaults.properties b/etc/config/compiler-explorer.defaults.properties index acfdbfce13b..f8818d1eb67 100644 --- a/etc/config/compiler-explorer.defaults.properties +++ b/etc/config/compiler-explorer.defaults.properties @@ -50,6 +50,8 @@ cmake=cmake useninja=false ld=ld readelf=readelf +mkfifo=/usr/bin/mkfifo +heaptrackPath= # set this true to keep temporary folders for a while for debugging purposes delayCleanupTemp=false diff --git a/lib/base-compiler.ts b/lib/base-compiler.ts index 96f9359f602..08671d51905 100644 --- a/lib/base-compiler.ts +++ b/lib/base-compiler.ts @@ -48,16 +48,19 @@ import type { LLVMOptPipelineOutput, } from '../types/compilation/llvm-opt-pipeline-output.interfaces.js'; import type {CompilerInfo, ICompiler, PreliminaryCompilerInfo} from '../types/compiler.interfaces.js'; -import type { +import { BasicExecutionResult, + ConfiguredRuntimeTool, + ConfiguredRuntimeTools, ExecutableExecutionOptions, + RuntimeToolType, UnprocessedExecResult, } from '../types/execution/execution.interfaces.js'; import type {CompilerOutputOptions, ParseFiltersAndOutputOptions} from '../types/features/filters.interfaces.js'; import type {Language} from '../types/languages.interfaces.js'; import type {Library, LibraryVersion, SelectedLibraryVersion} from '../types/libraries/libraries.interfaces.js'; import type {ResultLine} from '../types/resultline/resultline.interfaces.js'; -import type {Artifact, ToolResult, ToolTypeKey} from '../types/tool.interfaces.js'; +import {ArtifactType, type Artifact, type ToolResult, type ToolTypeKey} from '../types/tool.interfaces.js'; import {BuildEnvSetupBase, getBuildEnvTypeByKey} from './buildenvsetup/index.js'; import type {BuildEnvDownloadInfo} from './buildenvsetup/buildenv.interfaces.js'; @@ -108,6 +111,7 @@ import {LLVMIrBackendOptions} from '../types/compilation/ir.interfaces.js'; import {ParsedAsmResultLine} from '../types/asmresult/asmresult.interfaces.js'; import {unique} from '../shared/common-utils.js'; import {ClientOptionsType, OptionsHandlerLibrary, VersionInfo} from './options-handler.js'; +import {HeaptrackWrapper} from './runtime-tools/heaptrack-wrapper.js'; import {propsFor} from './properties.js'; import stream from 'node:stream'; import {SentryCapture} from './sentry.js'; @@ -572,6 +576,22 @@ export class BaseCompiler implements ICompiler { }; } + processUserExecutableExecutionResult( + input: UnprocessedExecResult, + stdErrlineParseOptions: utils.LineParseOptions, + ): BasicExecutionResult { + const start = performance.now(); + const stdout = utils.parseOutput(input.stdout, undefined, undefined, []); + const stderr = utils.parseOutput(input.stderr, undefined, undefined, stdErrlineParseOptions); + const end = performance.now(); + return { + ...input, + stdout, + stderr, + processExecutionResultTime: end - start, + }; + } + getEmptyExecutionResult(): BasicExecutionResult { return { code: -1, @@ -594,16 +614,69 @@ export class BaseCompiler implements ICompiler { }; } + protected setEnvironmentVariablesFromRuntime( + configuredTools: ConfiguredRuntimeTools, + execOptions: ExecutionOptions, + ) { + for (const runtime of configuredTools) { + if (runtime.name === RuntimeToolType.env) { + for (const env of runtime.options) { + if (!execOptions.env) execOptions.env = {}; + + execOptions.env[env.name] = env.value; + } + } + } + } + + protected async execBinaryMaybeWrapped( + executable: string, + args: string[], + execOptions: ExecutionOptions, + executeParameters: ExecutableExecutionOptions, + homeDir: string, + ): Promise { + let runWithHeaptrack: ConfiguredRuntimeTool | undefined = undefined; + + if (!execOptions.env) execOptions.env = {}; + + if (executeParameters.runtimeTools) { + this.setEnvironmentVariablesFromRuntime(executeParameters.runtimeTools, execOptions); + + for (const runtime of executeParameters.runtimeTools) { + if (runtime.name === RuntimeToolType.heaptrack) { + runWithHeaptrack = runtime; + } + } + } + + if (runWithHeaptrack && HeaptrackWrapper.isSupported(this.env)) { + const wrapper = new HeaptrackWrapper( + homeDir, + exec.sandbox, + this.exec, + runWithHeaptrack.options, + this.env.ceProps, + this.sandboxType, + ); + const execResult: UnprocessedExecResult = await wrapper.exec(executable, args, execOptions); + return this.processUserExecutableExecutionResult(execResult, [utils.LineParseOption.AtFileLine]); + } else { + const execResult: UnprocessedExecResult = await exec.sandbox(executable, args, execOptions); + return this.processUserExecutableExecutionResult(execResult, []); + } + } + async execBinary( - executable, - maxSize, + executable: string, + maxSize: number, executeParameters: ExecutableExecutionOptions, - homeDir, + homeDir: string, ): Promise { // We might want to save this in the compilation environment once execution is made available const timeoutMs = this.env.ceProps('binaryExecTimeoutMs', 2000); try { - const execResult: UnprocessedExecResult = await exec.sandbox(executable, executeParameters.args, { + const execOptions: ExecutionOptions = { maxOutput: maxSize, timeoutMs: timeoutMs, ldPath: _.union(this.compiler.ldPath, executeParameters.ldPath), @@ -611,9 +684,15 @@ export class BaseCompiler implements ICompiler { env: executeParameters.env, customCwd: homeDir, appHome: homeDir, - }); + }; - return this.processExecutionResult(execResult); + return this.execBinaryMaybeWrapped( + executable, + executeParameters.args, + execOptions, + executeParameters, + homeDir, + ); } catch (err: UnprocessedExecResult | any) { if (err.code && err.stderr) { return this.processExecutionResult(err); @@ -1857,19 +1936,25 @@ export class BaseCompiler implements ICompiler { runExecutable(executable: string, executeParameters: ExecutableExecutionOptions, homeDir) { const maxExecOutputSize = this.env.ceProps('max-executable-output-size', 32 * 1024); + + const execOptionsCopy: ExecutableExecutionOptions = JSON.parse( + JSON.stringify(executeParameters), + ) as ExecutableExecutionOptions; + // Hardcoded fix for #2339. Ideally I'd have a config option for this, but for now this is plenty good enough. - executeParameters.env = { + execOptionsCopy.env = { ASAN_OPTIONS: 'color=always', UBSAN_OPTIONS: 'color=always', MSAN_OPTIONS: 'color=always', LSAN_OPTIONS: 'color=always', ...executeParameters.env, }; + if (this.compiler.executionWrapper) { - executeParameters.args = [...this.compiler.executionWrapperArgs, executable, ...executeParameters.args]; + execOptionsCopy.args = [...this.compiler.executionWrapperArgs, executable, ...execOptionsCopy.args]; executable = this.compiler.executionWrapper; } - return this.execBinary(executable, maxExecOutputSize, executeParameters, homeDir); + return this.execBinary(executable, maxExecOutputSize, execOptionsCopy, homeDir); } protected fixExecuteParametersForInterpreting(executeParameters, outputFilename, key) { @@ -1962,7 +2047,27 @@ export class BaseCompiler implements ICompiler { }; } - async handleExecution(key, executeParameters, bypassCache: BypassCache): Promise { + async addHeaptrackResults(result: CompilationResult, dirPath?: string) { + let dirPathToUse: string = ''; + if (dirPath) { + dirPathToUse = dirPath; + } else if (result.buildResult && result.buildResult.dirPath) { + dirPathToUse = result.buildResult.dirPath; + } + + if (dirPathToUse === '') return; + + const flamegraphFilepath = path.join(dirPathToUse, HeaptrackWrapper.FlamegraphFilename); + if (await utils.fileExists(flamegraphFilepath)) { + await this.addArtifactToResult(result, flamegraphFilepath, ArtifactType.heaptracktxt, 'Heaptrack results'); + } + } + + async handleExecution( + key, + executeParameters: ExecutableExecutionOptions, + bypassCache: BypassCache, + ): Promise { // stringify now so shallow copying isn't a problem, I think the executeParameters get modified const execKey = JSON.stringify({key, executeParameters}); if (!bypassExecutionCache(bypassCache)) { @@ -1973,6 +2078,15 @@ export class BaseCompiler implements ICompiler { } const result = await this.doExecution(key, executeParameters, bypassCache); + + if (executeParameters.runtimeTools) { + for (const runtime of executeParameters.runtimeTools) { + if (runtime.name === RuntimeToolType.heaptrack) { + await this.addHeaptrackResults(result); + } + } + } + if (!bypassExecutionCache(bypassCache)) { await this.env.cachePut(execKey, result, undefined); } @@ -1990,7 +2104,7 @@ export class BaseCompiler implements ICompiler { cacheKey.api = 'cmake'; if (cacheKey.filters) delete cacheKey.filters.execute; - delete cacheKey.executionParameters; + delete cacheKey.executeParameters; delete cacheKey.tools; return cacheKey; @@ -2379,7 +2493,7 @@ export class BaseCompiler implements ICompiler { } async cmake(files, key, bypassCache: BypassCache) { - // key = {source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries}; + // key = {source, options, backendOptions, filters, bypassCache, tools, executeParameters, libraries}; if (!this.compiler.supportsBinary) { const errorResult: CompilationResult = { @@ -2403,10 +2517,12 @@ export class BaseCompiler implements ICompiler { const toolchainPath = this.getDefaultOrOverridenToolchainPath(key.backendOptions.overrides || []); const doExecute = key.filters.execute; - const executeParameters: ExecutableExecutionOptions = { + + const executeOptions: ExecutableExecutionOptions = { + args: key.executeParameters.args || [], + stdin: key.executeParameters.stdin || '', ldPath: this.getSharedLibraryPathsAsLdLibraryPaths(key.libraries), - args: key.executionParameters.args || [], - stdin: key.executionParameters.stdin || '', + runtimeTools: key.executeParameters?.runtimeTools || [], env: {}, }; @@ -2524,8 +2640,16 @@ export class BaseCompiler implements ICompiler { fullResult.result.dirPath = dirPath; if (this.compiler.supportsExecute && doExecute) { - fullResult.execResult = await this.runExecutable(outputFilename, executeParameters, dirPath); + fullResult.execResult = await this.runExecutable(outputFilename, executeOptions, dirPath); fullResult.didExecute = true; + + if (executeOptions.runtimeTools) { + for (const runtime of executeOptions.runtimeTools) { + if (runtime.name === RuntimeToolType.heaptrack) { + await this.addHeaptrackResults(fullResult, dirPath); + } + } + } } const optOutput = undefined; @@ -2534,7 +2658,7 @@ export class BaseCompiler implements ICompiler { fullResult.result, false, cacheKey, - [], + executeOptions, key.tools, cacheKey.backendOptions, cacheKey.filters, @@ -2597,7 +2721,7 @@ export class BaseCompiler implements ICompiler { filters, bypassCache: BypassCache, tools, - executionParameters, + executeParameters, libraries: CompileChildLibraries[], files, ) { @@ -2614,9 +2738,12 @@ export class BaseCompiler implements ICompiler { this.fixFiltersBeforeCacheKey(filters, options, files); - const executeParameters = { - args: executionParameters.args || [], - stdin: executionParameters.stdin || '', + const executeOptions: ExecutableExecutionOptions = { + args: executeParameters.args || [], + stdin: executeParameters.stdin || '', + ldPath: [], + env: {}, + runtimeTools: executeParameters.runtimeTools || [], }; const key = this.getCacheKey(source, options, backendOptions, filters, tools, libraries, files); @@ -2644,7 +2771,7 @@ export class BaseCompiler implements ICompiler { async () => { const start = performance.now(); executionQueueTimeHistogram.observe((start - queueTime) / 1000); - const res = await this.handleExecution(key, executeParameters, bypassCache); + const res = await this.handleExecution(key, executeOptions, bypassCache); executionTimeHistogram.observe((performance.now() - start) / 1000); return res; }, @@ -2667,7 +2794,7 @@ export class BaseCompiler implements ICompiler { source = this.preProcess(source, filters); if (backendOptions.executorRequest) { - const execResult = await this.handleExecution(key, executeParameters, bypassCache); + const execResult = await this.handleExecution(key, executeOptions, bypassCache); if (execResult && execResult.buildResult) { this.doTempfolderCleanup(execResult.buildResult); } @@ -2699,7 +2826,7 @@ export class BaseCompiler implements ICompiler { result, doExecute, key, - executeParameters, + executeOptions, tools, backendOptions, filters, @@ -2720,7 +2847,7 @@ export class BaseCompiler implements ICompiler { result, doExecute, key, - executeParameters, + executeOptions: ExecutableExecutionOptions, tools, backendOptions, filters, @@ -2732,7 +2859,7 @@ export class BaseCompiler implements ICompiler { ) { // Start the execution as soon as we can, but only await it at the end. const execPromise = - doExecute && result.code === 0 ? this.handleExecution(key, executeParameters, bypassCache) : null; + doExecute && result.code === 0 ? this.handleExecution(key, executeOptions, bypassCache) : null; if (result.hasOptOutput) { delete result.optPath; @@ -3171,6 +3298,33 @@ but nothing was dumped. Possible causes are: return await parser.getPossibleStdvers(this); } + async populatePossibleRuntimeTools() { + this.compiler.possibleRuntimeTools = []; + + if (HeaptrackWrapper.isSupported(this.env)) { + this.compiler.possibleRuntimeTools.push({ + name: RuntimeToolType.heaptrack, + description: + 'Heaptrack gets loaded into your code and collects the heap allocations, ' + + "we'll display them in a flamegraph.", + possibleOptions: [ + { + name: 'graph', + possibleValues: ['yes'], + }, + { + name: 'summary', + possibleValues: ['stderr'], + }, + { + name: 'details', + possibleValues: ['stderr'], + }, + ], + }); + } + } + async populatePossibleOverrides() { const targets = await this.getTargetsAsOverrideValues(); if (targets.length > 0) { @@ -3289,6 +3443,7 @@ but nothing was dumped. Possible causes are: const initResult = await this.getArgumentParser().parse(this); await this.populatePossibleOverrides(); + await this.populatePossibleRuntimeTools(); logger.info(`${compiler} ${version} is ready`); return initResult; diff --git a/lib/compiler-finder.ts b/lib/compiler-finder.ts index e97b29fed2f..c4b613f50c9 100644 --- a/lib/compiler-finder.ts +++ b/lib/compiler-finder.ts @@ -347,6 +347,7 @@ export class CompilerFinder { preamble: props('licensePreamble'), }, possibleOverrides: [], + possibleRuntimeTools: [], $order: undefined as unknown as number, // TODO(jeremy-rifkin): Very dirty }; diff --git a/lib/compilers/dotnet.ts b/lib/compilers/dotnet.ts index 5a1f4c12f9d..ad5709447b8 100644 --- a/lib/compilers/dotnet.ts +++ b/lib/compilers/dotnet.ts @@ -29,14 +29,12 @@ import _ from 'underscore'; import type {CompilationResult, ExecutionOptions} from '../../types/compilation/compilation.interfaces.js'; import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; -import type { - BasicExecutionResult, - ExecutableExecutionOptions, - UnprocessedExecResult, +import { + type BasicExecutionResult, + type ExecutableExecutionOptions, } from '../../types/execution/execution.interfaces.js'; import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; import {BaseCompiler} from '../base-compiler.js'; -import * as exec from '../exec.js'; import {DotNetAsmParser} from '../parsers/asm-parser-dotnet.js'; import * as utils from '../utils.js'; @@ -437,9 +435,9 @@ class DotNetCompiler extends BaseCompiler { override async execBinary( executable: string, - maxSize: number | undefined, + maxSize: number, executeParameters: ExecutableExecutionOptions, - homeDir: string | undefined, + homeDir: string, ): Promise { const programDir = path.dirname(executable); const programOutputPath = path.join(programDir, 'bin', this.buildConfig, this.targetFramework); @@ -457,9 +455,8 @@ class DotNetCompiler extends BaseCompiler { execOptions.input = executeParameters.stdin; const execArgs = ['-p', 'System.Runtime.TieredCompilation=false', programDllPath, ...executeParameters.args]; try { - const execResult: UnprocessedExecResult = await exec.sandbox(this.corerunPath, execArgs, execOptions); - return this.processExecutionResult(execResult); - } catch (err: UnprocessedExecResult | any) { + return this.execBinaryMaybeWrapped(this.corerunPath, execArgs, execOptions, executeParameters, homeDir); + } catch (err: any) { if (err.code && err.stderr) { return this.processExecutionResult(err); } else { diff --git a/lib/compilers/fake-for-test.ts b/lib/compilers/fake-for-test.ts index 60616207c0c..f62d534fb82 100644 --- a/lib/compilers/fake-for-test.ts +++ b/lib/compilers/fake-for-test.ts @@ -67,7 +67,7 @@ export class FakeCompiler implements ICompiler { return null; } - compile(source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries, files) { + compile(source, options, backendOptions, filters, bypassCache, tools, executeParameters, libraries, files) { const inputBody = { input: { source: source, diff --git a/lib/handlers/compile.interfaces.ts b/lib/handlers/compile.interfaces.ts index 0df959fd432..464f0c647ae 100644 --- a/lib/handlers/compile.interfaces.ts +++ b/lib/handlers/compile.interfaces.ts @@ -22,7 +22,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -import {BypassCache} from '../../types/compilation/compilation.interfaces.js'; +import {BypassCache, ExecutionParams} from '../../types/compilation/compilation.interfaces.js'; // IF YOU MODIFY ANYTHING HERE PLEASE UPDATE THE DOCUMENTATION! @@ -36,16 +36,11 @@ export type CompileRequestQueryArgs = { skipPopArgs?: string; }; -export type ExecutionRequestParams = { - args?: string | string[]; - stdin?: string; -}; - // TODO find more types for these. export type CompilationRequestArgs = { userArguments: string; compilerOptions: Record; - executeParameters: ExecutionRequestParams; + executeParameters: ExecutionParams; filters: Record; tools: any; libraries: any[]; diff --git a/lib/handlers/compile.ts b/lib/handlers/compile.ts index 9350a987878..f5eadf28174 100644 --- a/lib/handlers/compile.ts +++ b/lib/handlers/compile.ts @@ -42,12 +42,7 @@ import {logger} from '../logger.js'; import {PropertyGetter} from '../properties.interfaces.js'; import * as utils from '../utils.js'; -import { - CompileRequestJsonBody, - CompileRequestQueryArgs, - CompileRequestTextBody, - ExecutionRequestParams, -} from './compile.interfaces.js'; +import {CompileRequestJsonBody, CompileRequestQueryArgs, CompileRequestTextBody} from './compile.interfaces.js'; import {remove} from '../../shared/common-utils.js'; import {CompilerOverrideOptions} from '../../types/compilation/compiler-overrides.interfaces.js'; import {BypassCache, CompileChildLibraries, ExecutionParams} from '../../types/compilation/compilation.interfaces.js'; @@ -92,7 +87,7 @@ type ParsedRequest = { filters: ParseFiltersAndOutputOptions; bypassCache: BypassCache; tools: any; - executionParameters: ExecutionParams; + executeParameters: ExecutionParams; libraries: CompileChildLibraries[]; }; @@ -353,7 +348,7 @@ export class CompileHandler { filters: ParseFiltersAndOutputOptions, bypassCache = BypassCache.None, tools; - const execReqParams: ExecutionRequestParams = {}; + const execReqParams: ExecutionParams = {}; let libraries: any[] = []; // IF YOU MODIFY ANYTHING HERE PLEASE UPDATE THE DOCUMENTATION! if (req.is('json')) { @@ -366,6 +361,7 @@ export class CompileHandler { const execParams = requestOptions.executeParameters || {}; execReqParams.args = execParams.args; execReqParams.stdin = execParams.stdin; + execReqParams.runtimeTools = execParams.runtimeTools; backendOptions = requestOptions.compilerOptions || {}; filters = {...compiler.getDefaultFilters(), ...requestOptions.filters}; tools = requestOptions.tools; @@ -410,11 +406,12 @@ export class CompileHandler { backendOptions.skipAsm = query.skipAsm === 'true'; backendOptions.skipPopArgs = query.skipPopArgs === 'true'; } - const executionParameters: ExecutionParams = { + const executeParameters: ExecutionParams = { args: Array.isArray(execReqParams.args) ? execReqParams.args || '' : utils.splitArguments(execReqParams.args), stdin: execReqParams.stdin || '', + runtimeTools: execReqParams.runtimeTools || [], }; tools = tools || []; @@ -433,7 +430,7 @@ export class CompileHandler { filters, bypassCache, tools, - executionParameters, + executeParameters, libraries, }; } @@ -539,7 +536,7 @@ export class CompileHandler { return this.handleApiError(error, res, next); } - const {source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries} = + const {source, options, backendOptions, filters, bypassCache, tools, executeParameters, libraries} = parsedRequest; let files; @@ -563,17 +560,7 @@ export class CompileHandler { this.compileCounter.inc({language: compiler.lang.id}); // eslint-disable-next-line promise/catch-or-return compiler - .compile( - source, - options, - backendOptions, - filters, - bypassCache, - tools, - executionParameters, - libraries, - files, - ) + .compile(source, options, backendOptions, filters, bypassCache, tools, executeParameters, libraries, files) .then( result => { if (result.didExecute || (result.execResult && result.execResult.didExecute)) diff --git a/lib/runtime-tools/_all.ts b/lib/runtime-tools/_all.ts new file mode 100644 index 00000000000..c8d11b8b4e1 --- /dev/null +++ b/lib/runtime-tools/_all.ts @@ -0,0 +1,25 @@ +// Copyright (c) 2023, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +export {HeaptrackWrapper} from './heaptrack-wrapper.js'; diff --git a/lib/runtime-tools/base-runtime-tool.ts b/lib/runtime-tools/base-runtime-tool.ts new file mode 100644 index 00000000000..d3de792131a --- /dev/null +++ b/lib/runtime-tools/base-runtime-tool.ts @@ -0,0 +1,52 @@ +// Copyright (c) 2023, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import {RuntimeToolOptions, TypicalExecutionFunc} from '../../types/execution/execution.interfaces.js'; + +export class BaseRuntimeTool { + protected dirPath: string; + protected sandboxFunc: TypicalExecutionFunc; + protected execFunc: TypicalExecutionFunc; + protected options: RuntimeToolOptions; + protected sandboxType: string; + + constructor( + dirPath: string, + sandboxFunc: TypicalExecutionFunc, + execFunc: TypicalExecutionFunc, + options: RuntimeToolOptions, + sandboxType: string, + ) { + this.dirPath = dirPath; + this.sandboxFunc = sandboxFunc; + this.execFunc = execFunc; + this.options = options; + this.sandboxType = sandboxType; + } + + protected getOptionValue(name: string): string | undefined { + const option = this.options.find(opt => opt.name === name); + if (option) return option.value; + } +} diff --git a/lib/runtime-tools/heaptrack-wrapper.ts b/lib/runtime-tools/heaptrack-wrapper.ts new file mode 100644 index 00000000000..ef46a359820 --- /dev/null +++ b/lib/runtime-tools/heaptrack-wrapper.ts @@ -0,0 +1,191 @@ +// Copyright (c) 2023, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import * as path from 'path'; +import {ExecutionOptions} from '../../types/compilation/compilation.interfaces.js'; +import { + RuntimeToolOptions, + TypicalExecutionFunc, + UnprocessedExecResult, +} from '../../types/execution/execution.interfaces.js'; +import {O_NONBLOCK, O_RDWR} from 'constants'; +import * as fs from 'fs'; +import * as net from 'net'; +import {pipeline} from 'stream'; +import {unwrap} from '../assert.js'; +import {logger} from '../logger.js'; +import {executeDirect} from '../exec.js'; +import {PropertyGetter} from '../properties.interfaces.js'; +import {BaseRuntimeTool} from './base-runtime-tool.js'; +import {CompilationEnvironment} from '../compilation-env.js'; + +export class HeaptrackWrapper extends BaseRuntimeTool { + private rawOutput: string; + private pipe: string; + private interpretedPath: string; + private heaptrackPath: string; + private mkfifoPath: string; + private preload: string; + private interpreter: string; + private printer: string; + + public static FlamegraphFilename = 'heaptrack.flamegraph.txt'; + + constructor( + dirPath: string, + sandboxFunc: TypicalExecutionFunc, + execFunc: TypicalExecutionFunc, + options: RuntimeToolOptions, + ceProps: PropertyGetter, + sandboxType: string, + ) { + super(dirPath, sandboxFunc, execFunc, options, sandboxType); + + this.mkfifoPath = ceProps('mkfifo', '/usr/bin/mkfifo'); + + this.pipe = path.join(this.dirPath, 'heaptrack_fifo'); + this.rawOutput = path.join(this.dirPath, 'heaptrack_raw.txt'); + this.interpretedPath = path.join(this.dirPath, 'heaptrack_interpreted.txt'); + + this.heaptrackPath = ceProps('heaptrackPath', ''); + + this.preload = path.join(this.heaptrackPath, 'lib/libheaptrack_preload.so'); + this.interpreter = path.join(this.heaptrackPath, 'libexec/heaptrack_interpret'); + this.printer = path.join(this.heaptrackPath, 'bin/heaptrack_print'); + } + + public static isSupported(compiler: CompilationEnvironment) { + return process.platform !== 'win32' && compiler.ceProps('heaptrackPath', '') !== ''; + } + + private async mkfifo(path: string, rights: number) { + await executeDirect(this.mkfifoPath, ['-m', rights.toString(8), path], {}); + } + + private async makePipe() { + await this.mkfifo(this.pipe, 0o666); + } + + private addToEnv(execOptions: ExecutionOptions) { + if (!execOptions.env) execOptions.env = {}; + + if (execOptions.env.LD_PRELOAD) { + execOptions.env.LD_PRELOAD = this.preload + ':' + execOptions.env.LD_PRELOAD; + } else { + execOptions.env.LD_PRELOAD = this.preload; + } + + if (this.sandboxType === 'nsjail') { + execOptions.env.DUMP_HEAPTRACK_OUTPUT = '/app/heaptrack_fifo'; + } else { + execOptions.env.DUMP_HEAPTRACK_OUTPUT = this.pipe; + } + } + + private async interpret(execOptions: ExecutionOptions): Promise { + return this.execFunc(this.interpreter, [this.rawOutput], execOptions); + } + + private async finishPipesAndStreams(fd: number, file: fs.WriteStream, socket: net.Socket) { + socket.push(null); + await new Promise(resolve => socket.end(() => resolve(true))); + + await new Promise(resolve => file.end(() => resolve(true))); + + file.write(Buffer.from([0])); + + socket.resetAndDestroy(); + socket.unref(); + + await new Promise(resolve => { + file.close(err => { + if (err) logger.error('Error while closing heaptrack log: ', err); + resolve(true); + }); + }); + + await new Promise(resolve => fs.close(fd, () => resolve(true))); + } + + private async interpretAndSave(execOptions: ExecutionOptions, result: UnprocessedExecResult) { + const dirPath = unwrap(execOptions.appHome); + execOptions.input = fs.readFileSync(this.rawOutput).toString('utf8'); + + const interpretResults = await this.interpret(execOptions); + + if (this.getOptionValue('summary') === 'stderr') { + result.stderr += interpretResults.stderr; + } + + fs.writeFileSync(this.interpretedPath, interpretResults.stdout); + } + + private async saveFlamegraph(execOptions: ExecutionOptions, result: UnprocessedExecResult) { + const args = [this.interpretedPath]; + + if (this.getOptionValue('graph') === 'yes') { + const flamesFilepath = path.join(this.dirPath, HeaptrackWrapper.FlamegraphFilename); + args.push('-F', flamesFilepath); + } + + const printResults = await this.execFunc(this.printer, args, execOptions); + if (printResults.stderr) result.stderr += printResults.stderr; + + if (this.getOptionValue('details') === 'stderr') { + result.stderr += printResults.stdout; + } + } + + public async exec(filepath: string, args: string[], execOptions: ExecutionOptions): Promise { + const dirPath = unwrap(execOptions.appHome); + + const runOptions = JSON.parse(JSON.stringify(execOptions)); + const interpretOptions = JSON.parse(JSON.stringify(execOptions)); + this.addToEnv(runOptions); + + await this.makePipe(); + + const fd = fs.openSync(this.pipe, O_NONBLOCK | O_RDWR); + const socket = new net.Socket({fd, readable: true, writable: true}); + + const file = fs.createWriteStream(this.rawOutput); + pipeline(socket, file, err => { + if (err) { + logger.error('Error during heaptrack pipeline: ', err); + } + }); + + const result = await this.sandboxFunc(filepath, args, runOptions); + + await this.finishPipesAndStreams(fd, file, socket); + + fs.unlinkSync(this.pipe); + + await this.interpretAndSave(interpretOptions, result); + + await this.saveFlamegraph(execOptions, result); + + return result; + } +} diff --git a/lib/utils.ts b/lib/utils.ts index bf15b1cdf5a..ae2b352f65f 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -104,40 +104,103 @@ function parseSeverity(message: string): number { const SOURCE_RE = /^\s*[(:](\d+)(:?,?(\d+):?)?[):]*\s*(.*)/; const SOURCE_WITH_FILENAME = /^\s*([\w.]*)[(:](\d+)(:?,?(\d+):?)?[):]*\s*(.*)/; +const ATFILELINE_RE = /\s*at ([\w-/.]*):(\d+)/; + +export enum LineParseOption { + SourceMasking, + RootMasking, + SourceWithLineMessage, + FileWithLineMessage, + AtFileLine, +} + +export type LineParseOptions = LineParseOption[]; + +export const DefaultLineParseOptions = [ + LineParseOption.SourceMasking, + LineParseOption.RootMasking, + LineParseOption.SourceWithLineMessage, + LineParseOption.FileWithLineMessage, +]; + +function applyParse_SourceWithLine(lineObj: ResultLine, filteredLine: string, inputFilename?: string) { + const match = filteredLine.match(SOURCE_RE); + if (match) { + const message = match[4].trim(); + lineObj.tag = { + line: parseInt(match[1]), + column: parseInt(match[3] || '0'), + text: message, + severity: parseSeverity(message), + file: inputFilename ? path.basename(inputFilename) : undefined, + }; + } +} + +function applyParse_FileWithLine(lineObj: ResultLine, filteredLine: string) { + const match = filteredLine.match(SOURCE_WITH_FILENAME); + if (match) { + const message = match[5].trim(); + lineObj.tag = { + file: match[1], + line: parseInt(match[2]), + column: parseInt(match[4] || '0'), + text: message, + severity: parseSeverity(message), + }; + } +} + +function applyParse_AtFileLine(lineObj: ResultLine, filteredLine: string) { + const match = filteredLine.match(ATFILELINE_RE); + if (match) { + if (match[1].startsWith('/app/')) { + lineObj.tag = { + file: match[1].replace(/^\/app\//, ''), + line: parseInt(match[2]), + column: 0, + text: filteredLine, + severity: 3, + }; + } else if (!match[1].startsWith('/')) { + lineObj.tag = { + file: match[1], + line: parseInt(match[2]), + column: 0, + text: filteredLine, + severity: 3, + }; + } + } +} -export function parseOutput(lines: string, inputFilename?: string, pathPrefix?: string): ResultLine[] { +export function parseOutput( + lines: string, + inputFilename?: string, + pathPrefix?: string, + options: LineParseOptions = DefaultLineParseOptions, +): ResultLine[] { const result: ResultLine[] = []; eachLine(lines, line => { - line = _parseOutputLine(line, inputFilename, pathPrefix); - if (!inputFilename) { + if (options.includes(LineParseOption.SourceMasking)) { + line = _parseOutputLine(line, inputFilename, pathPrefix); + } + if (!inputFilename && options.includes(LineParseOption.RootMasking)) { line = maskRootdir(line); } if (line !== null) { const lineObj: ResultLine = {text: line}; - const filteredline = line.replace(ansiColoursRe, ''); - let match = filteredline.match(SOURCE_RE); - if (match) { - const message = match[4].trim(); - lineObj.tag = { - line: parseInt(match[1]), - column: parseInt(match[3] || '0'), - text: message, - severity: parseSeverity(message), - file: inputFilename ? path.basename(inputFilename) : undefined, - }; - } else { - match = filteredline.match(SOURCE_WITH_FILENAME); - if (match) { - const message = match[5].trim(); - lineObj.tag = { - file: match[1], - line: parseInt(match[2]), - column: parseInt(match[4] || '0'), - text: message, - severity: parseSeverity(message), - }; - } - } + const filteredLine = line.replace(ansiColoursRe, ''); + + if (options.includes(LineParseOption.SourceWithLineMessage)) + applyParse_SourceWithLine(lineObj, filteredLine, inputFilename); + + if (!lineObj.tag && options.includes(LineParseOption.FileWithLineMessage)) + applyParse_FileWithLine(lineObj, filteredLine); + + if (!lineObj.tag && options.includes(LineParseOption.AtFileLine)) + applyParse_AtFileLine(lineObj, filteredLine); + result.push(lineObj); } }); diff --git a/static/compiler-shared.interfaces.ts b/static/compiler-shared.interfaces.ts index a04817f81e6..fbe4306fc05 100644 --- a/static/compiler-shared.interfaces.ts +++ b/static/compiler-shared.interfaces.ts @@ -23,10 +23,12 @@ // POSSIBILITY OF SUCH DAMAGE. import type {ConfiguredOverrides} from './compilation/compiler-overrides.interfaces.js'; +import type {ConfiguredRuntimeTools} from './execution/execution.interfaces.js'; import type {CompilerState} from './panes/compiler.interfaces.js'; import type {ExecutorState} from './panes/executor.interfaces.js'; export interface ICompilerShared { updateState(state: CompilerState | ExecutorState); getOverrides(): ConfiguredOverrides | undefined; + getRuntimeTools(): ConfiguredRuntimeTools | undefined; } diff --git a/static/compiler-shared.ts b/static/compiler-shared.ts index bb3ddaba2e8..6a7bba38129 100644 --- a/static/compiler-shared.ts +++ b/static/compiler-shared.ts @@ -27,11 +27,15 @@ import {CompilerOverridesWidget} from './widgets/compiler-overrides.js'; import type {CompilerState} from './panes/compiler.interfaces.js'; import type {ConfiguredOverrides} from './compilation/compiler-overrides.interfaces.js'; import type {ExecutorState} from './panes/executor.interfaces.js'; +import {RuntimeToolsWidget} from './widgets/runtime-tools.js'; +import {ConfiguredRuntimeTools} from './execution/execution.interfaces.js'; export class CompilerShared implements ICompilerShared { private domRoot: JQuery; private overridesButton: JQuery; private overridesWidget: CompilerOverridesWidget; + private runtimeToolsButton: JQuery; + private runtimeToolsWidget?: RuntimeToolsWidget; constructor(domRoot: JQuery, onChange: () => void) { this.domRoot = domRoot; @@ -43,6 +47,10 @@ export class CompilerShared implements ICompilerShared { return this.overridesWidget.get(); } + public getRuntimeTools(): ConfiguredRuntimeTools | undefined { + return this.runtimeToolsWidget?.get(); + } + public updateState(state: CompilerState | ExecutorState) { this.overridesWidget.setCompiler(state.compiler, state.lang); @@ -51,17 +59,37 @@ export class CompilerShared implements ICompilerShared { } else { this.overridesWidget.setDefaults(); } + + if (this.runtimeToolsWidget) { + this.runtimeToolsWidget.setCompiler(state.compiler, state.lang); + if (state.runtimeTools) { + this.runtimeToolsWidget.set(state.runtimeTools); + } else { + this.runtimeToolsWidget.setDefaults(); + } + } } private initButtons(onChange: () => void) { this.overridesButton = this.domRoot.find('.btn.show-overrides'); this.overridesWidget = new CompilerOverridesWidget(this.domRoot, this.overridesButton, onChange); + + this.runtimeToolsButton = this.domRoot.find('.btn.show-runtime-tools'); + if (this.runtimeToolsButton.length > 0) { + this.runtimeToolsWidget = new RuntimeToolsWidget(this.domRoot, this.runtimeToolsButton, onChange); + } } private initCallbacks() { this.overridesButton.on('click', () => { this.overridesWidget.show(); }); + + if (this.runtimeToolsButton.length > 0) { + this.runtimeToolsButton.on('click', () => { + this.runtimeToolsWidget?.show(); + }); + } } } diff --git a/static/components.interfaces.ts b/static/components.interfaces.ts index 41481ca5318..932b3214b49 100644 --- a/static/components.interfaces.ts +++ b/static/components.interfaces.ts @@ -27,6 +27,7 @@ import {CfgState} from './panes/cfg-view.interfaces.js'; import {LLVMOptPipelineViewState} from './panes/llvm-opt-pipeline.interfaces.js'; import {GccDumpViewState} from './panes/gccdump-view.interfaces.js'; import {ConfiguredOverrides} from './compilation/compiler-overrides.interfaces.js'; +import {ConfiguredRuntimeTools} from './execution/execution.interfaces.js'; import {IrState} from './panes/ir-view.interfaces.js'; export const COMPILER_COMPONENT_NAME = 'compiler'; export const EXECUTOR_COMPONENT_NAME = 'executor'; @@ -94,6 +95,7 @@ export type PopulatedExecutorState = StateWithLanguage & compilationPanelShown: boolean; compilerOutShown: boolean; overrides?: ConfiguredOverrides; + runtimeTools?: ConfiguredRuntimeTools; }; export type ExecutorForTreeState = StateWithLanguage & StateWithTree & { diff --git a/static/components.ts b/static/components.ts index 8709d1f9ab9..fad988dc28b 100644 --- a/static/components.ts +++ b/static/components.ts @@ -108,6 +108,7 @@ import { EmptyStackUsageViewState, } from './components.interfaces.js'; import {ConfiguredOverrides} from './compilation/compiler-overrides.interfaces.js'; +import {ConfiguredRuntimeTools} from './execution/execution.interfaces.js'; /** Get an empty compiler component. */ export function getCompiler(editorId: number, lang: string): ComponentConfig { @@ -186,6 +187,7 @@ export function getExecutorWith( compilerArgs, treeId: number, overrides?: ConfiguredOverrides, + runtimeTools?: ConfiguredRuntimeTools, ): ComponentConfig { return { type: 'component', @@ -200,6 +202,7 @@ export function getExecutorWith( compilationPanelShown: true, compilerOutShown: true, overrides: overrides, + runtimeTools: runtimeTools, }, }; } diff --git a/static/panes/compiler.interfaces.ts b/static/panes/compiler.interfaces.ts index 348ef03c8f5..d1afaba0df1 100644 --- a/static/panes/compiler.interfaces.ts +++ b/static/panes/compiler.interfaces.ts @@ -23,6 +23,7 @@ // POSSIBILITY OF SUCH DAMAGE. import type {ConfiguredOverrides} from '../compilation/compiler-overrides.interfaces.js'; +import {ConfiguredRuntimeTools} from '../execution/execution.interfaces.js'; import {WidgetState} from '../widgets/libs-widget.interfaces.js'; import {MonacoPaneState} from './pane.interfaces.js'; @@ -36,6 +37,7 @@ export type CompilerState = WidgetState & { wantOptInfo?: boolean; lang?: string; overrides?: ConfiguredOverrides; + runtimeTools?: ConfiguredRuntimeTools; }; // TODO(jeremy-rifkin): This omit is ugly. There should be a better way to do this. diff --git a/static/panes/compiler.ts b/static/panes/compiler.ts index b6c9d058e1a..35aec07217b 100644 --- a/static/panes/compiler.ts +++ b/static/panes/compiler.ts @@ -679,6 +679,7 @@ export class Compiler extends MonacoPanehere' + + ' to view ' + + artifact.title + + ' in Speedscope', + { + group: artifact.type, + collapseSimilar: false, + dismissTime: 10000, + onBeforeShow: function (elem) { + elem.find('#download_link').on('click', () => { + const tmstr = Date.now(); + const live_url = 'https://static.ce-cdn.net/speedscope/index.html'; + const speedscope_url = + live_url + + '?' + + tmstr + + '#customFilename=' + + artifact.name + + '&b64data=' + + artifact.content; + window.open(speedscope_url); + }); + }, + }, + ); + } + offerViewInPerfetto(artifact: Artifact): void { this.alertSystem.notify( 'Click ' + @@ -1784,8 +1818,8 @@ export class Compiler extends MonacoPane { @@ -3202,6 +3236,7 @@ export class Compiler extends MonacoPane { executeParameters: { args: this.executionArguments, stdin: this.executionStdin, + runtimeTools: this.compilerShared.getRuntimeTools(), }, compilerOptions: { executorRequest: true, @@ -678,6 +680,48 @@ export class Executor extends Pane { if (this.currentLangId) this.eventHub.emit('executeResult', this.id, this.compiler, result, languages[this.currentLangId]); + + this.offerFilesIfPossible(result); + } + + offerFilesIfPossible(result: CompilationResult) { + if (result.artifacts) { + for (const artifact of result.artifacts) { + if (artifact.type === ArtifactType.heaptracktxt) { + this.offerViewInSpeedscope(artifact); + } + } + } + } + + offerViewInSpeedscope(artifact: Artifact): void { + this.alertSystem.notify( + 'Click ' + + 'here' + + ' to view ' + + artifact.title + + ' in Speedscope', + { + group: artifact.type, + collapseSimilar: false, + dismissTime: 10000, + onBeforeShow: function (elem) { + elem.find('#download_link').on('click', () => { + const tmstr = Date.now(); + const live_url = 'https://static.ce-cdn.net/speedscope/index.html'; + const speedscope_url = + live_url + + '?' + + tmstr + + '#customFilename=' + + artifact.name + + '&b64data=' + + artifact.content; + window.open(speedscope_url); + }); + }, + }, + ); } onCompileResponse(request: CompilationRequest, result: CompilationResult, cached: boolean): void { @@ -973,35 +1017,34 @@ export class Executor extends Pane { return this.settings.executorCompileOnChange; } - onOptionsChange(options: string): void { - this.options = options; + doTypicalOnChange() { this.updateState(); if (this.shouldEmitExecutionOnFieldChange()) { this.compile(); } } + onOptionsChange(options: string): void { + this.options = options; + this.doTypicalOnChange(); + } + onExecArgsChange(args: string): void { this.executionArguments = args; - this.updateState(); - if (this.shouldEmitExecutionOnFieldChange()) { - this.compile(); - } + this.doTypicalOnChange(); } onCompilerOverridesChange(): void { - this.updateState(); - if (this.shouldEmitExecutionOnFieldChange()) { - this.compile(); - } + this.doTypicalOnChange(); + } + + onRuntimeToolsChange(): void { + this.doTypicalOnChange(); } onExecStdinChange(newStdin: string): void { this.executionStdin = newStdin; - this.updateState(); - if (this.shouldEmitExecutionOnFieldChange()) { - this.compile(); - } + this.doTypicalOnChange(); } onRequestCompilation(editorId: number | boolean, treeId: number | boolean): void { @@ -1086,6 +1129,7 @@ export class Executor extends Pane { stdinPanelShown: !this.panelStdin.hasClass('d-none'), wrap: this.toggleWrapButton.get().wrap, overrides: this.compilerShared.getOverrides(), + runtimeTools: this.compilerShared.getRuntimeTools(), }; this.paneRenaming.addState(state); diff --git a/static/styles/explorer.scss b/static/styles/explorer.scss index 803465b9239..df5ed3b6fe3 100644 --- a/static/styles/explorer.scss +++ b/static/styles/explorer.scss @@ -380,6 +380,10 @@ pre.content.wrap * { margin-right: 5px; } +.toast a { + text-decoration: underline !important; +} + .font-size-list { min-width: 43px !important; max-height: 70% !important; @@ -693,10 +697,6 @@ div.populararguments div.dropdown-menu { overflow-y: scroll; } -#overrides-selection .override-search-button { - margin-left: 10px; -} - #overrides-selection .overrides-how-to-use { font-size: smaller; } @@ -744,6 +744,72 @@ div.populararguments div.dropdown-menu { display: none; } +#runtimetools-selection .modal-body { + overflow-y: scroll; +} + +#runtimetools-selection .runtimetools-how-to-use { + font-size: smaller; +} + +#runtimetools-selection .runtimetools-selected-col { + padding: 0 15px 0 0; + min-width: 250px; + max-width: 250px; +} + +#runtimetools-selection .runtimetools-results-col { + padding: 0 0 0 0; + min-width: 450px; + max-width: 650px; +} + +#runtimetools-selection .runtimetool-results-items .card { + margin-bottom: 3px; +} + +#runtimetools-selection.mobile .runtimetools-results-col { + min-width: 250px; + max-width: 450px; +} + +#runtimetools-selection .runtimetools-results-col span.override { + float: right; +} + +#runtimetools-selection .runtimetools-results-col span.override-fav { + float: right; +} + +#runtimetools-selection .runtimetools-favorites-col { + padding: 0 0 0 15px; + min-width: 325px; + max-width: 350px; +} + +#runtimetools-selection .runtimetools-favorites-col button { + width: 300px; +} + +#runtimetools-selection.mobile .runtimetools-favorites-col { + display: none; +} + +#runtimetools-selection .runtime-tool-option { + line-height: 35px; +} +#runtimetools-selection .tool-option-name { + display: inline-block; + min-width: 150px; +} +#runtimetools-selection .tool-option-select { + min-width: 100px; +} + +#runtimetools-selection .tool-fav { + float: right; +} + .ces-content-root { min-height: 100px; max-height: calc( diff --git a/static/widgets/runtime-tools.ts b/static/widgets/runtime-tools.ts new file mode 100644 index 00000000000..6c09f993a69 --- /dev/null +++ b/static/widgets/runtime-tools.ts @@ -0,0 +1,412 @@ +// Copyright (c) 2023, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import $ from 'jquery'; +import {options} from '../options.js'; +import {CompilerInfo} from '../compiler.interfaces.js'; +import {assert} from '../assert.js'; +import {localStorage} from '../local.js'; +import { + ConfiguredRuntimeTool, + ConfiguredRuntimeTools, + PossibleRuntimeTools, + RuntimeToolOption, + RuntimeToolOptions, + RuntimeToolType, +} from '../../types/execution/execution.interfaces.js'; + +const FAV_RUNTIMETOOLS_STORE_KEY = 'favruntimetools'; + +export type RuntimeToolsChangeCallback = () => void; + +type FavRuntimeTool = { + name: RuntimeToolType; + options: string; + meta: string; +}; + +type FavRuntimeTools = FavRuntimeTool[]; + +export class RuntimeToolsWidget { + private domRoot: JQuery; + private popupDomRoot: JQuery; + private envVarsInput: JQuery; + private dropdownButton: JQuery; + private onChangeCallback: RuntimeToolsChangeCallback; + private configured: ConfiguredRuntimeTools = []; + private compiler: CompilerInfo | undefined; + private possibleTools: PossibleRuntimeTools; + + constructor(domRoot: JQuery, dropdownButton: JQuery, onChangeCallback: RuntimeToolsChangeCallback) { + this.domRoot = domRoot; + this.popupDomRoot = $('#runtimetools-selection'); + this.dropdownButton = dropdownButton; + this.envVarsInput = this.popupDomRoot.find('.envvars'); + this.onChangeCallback = onChangeCallback; + this.possibleTools = []; + } + + private loadStateFromUI(): ConfiguredRuntimeTools { + const tools: ConfiguredRuntimeTools = []; + + const envOverrides = this.getEnvOverrides(); + if (envOverrides.length > 0) { + tools.push({ + name: RuntimeToolType.env, + options: envOverrides, + }); + } + + const selects = this.popupDomRoot.find('select'); + for (const select of selects) { + const jqSelect = $(select); + + const rawName = jqSelect.data('tool-name'); + const optionName = jqSelect.data('tool-option'); + + const val = jqSelect.val(); + if (val) { + const name = rawName as RuntimeToolType; + assert(name !== RuntimeToolType.env); + let tool = tools.find(tool => tool.name === name); + if (!tool) { + tool = { + name: name, + options: [], + }; + tools.push(tool); + } + + const option: RuntimeToolOption = { + name: optionName, + value: (val || '') as string, + }; + tool.options.push(option); + } + } + + return tools; + } + + private optionsToString(options: RuntimeToolOptions): string { + return options.map(env => `${env.name}=${env.value}`).join('\n'); + } + + private stringToOptions(options: string): RuntimeToolOptions { + return options + .split('\n') + .map(env => { + const arr = env.split('='); + if (arr[0]) { + return { + name: arr[0], + value: arr[1], + }; + } else { + return false; + } + }) + .filter(Boolean) as RuntimeToolOptions; + } + + private getEnvOverrides(): RuntimeToolOptions { + return this.stringToOptions(this.envVarsInput.val() as string); + } + + private selectOverrideFromFave(event) { + const elem = $(event.target).parent(); + const name = elem.data('ov-name'); + const optionsStr = elem.data('ov-options'); + const options = this.stringToOptions(optionsStr); + + const tool = this.possibleTools.find(ov => ov.name === name); + if (tool) { + const configuredTools = this.loadStateFromUI(); + let configuredTool = configuredTools.find(t => t.name === name); + if (!configuredTool) { + configuredTool = { + name: name, + options: [], + }; + configuredTools.push(configuredTool); + } + + configuredTool.options = options; + + this.loadStateIntoUI(configuredTools); + } + } + + private newFavoriteOverrideDiv(fave: FavRuntimeTool) { + const div = $('#overrides-favorite-tpl').children().clone(); + const prefix = fave.name + ': '; + div.find('.overrides-name').html(prefix + fave.options.replace(/\n/g, ', ')); + div.data('ov-name', fave.name); + div.data('ov-options', fave.options); + div.on('click', this.selectOverrideFromFave.bind(this)); + return div; + } + + private loadFavoritesIntoUI() { + const favoritesDiv = this.popupDomRoot.find('.runtimetools-favorites'); + favoritesDiv.html(''); + + const faves = this.getFavorites(); + for (const fave of faves) { + const div: any = this.newFavoriteOverrideDiv(fave); + favoritesDiv.append(div); + } + } + + private addToFavorites(override: ConfiguredRuntimeTool) { + if (override.name === RuntimeToolType.env) return; + + const faves = this.getFavorites(); + + const fave: FavRuntimeTool = { + name: override.name, + options: this.optionsToString(override.options), + meta: this.compiler?.baseName || this.compiler?.groupName || this.compiler?.name || this.compiler?.id || '', + }; + + faves.push(fave); + + this.setFavorites(faves); + } + + private removeFromFavorites(override: ConfiguredRuntimeTool) { + if (override.name === RuntimeToolType.env) return; + + const overrideOptions = this.optionsToString(override.options); + + const faves = this.getFavorites(); + const faveIdx = faves.findIndex(f => f.name === override.name && f.options === overrideOptions); + if (faveIdx !== -1) { + faves.splice(faveIdx, 1); + this.setFavorites(faves); + } + } + + private isAFavorite(override: ConfiguredRuntimeTool) { + if (override.name === RuntimeToolType.env) return false; + + const overrideOptions = this.optionsToString(override.options); + + const faves = this.getFavorites(); + const fave = faves.find(f => f.name === override.name && f.options === overrideOptions); + return !!fave; + } + + private cap(text: string) { + if (text.length > 0) { + return text[0].toUpperCase() + text.substring(1); + } + + return ''; + } + + private loadStateIntoUI(configured: ConfiguredRuntimeTools) { + this.envVarsInput.val(''); + + for (const config of configured) { + if (config.name === RuntimeToolType.env) { + this.envVarsInput.val(this.optionsToString(config.options)); + } + } + + const container = this.popupDomRoot.find('.possible-runtimetools'); + container.html(''); + + this.possibleTools = this.compiler?.possibleRuntimeTools || []; + + for (const possibleTool of this.possibleTools) { + const card = $('#possible-runtime-tool').children().clone(); + card.find('.tool-name').html(this.cap(possibleTool.name)); + card.find('.tool-description').html(possibleTool.description); + + const toolOptionsDiv = card.find('.tool-options'); + + const faveButton = card.find('.tool-fav-button'); + faveButton.hide(); + const faveStar = faveButton.find('.tool-fav-btn-icon'); + + const config = configured.find(c => c.name === possibleTool.name); + + for (const toolOption of possibleTool.possibleOptions) { + const optionDiv = $('#possible-runtime-tool-option').children().clone(); + optionDiv.attr('name', toolOption.name); + const display_text = this.cap(toolOption.name); + optionDiv.find('.tool-option-name').html(display_text); + + const select = optionDiv.find('select'); + select.data('tool-name', possibleTool.name); + select.data('tool-option', toolOption.name); + + const option = $('