From 349f95386b1d502b202c9aabd70ef1d0ca4a3de1 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 12 Nov 2025 11:34:50 -0800 Subject: [PATCH 1/3] hidemetadata --- .../configs/teams/compilers/config.ts | 28 +- .../api_helper/backend/common/type.ts | 6 +- .../api_helper/backend/common/utils.ts | 12 +- .../compilers/compiler_benchmark_data.ts | 80 +----- .../backend/compilers/helpers/common.ts | 15 -- .../queryBuilderUtils/compilerQueryBuilder.ts | 242 ++++++++++++++++++ .../queryBuilderUtils/queryBuilder.ts | 2 +- .../api_helper/backend/dataFetchers/type.ts | 5 + 8 files changed, 293 insertions(+), 97 deletions(-) create mode 100644 torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts diff --git a/torchci/components/benchmark_v3/configs/teams/compilers/config.ts b/torchci/components/benchmark_v3/configs/teams/compilers/config.ts index e5cb9292b5..318072d430 100644 --- a/torchci/components/benchmark_v3/configs/teams/compilers/config.ts +++ b/torchci/components/benchmark_v3/configs/teams/compilers/config.ts @@ -140,6 +140,21 @@ const RENDER_MAPPING_BOOK = { }, }; +export function toQueryArch(device: string, arch: string) { + if (arch === undefined) return []; + if (!device) return []; + switch (device) { + case "rocm": + if (arch === "mi300x" || arch == "") return ["mi300x", "mi325x"]; + return [arch]; + default: + if (arch === "") { + return []; + } + return [arch]; + } +} + export const compilerQueryParameterConverter: QueryParameterConverter = ( inputs: QueryParameterConverterInputs ) => { @@ -154,16 +169,21 @@ export const compilerQueryParameterConverter: QueryParameterConverter = ( } let models = getModels(f.model); + + const device = DISPLAY_NAMES_TO_DEVICE_NAMES[f.deviceName]; + const arch = DISPLAY_NAMES_TO_ARCH_NAMES[f.deviceName]; + const arches = toQueryArch(device, arch); + const params = { commits: i.commits ?? [], branches: i.branches ?? [], workflows: workflows, compilers: compilerList, - arch: DISPLAY_NAMES_TO_ARCH_NAMES[f.deviceName], - device: DISPLAY_NAMES_TO_DEVICE_NAMES[f.deviceName], - dtype: f.dtype === "none" ? "" : f.dtype, + arches: arches, + devices: [device], + dtypes: f.dtype === "none" ? [] : [f.dtype], granularity: "hour", - mode: f.mode, + modes: [f.mode], models: models, startTime: dayjs.utc(i.timeRange.start).format("YYYY-MM-DDTHH:mm:ss"), stopTime: dayjs.utc(i.timeRange.end).format("YYYY-MM-DDTHH:mm:ss"), diff --git a/torchci/lib/benchmark/api_helper/backend/common/type.ts b/torchci/lib/benchmark/api_helper/backend/common/type.ts index 4bf5cac75d..29f1f72552 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/type.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/type.ts @@ -37,10 +37,10 @@ export const defaultCompilerGetBenchmarkDataInputs: any = { export const defaultListCommitsInputs: any = { branches: [], - device: "", + devices: [], arch: [], - dtype: "", - mode: "", + dtypes: [], + modes: [], startTime: "", stopTime: "", suites: [], diff --git a/torchci/lib/benchmark/api_helper/backend/common/utils.ts b/torchci/lib/benchmark/api_helper/backend/common/utils.ts index 8edc95652c..80460e4022 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/utils.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/utils.ts @@ -1,7 +1,7 @@ // Utility to extract params from either GET or POST import dayjs from "dayjs"; -import { queryClickhouseSaved } from "lib/clickhouse"; import { NextApiRequest } from "next"; +import { BenchmarkCompilerListCommitQueryBuilder } from "../dataFetchers/queryBuilderUtils/compilerQueryBuilder"; import { CommitResult } from "./type"; /** @@ -377,11 +377,19 @@ function subsampleCommitsByDate(data: any[], maxCount: number | undefined) { }; } +async function listCommitsFromDb(queryParams: any) { + // fetch metadata from db + const fetcher = new BenchmarkCompilerListCommitQueryBuilder(); + const data = await fetcher.applyQuery(queryParams); + const result = fetcher.postProcess(data); + return result; +} + export async function getCommitsWithSampling( tableName: string, queryParams: any ): Promise { - const commit_results = await queryClickhouseSaved(tableName, queryParams); + const commit_results = await listCommitsFromDb(queryParams); let maxCount = undefined; // if subsampling is specified, use it if (queryParams.sampling) { diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts index 12456d1d3e..09c60f0f40 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts @@ -1,20 +1,14 @@ -import { queryClickhouseSaved } from "lib/clickhouse"; import { CommitResult, CompilerQueryType, - defaultCompilerGetBenchmarkDataInputs, defaultCompilerGetTimeSeriesInputs, - defaultListCommitsInputs, } from "../common/type"; import { emptyTimeSeriesResponse, getCommitsWithSampling, } from "../common/utils"; -import { - extractBackendSqlStyle, - toApiArch, - toQueryArch, -} from "./helpers/common"; +import { BenchmarkCompilerBenchmarkDataQueryBuilder } from "../dataFetchers/queryBuilderUtils/compilerQueryBuilder"; +import { extractBackendSqlStyle, toApiArch } from "./helpers/common"; import { toGeneralCompilerData } from "./helpers/general"; import { toPrecomputeCompilerData } from "./helpers/precompute"; @@ -23,26 +17,6 @@ const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query"; const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = "compilers_benchmark_api_commit_query"; -// TODO(ELAINEWY): add GET BENCHMARK DATA API -/** - * backend method to get single compiler benchmark data - * must provide workflow and branch in inputParams - */ -export async function getSingleCompilerBenchmarkData( - request_name: string, - inputParams: any, - formats: string[] = ["raw"] -) { - const queryParams = await getSingleCompilerBenchmarkDataQueryParams( - inputParams - ); - const rows = await fetchCompilerDataFromDb(queryParams); - if (rows.length === 0) { - return emptyTimeSeriesResponse(); - } - return toCompilerResponseFormat(rows, formats, request_name); -} - /** * backend method to get time series data */ @@ -84,47 +58,15 @@ export function toCompilerResponseFormat( } export async function getCompilerCommits( - inputparams: any + inputParams: any ): Promise { - if (!inputparams.startTime || !inputparams.stopTime) { + if (!inputParams.startTime || !inputParams.stopTime) { throw new Error("no start/end time provided in request"); } - const queryParams = { - ...defaultListCommitsInputs, // base defaults - ...inputparams, // override with caller's values - }; - - const arch_list = toQueryArch(inputparams.device, inputparams.arch); - queryParams["arch"] = arch_list; return await getCommitsWithSampling( COMPILER_BENCHMARK_COMMITS_TABLE_NAME, - queryParams - ); -} - -// TODO(ELAINEWY): add GET BENCHMARK DATA API -async function getSingleCompilerBenchmarkDataQueryParams( - inputparams: any -): Promise { - const queryParams = { - ...defaultCompilerGetBenchmarkDataInputs, // base defaults - ...inputparams, // override with caller's values - }; - const arch_list = toQueryArch(queryParams.device, queryParams.arch); - queryParams["arch"] = arch_list; - - if (!queryParams.workflow || !queryParams.branch) { - throw new Error( - "no workflow or branch provided in request for single data fetch" - ); - } - queryParams["workflows"] = [queryParams.workflow]; - queryParams["branches"] = [queryParams.branch]; - console.log( - "(getSingleCompilerBenchmarkDataQueryParams) workflows provided in request", - queryParams.workflows + inputParams ); - return queryParams; } /** @@ -140,10 +82,6 @@ export async function getCompilerBenchmarkTimeRangeQueryParams( ...defaultCompilerGetTimeSeriesInputs, // base defaults ...inputparams, // override with caller's values }; - - const arch_list = toQueryArch(queryParams.device, queryParams.arch); - queryParams["arch"] = arch_list; - // todo(elainewy): support lworkfow and rworkflow in the future for time range query // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided @@ -152,7 +90,6 @@ export async function getCompilerBenchmarkTimeRangeQueryParams( "(getCompilerBenchmarkTimeRangeQueryParams) no start/end time provided in request" ); } - if (!queryParams.workflows || queryParams.workflows.length == 0) { const { data: commit_results } = await getCommitsWithSampling( COMPILER_BENCHMARK_COMMITS_TABLE_NAME, @@ -188,10 +125,9 @@ async function fetchCompilerDataFromDb(queryParams: any): Promise { const start = Date.now(); let rows: any[] = []; try { - rows = await queryClickhouseSaved( - COMPILER_BENCHMARK_TABLE_NAME, - queryParams - ); + const fetcher = new BenchmarkCompilerBenchmarkDataQueryBuilder(); + const data = await fetcher.applyQuery(queryParams); + rows = fetcher.postProcess(data); } catch (err: any) { throw Error( `${COMPILER_BENCHMARK_TABLE_NAME}(clickhouse query issue) ${err.message}` diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts index d058f41972..4f53103c63 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts @@ -14,21 +14,6 @@ export function extractBackendSqlStyle( return m ? m[1] : null; } -export function toQueryArch(device: string, arch: string) { - if (arch === undefined) return []; - if (!device) return []; - switch (device) { - case "rocm": - if (arch === "mi300x" || arch == "") return ["mi300x", "mi325x"]; - return [arch]; - default: - if (arch === "") { - return []; - } - return [arch]; - } -} - export function toApiArch(device: string, arch: string): string { const norm = arch.toLowerCase(); switch (device) { diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts new file mode 100644 index 0000000000..154e3d03fb --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts @@ -0,0 +1,242 @@ +import { BenchmarkListCommitFetcher } from "../type"; +import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; + +export class BenchmarkCompilerBenchmarkDataQueryBuilder + extends ExecutableQueryBase + implements BenchmarkListCommitFetcher +{ + private builder: QueryBuilder; + private _DEFAULT_QUERY_PARAMS = { + branches: [], + devices: [], + arch: [], + dtypes: [], + modes: [], + suites: [], + startTime: "", + stopTime: "", + }; + constructor() { + super(); + this.builder = new QueryBuilder( + { + table: "benchmark.oss_ci_benchmark_torchinductor", + select_exists: true, + where_exists: true, + }, + ` + SELECT + workflow_id, + job_id, + head_sha AS commit, + replaceOne(head_branch, 'refs/heads/', '') AS branch, + suite, + model_name AS model, + metric_name AS metric, + value, + metric_extra_info AS extra_info, + benchmark_extra_info['output'] AS output, + benchmark_dtype AS dtype, + benchmark_mode AS mode, + device, + arch, + timestamp, + DATE_TRUNC({granularity: String}, fromUnixTimestamp(timestamp)) + AS granularity_bucket + {{SELECT}} + FROM {{TABLE}} + {{PREWHERE}} + WHERE + workflow_id IN ({workflows: Array(UInt64)}) + AND ( + has( + {branches: Array(String)}, + replaceOne(head_branch, 'refs/heads/', '') + ) + OR empty({branches: Array(String)}) + ) + AND ( + has({suites: Array(String) }, suite) + OR empty({suites: Array(String) }) + ) + AND ( + has({models: Array(String)}, model_name) + OR empty({models: Array(String) }) + ) + AND ( + has({dtypes: Array(String)}, benchmark_dtype) + OR empty({dtypes: Array(String) }) + ) + AND ( + has({modes: Array(String)}, benchmark_mode) + OR empty({modes: Array(String) }) + ) + AND ( + has({devices: Array(String)}, device) + OR empty({devices: Array(String) }) + ) + AND ( + multiSearchAnyCaseInsensitive(arch, {arches: Array(String)}) + OR empty({arches: Array(String)}) + ) + {{WHERE}} +ORDER BY timestamp +SETTINGS session_timezone = 'UTC'; +` + ); + } + + build() { + return this.builder.build(); + } + addWhere(where: string[]) { + this.builder.addWhere(where); + } + toQueryParams(inputs: any) { + const params = compilerParmsToQueryInput(inputs); + return { + ...this._DEFAULT_QUERY_PARAMS, + ...params, + }; + } + postProcess(data: any) { + return data; + } +} + +export class BenchmarkCompilerListCommitQueryBuilder + extends ExecutableQueryBase + implements BenchmarkListCommitFetcher +{ + private builder: QueryBuilder; + private _DEFAULT_QUERY_PARAMS = { + branches: [], + devices: [], + arch: [], + dtypes: [], + modes: [], + suites: [], + startTime: "", + stopTime: "", + }; + constructor() { + super(); + this.builder = new QueryBuilder( + { + table: "benchmark.oss_ci_benchmark_torchinductor", + select_exists: true, + where_exists: true, + prewhere: [ + "timestamp >= toUnixTimestamp({startTime: DateTime64(3)})", + "timestamp < toUnixTimestamp({stopTime: DateTime64(3)})", + ], + }, + ` + SELECT + replaceOne(head_branch, 'refs/heads/', '') AS branch, + head_sha AS commit, + workflow_id, + toStartOfHour(min(fromUnixTimestamp(timestamp))) AS date + {{SELECT}} + FROM {{TABLE}} + {{PREWHERE}} +WHERE + ( + has( + {branches: Array(String)}, + replaceOne(head_branch, 'refs/heads/', '') + ) + OR empty({branches: Array(String)}) + ) + AND ( + has({suites: Array(String)}, suite) + OR empty({suites: Array(String)}) + ) + AND ( + has({dtypes: Array(String) },benchmark_dtype) + OR empty({dtypes: Array(String) }) + ) + AND ( + has({modes: Array(String) },benchmark_mode) + OR empty({modes: Array(String) }) + ) + AND ( + has({devices: Array(String) },device) + OR empty({devices: Array(String) }) + ) + AND ( + multiSearchAnyCaseInsensitive(arch, {arches: Array(String)}) + OR empty({arches: Array(String)}) + ) + {{WHERE}} +GROUP BY + branch, commit, workflow_id +ORDER BY + branch, date +SETTINGS session_timezone = 'UTC'; +` + ); + } + + build() { + return this.builder.build(); + } + addWhere(where: string[]) { + this.builder.addWhere(where); + } + toQueryParams(inputs: any) { + const hasRocm = [inputs?.device, inputs?.devices].some((v) => + Array.isArray(v) ? v.includes("rocm") : v === "rocm" + ); + const hasA100 = [inputs?.arch, inputs?.arches].some((v) => + Array.isArray(v) ? v.includes("a100") : v === "a100" + ); + if (hasRocm || hasA100) { + this.builder.addWhere([ + ` + NOT endsWith(benchmark_extra_info['output'], 'huggingface.csv') + AND NOT endsWith(benchmark_extra_info['output'], 'torchbench.csv') + AND NOT endsWith(benchmark_extra_info['output'], 'timm_models.csv') + `, + ]); + } + const params = compilerParmsToQueryInput(inputs); + return { + ...this._DEFAULT_QUERY_PARAMS, + ...params, + }; + } + postProcess(data: any) { + return data; + } +} + +function toPlural(inputs: any) { + if (inputs.branch && !inputs.branches) { + inputs.branches = [inputs.branch]; + } + if (inputs.suite && !inputs.suites) { + inputs.suites = [inputs.suite]; + } + + if (inputs.dtype && !inputs.dtypes) { + inputs.dtypes = [inputs.dtype]; + } + if (inputs.model && !inputs.models) { + inputs.models = [inputs.model]; + } + if (inputs.device && !inputs.devices) { + inputs.devices = [inputs.devices]; + } + if (inputs.arch && !inputs.arches) { + inputs.arch = inputs.arches; + } +} + +function compilerParmsToQueryInput(inputs: any) { + const queryParams = { + ...inputs, // override with caller's values + }; + toPlural(queryParams); + return queryParams; +} diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts index 02b3bacbc7..7337dff8d7 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts @@ -198,8 +198,8 @@ export abstract class ExecutableQueryBase implements BuildableQuery { inputs: any, executor: QueryExecutor = queryClickhouse ): Promise { - const sql = this.build(); const params = this.toQueryParams(inputs); + const sql = this.build(); return executor(sql, params); } } diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts index 07641e7c9e..79a8c171b0 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts @@ -11,3 +11,8 @@ export interface BenchmarkMetadataFetcher { postProcess(data: any[]): any; applyQuery(inputs: string): Promise; } + +export interface BenchmarkListCommitFetcher { + postProcess(data: any[]): any; + applyQuery(inputs: string): Promise; +} From 98677b0778155381da8184ab7567db0129483eb0 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 12 Nov 2025 11:45:03 -0800 Subject: [PATCH 2/3] hidemetadata --- .../queryBuilderUtils/compilerQueryBuilder.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts index 154e3d03fb..f349cbe4f5 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts @@ -201,6 +201,11 @@ SETTINGS session_timezone = 'UTC'; ]); } const params = compilerParmsToQueryInput(inputs); + + console.log("apply query", { + ...this._DEFAULT_QUERY_PARAMS, + ...params, + }); return { ...this._DEFAULT_QUERY_PARAMS, ...params, @@ -226,10 +231,10 @@ function toPlural(inputs: any) { inputs.models = [inputs.model]; } if (inputs.device && !inputs.devices) { - inputs.devices = [inputs.devices]; + inputs.devices = [inputs.device]; } if (inputs.arch && !inputs.arches) { - inputs.arch = inputs.arches; + inputs.arches = [inputs.arch]; } } From 91843860e4aa1147954eb9efd5ffce08ec182c9a Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 12 Nov 2025 13:01:12 -0800 Subject: [PATCH 3/3] hidemetadata --- .../dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts index f349cbe4f5..5a4fd8ad34 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/compilerQueryBuilder.ts @@ -201,11 +201,6 @@ SETTINGS session_timezone = 'UTC'; ]); } const params = compilerParmsToQueryInput(inputs); - - console.log("apply query", { - ...this._DEFAULT_QUERY_PARAMS, - ...params, - }); return { ...this._DEFAULT_QUERY_PARAMS, ...params,