From 87e009271c0067fd67276e78d85aedd08635139b Mon Sep 17 00:00:00 2001 From: Kevin Zhu Date: Wed, 28 Feb 2024 22:19:55 +1100 Subject: [PATCH 1/6] feat: repo selector --- src/api/sastEvents.ts | 37 +++++++---- src/api/sastSummary.ts | 29 ++++++--- src/api/scaEvents.ts | 31 ++++++---- src/api/scaSummary.ts | 28 ++++++--- src/api/secretsEvents.ts | 28 +++++---- src/api/secretsSummary.ts | 30 ++++++--- src/components/Fields/RepositoryField.tsx | 62 +++++++++++++++++++ src/components/Query/QueryEditor.tsx | 3 +- src/components/Query/SastEventsSubquery.tsx | 27 ++++---- src/components/Query/SastSummarySubquery.tsx | 27 ++++---- src/components/Query/ScaEventsSubquery.tsx | 23 ++++--- src/components/Query/ScaSummarySubquery.tsx | 23 ++++--- .../Query/SecretsEventsSubquery.tsx | 27 ++++---- .../Query/SecretsSummarySubquery.tsx | 23 ++++--- src/datasource.ts | 19 ++++++ src/types.ts | 12 ++-- 16 files changed, 300 insertions(+), 129 deletions(-) create mode 100644 src/components/Fields/RepositoryField.tsx diff --git a/src/api/sastEvents.ts b/src/api/sastEvents.ts index f64500e..94af29f 100644 --- a/src/api/sastEvents.ts +++ b/src/api/sastEvents.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { DataFrame, FieldType, TimeRange, createDataFrame } from '@grafana/data'; -import { FetchResponse } from '@grafana/runtime'; +import { FetchResponse, isFetchError } from '@grafana/runtime'; import { SastEventsQueryOptions } from 'types'; import { SastFinding } from './sastCommon'; @@ -254,6 +254,15 @@ const SastEventsApiResponseSchema = z.object({ type SastEventsEvent = z.infer; +interface SastEventsApiRequest { + branch?: string; + githubRepositoryIds?: string; + fromTime?: string; // ISO string + fromEvent?: string; + numItems?: number; //max 100 + sort?: "asc" | "desc"; +} + export const processSastEvents = async ( queryOptions: SastEventsQueryOptions, range: TimeRange, @@ -262,26 +271,32 @@ export const processSastEvents = async ( let events: SastEventsEvent[] = []; let prevEventId = null; for (let i = 0; i < MAX_API_REQUESTS; ++i) { - const params = { - ...(queryOptions.queryParameters.githubRepositoryId - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryId } + const params: SastEventsApiRequest = { + ...(queryOptions.queryParameters.githubRepositoryIds + ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } : {}), - ...(queryOptions.queryParameters.branch ? { severity: queryOptions.queryParameters.branch } : {}), + ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 ? { eventTypes: queryOptions.queryParameters.eventTypes.join(',') } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), sort: 'asc', }; - - const response: any = await request_fn('sast/events', params); + const endpointPath = 'sast/events'; + console.log(`[${endpointPath}] starting request with params:`, params); + const response: any = await request_fn(endpointPath, params); const parseResult = SastEventsApiResponseSchema.safeParse(response.data); if (!parseResult.success) { - console.error('Error in data from sast events API', parseResult.error); - console.log('sast events request:', params); - console.log('sast events response:', response); - throw new Error(`Data from the API is misformed. See console log for more details.`); + throw { + message: `Data from the API is misformed. Contact Nullify with the data below for help`, + data: { + endpoint: endpointPath, + request_params: params, + response: response, + data_validation_error: parseResult.error, + } + }; } if (parseResult.data.events) { diff --git a/src/api/sastSummary.ts b/src/api/sastSummary.ts index 52f02c2..7805963 100644 --- a/src/api/sastSummary.ts +++ b/src/api/sastSummary.ts @@ -10,23 +10,36 @@ const SastSummaryApiResponseSchema = z.object({ numItems: z.number(), }); +interface SastSummaryApiRequest { + githubRepositoryIds?: string; + severity?: string; +} + export const processSastSummary = async ( queryOptions: SastSummaryQueryOptions, request_fn: (endpoint_path: string, params?: Record) => Promise> ): Promise => { - const response = await request_fn('sast/summary', { - ...(queryOptions.queryParameters.githubRepositoryId - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryId } + const params: SastSummaryApiRequest = { + ...(queryOptions.queryParameters.githubRepositoryIds + ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } : {}), ...(queryOptions.queryParameters.severity ? { severity: queryOptions.queryParameters.severity } : {}), - }); - console.log('sast summary response:', response); + }; + const endpointPath = 'sast/summary'; + console.log(`[${endpointPath}] starting request with params:`, params); + const response = await request_fn(endpointPath, params); const parseResult = SastSummaryApiResponseSchema.safeParse(response.data); if (!parseResult.success) { - console.error('Error in data from sast summary API', parseResult.error); - console.log('SAST summary response:', response); - throw new Error(`Data from the API is misformed. See console log for more details.`); + throw { + message: `Data from the API is misformed. Contact Nullify with the data below for help`, + data: { + endpoint: endpointPath, + request_params: params, + response: response, + data_validation_error: parseResult.error, + } + }; } return createDataFrame({ diff --git a/src/api/scaEvents.ts b/src/api/scaEvents.ts index afcf046..5301f8d 100644 --- a/src/api/scaEvents.ts +++ b/src/api/scaEvents.ts @@ -205,11 +205,12 @@ type ScaEventsEvent = z.infer; interface ScaEventsApiRequest { branch?: string; - githubRepositoryId?: string; + githubRepositoryIds?: string; + eventTypes?: string; fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 - sort?: string; // asc | desc + sort?: "asc" | "desc"; } export const processScaEvents = async ( @@ -220,26 +221,32 @@ export const processScaEvents = async ( let events: ScaEventsEvent[] = []; let prevEventId = null; for (let i = 0; i < MAX_API_REQUESTS; ++i) { - const params: any = { - ...(queryOptions.queryParameters.githubRepositoryId - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryId } + const params: ScaEventsApiRequest = { + ...(queryOptions.queryParameters.githubRepositoryIds + ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } : {}), - ...(queryOptions.queryParameters.branch ? { severity: queryOptions.queryParameters.branch } : {}), + ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 ? { eventTypes: queryOptions.queryParameters.eventTypes.join(',') } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), sort: 'asc', }; - - const response = await request_fn('sca/events', params); + const endpointPath = 'sca/events'; + console.log(`[${endpointPath}] starting request with params:`, params); + const response = await request_fn(endpointPath, params); const parseResult = ScaEventsApiResponseSchema.safeParse(response.data); if (!parseResult.success) { - console.error('Error in data from sca events API', parseResult.error); - console.log('sca events request:', params); - console.log('sca events response:', response); - throw new Error(`Data from the API is misformed. See console log for more details.`); + throw { + message: `Data from the API is misformed. Contact Nullify with the data below for help`, + data: { + endpoint: endpointPath, + request_params: params, + response: response, + data_validation_error: parseResult.error, + } + }; } if (parseResult.data.events) { diff --git a/src/api/scaSummary.ts b/src/api/scaSummary.ts index 0843d41..2c7278c 100644 --- a/src/api/scaSummary.ts +++ b/src/api/scaSummary.ts @@ -9,22 +9,36 @@ const ScaSummaryApiResponseSchema = z.object({ numItems: z.number(), }); +interface ScaSummaryApiRequest { + githubRepositoryIds?: string; + severity?: string; +} + export const processScaSummary = async ( queryOptions: ScaSummaryQueryOptions, request_fn: (endpoint_path: string, params?: Record) => Promise> ): Promise => { - const response = await request_fn('sca/summary', { - ...(queryOptions.queryParameters.githubRepositoryId - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryId } + const params: ScaSummaryApiRequest = { + ...(queryOptions.queryParameters.githubRepositoryIds + ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } : {}), ...(queryOptions.queryParameters.package ? { package: queryOptions.queryParameters.package } : {}), - }); + }; + const endpointPath = 'sca/summary'; + console.log(`[${endpointPath}] starting request with params:`, params); + const response = await request_fn(endpointPath, params); const parseResult = ScaSummaryApiResponseSchema.safeParse(response.data); if (!parseResult.success) { - console.error('Error in data from sca summary API', parseResult.error); - console.log('sca summary response:', response); - throw new Error(`Data from the API is misformed. See console log for more details.`); + throw { + message: `Data from the API is misformed. Contact Nullify with the data below for help`, + data: { + endpoint: endpointPath, + request_params: params, + response: response, + data_validation_error: parseResult.error, + } + }; } return createDataFrame({ diff --git a/src/api/secretsEvents.ts b/src/api/secretsEvents.ts index 0d9cffc..006105c 100644 --- a/src/api/secretsEvents.ts +++ b/src/api/secretsEvents.ts @@ -91,11 +91,11 @@ type SecretsEventsEvent = z.infer; interface SecretsEventsApiRequest { branch?: string; - githubRepositoryId?: string; + githubRepositoryIds?: string; fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 - sort?: string; // asc | desc + sort?: "asc" | "desc"; } export const processSecretsEvents = async ( @@ -106,9 +106,9 @@ export const processSecretsEvents = async ( let events: SecretsEventsEvent[] = []; let prevEventId = null; for (let i = 0; i < MAX_API_REQUESTS; ++i) { - const params: any = { - ...(queryOptions.queryParameters.githubRepositoryId - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryId } + const params: SecretsEventsApiRequest = { + ...(queryOptions.queryParameters.githubRepositoryIds + ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } : {}), ...(queryOptions.queryParameters.branch ? { severity: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 @@ -117,15 +117,21 @@ export const processSecretsEvents = async ( ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), sort: 'asc', }; - - const response = await request_fn('secrets/events', params); + const endpointPath = 'secrets/events'; + console.log(`[${endpointPath}] starting request with params:`, params); + const response = await request_fn(endpointPath, params); const parseResult = SecretsEventsApiResponseSchema.safeParse(response.data); if (!parseResult.success) { - console.error('Error in data from secrets event API', parseResult.error); - console.log('Secrets event request:', params); - console.log('Secrets event response:', response); - throw new Error(`Data from the API is misformed. See console log for more details.`); + throw { + message: `Data from the API is misformed. Contact Nullify with the data below for help`, + data: { + endpoint: endpointPath, + request_params: params, + response: response, + data_validation_error: parseResult.error, + } + }; } if (parseResult.data.events) { diff --git a/src/api/secretsSummary.ts b/src/api/secretsSummary.ts index 7f98b6c..f87e083 100644 --- a/src/api/secretsSummary.ts +++ b/src/api/secretsSummary.ts @@ -9,24 +9,40 @@ const SecretsSummaryApiResponseSchema = z.object({ numItems: z.number(), }); +interface SecretsSummaryApiRequest { + githubRepositoryIds?: string; + branch?: string; + type?: string; + allowlisted?: boolean; +} + export const processSecretsSummary = async ( queryOptions: SecretsSummaryQueryOptions, request_fn: (endpoint_path: string, params?: Record) => Promise> ): Promise => { - const response = await request_fn('secrets/summary', { - ...(queryOptions.queryParameters.githubRepositoryId - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryId } + const params: SecretsSummaryApiRequest = { + ...(queryOptions.queryParameters.githubRepositoryIds + ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.type ? { type: queryOptions.queryParameters.type } : {}), ...(queryOptions.queryParameters.allowlisted ? { allowlisted: queryOptions.queryParameters.allowlisted } : {}), - }); + }; + const endpointPath = 'secrets/summary'; + console.log(`[${endpointPath}] starting request with params:`, params); + const response = await request_fn(endpointPath, params); const parseResult = SecretsSummaryApiResponseSchema.safeParse(response.data); if (!parseResult.success) { - console.error('Error in data from secrets summary API', parseResult.error); - console.log('secrets summary response:', response); - throw new Error(`Data from the API is misformed. See console log for more details.`); + throw { + message: `Data from the API is misformed. Contact Nullify with the data below for help`, + data: { + endpoint: endpointPath, + request_params: params, + response: response, + data_validation_error: parseResult.error, + } + }; } // console.log('parseResult', parseResult); diff --git a/src/components/Fields/RepositoryField.tsx b/src/components/Fields/RepositoryField.tsx new file mode 100644 index 0000000..ef7b2d1 --- /dev/null +++ b/src/components/Fields/RepositoryField.tsx @@ -0,0 +1,62 @@ +import { SelectableValue } from '@grafana/data'; +import { MultiSelect } from '@grafana/ui'; +import React, { useEffect, useState } from 'react'; + +export interface Repository { + id: string; + name: string; +} + +export interface RepositoryFieldProps { + getRepositories: () => Promise; + selectedRepositoryIds: number[]; + setSelectedRepositoryIds: (selectedRepositories: number[]) => void; +} + +export const RepositoryField = (props: RepositoryFieldProps) => { + const [selectedValues, setSelectedValues] = useState>>([]); + const [customOptions, setCustomOptions] = useState>>([]); + const [allRepositoryOptions, setAllRepositoryOptions] = useState> | undefined>( + undefined + ); + + useEffect(() => { + props.getRepositories().then((repos) => { + if (repos) { + setAllRepositoryOptions( + repos.map((repo) => { + return { label: `${repo.name} (${repo.id})`, value: parseInt(repo.id, 10) }; + }) + ); + } else { + setAllRepositoryOptions([]); + } + }); + }); + + return ( + <> + { + setSelectedValues(values); + props.setSelectedRepositoryIds(values.map((value) => value.value ?? -1)); + }} + isValidNewOption={(inputValue) => { + return !isNaN(Number(inputValue)); + }} + onCreateOption={(optionValue) => { + const optionNumber = Number(optionValue); + setCustomOptions([...customOptions, { value: optionNumber, label: `${optionNumber}` }]); + setSelectedValues([...selectedValues, { value: optionNumber, label: `${optionNumber}` }]); + }} + /> + + ); +}; diff --git a/src/components/Query/QueryEditor.tsx b/src/components/Query/QueryEditor.tsx index fe411bb..ec1d25d 100644 --- a/src/components/Query/QueryEditor.tsx +++ b/src/components/Query/QueryEditor.tsx @@ -38,7 +38,8 @@ export function QueryEditor(props: Props) { diff --git a/src/components/Query/SastSummarySubquery.tsx b/src/components/Query/SastSummarySubquery.tsx index e82ac73..94c10b5 100644 --- a/src/components/Query/SastSummarySubquery.tsx +++ b/src/components/Query/SastSummarySubquery.tsx @@ -1,8 +1,9 @@ -import React, { ChangeEvent } from 'react'; -import { Field, Input, Select } from '@grafana/ui'; +import React, { useState } from 'react'; +import { Field, Select } from '@grafana/ui'; import { QueryEditorProps, SelectableValue } from '@grafana/data'; import { NullifyDataSource } from '../../datasource'; -import { NullifyDataSourceOptions, NullifyEndpointPaths, NullifyQueryOptions } from '../../types'; +import { NullifyDataSourceOptions, NullifyQueryOptions } from '../../types'; +import { RepositoryField } from 'components/Fields/RepositoryField'; type Props = QueryEditorProps; @@ -16,13 +17,16 @@ const severity_options: Array> = [ export function SastSummarySubquery(props: Props) { const { query, onChange, onRunQuery } = props; + const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); - const onRepoIdChange = (event: ChangeEvent) => { + const onRepoIdsChange = (respositoryIds: number[]) => { + setSelectedRepositoryIds(respositoryIds); onChange({ ...query, endpoint: 'sast/summary', - queryParameters: { ...query.queryParameters, githubRepositoryId: event.target.value }, + queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, }); + onRunQuery(); }; const onSeverityChange = (new_severity: string) => { @@ -37,14 +41,13 @@ export function SastSummarySubquery(props: Props) { return query.endpoint === 'sast/summary' ? ( <> - ; @@ -22,13 +23,16 @@ const ScaEventTypeOptions: Array> = [ export function ScaEventsSubquery(props: Props) { const { query, onChange, onRunQuery } = props; + const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); - const onRepoIdChange = (event: ChangeEvent) => { + const onRepoIdsChange = (respositoryIds: number[]) => { + setSelectedRepositoryIds(respositoryIds); onChange({ ...query, endpoint: 'sca/events', - queryParameters: { ...query.queryParameters, githubRepositoryId: event.target.value }, + queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, }); + onRunQuery(); }; const onBranchChange = (event: ChangeEvent) => { @@ -43,14 +47,13 @@ export function ScaEventsSubquery(props: Props) { return query.endpoint === 'sca/events' ? ( <> - diff --git a/src/components/Query/ScaSummarySubquery.tsx b/src/components/Query/ScaSummarySubquery.tsx index db50464..931cdb0 100644 --- a/src/components/Query/ScaSummarySubquery.tsx +++ b/src/components/Query/ScaSummarySubquery.tsx @@ -1,20 +1,24 @@ -import React, { ChangeEvent } from 'react'; +import React, { ChangeEvent, useState } from 'react'; import { Field, Input } from '@grafana/ui'; import { QueryEditorProps } from '@grafana/data'; import { NullifyDataSource } from '../../datasource'; import { NullifyDataSourceOptions, NullifyQueryOptions } from '../../types'; +import { RepositoryField } from 'components/Fields/RepositoryField'; type Props = QueryEditorProps; export function ScaSummarySubquery(props: Props) { const { query, onChange, onRunQuery } = props; + const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); - const onRepoIdChange = (event: ChangeEvent) => { + const onRepoIdsChange = (respositoryIds: number[]) => { + setSelectedRepositoryIds(respositoryIds); onChange({ ...query, endpoint: 'sca/summary', - queryParameters: { ...query.queryParameters, githubRepositoryId: event.target.value }, + queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, }); + onRunQuery(); }; const onPackageChange = (event: ChangeEvent) => { @@ -28,14 +32,13 @@ export function ScaSummarySubquery(props: Props) { return query.endpoint === 'sca/summary' ? ( <> - diff --git a/src/components/Query/SecretsEventsSubquery.tsx b/src/components/Query/SecretsEventsSubquery.tsx index 37967f2..1f535c4 100644 --- a/src/components/Query/SecretsEventsSubquery.tsx +++ b/src/components/Query/SecretsEventsSubquery.tsx @@ -1,8 +1,9 @@ -import React, { ChangeEvent, FormEvent } from 'react'; -import { Checkbox, Field, Input, Select, VerticalGroup } from '@grafana/ui'; +import React, { ChangeEvent, FormEvent, useState } from 'react'; +import { Checkbox, Field, Input, VerticalGroup } from '@grafana/ui'; import { QueryEditorProps, SelectableValue } from '@grafana/data'; import { NullifyDataSource } from '../../datasource'; -import { NullifyDataSourceOptions, NullifyEndpointPaths, NullifyQueryOptions } from '../../types'; +import { NullifyDataSourceOptions, NullifyQueryOptions } from '../../types'; +import { RepositoryField } from 'components/Fields/RepositoryField'; type Props = QueryEditorProps; @@ -15,13 +16,16 @@ const SecretsEventTypeOptions: Array> = [ export function SecretsEventsSubquery(props: Props) { const { query, onChange, onRunQuery } = props; + const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); - const onRepoIdChange = (event: ChangeEvent) => { + const onRepoIdsChange = (respositoryIds: number[]) => { + setSelectedRepositoryIds(respositoryIds); onChange({ ...query, endpoint: 'secrets/events', - queryParameters: { ...query.queryParameters, githubRepositoryId: event.target.value }, + queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, }); + onRunQuery(); }; const onBranchChange = (event: ChangeEvent) => { @@ -36,14 +40,13 @@ export function SecretsEventsSubquery(props: Props) { return query.endpoint === 'secrets/events' ? ( <> - diff --git a/src/components/Query/SecretsSummarySubquery.tsx b/src/components/Query/SecretsSummarySubquery.tsx index b28b295..e575e5d 100644 --- a/src/components/Query/SecretsSummarySubquery.tsx +++ b/src/components/Query/SecretsSummarySubquery.tsx @@ -1,20 +1,24 @@ -import React, { ChangeEvent } from 'react'; +import React, { ChangeEvent, useState } from 'react'; import { Field, Input, Switch } from '@grafana/ui'; import { QueryEditorProps } from '@grafana/data'; import { NullifyDataSource } from '../../datasource'; import { NullifyDataSourceOptions, NullifyQueryOptions } from '../../types'; +import { RepositoryField } from 'components/Fields/RepositoryField'; type Props = QueryEditorProps; export function SecretsSummarySubquery(props: Props) { const { query, onChange, onRunQuery } = props; + const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); - const onRepoIdChange = (event: ChangeEvent) => { + const onRepoIdsChange = (respositoryIds: number[]) => { + setSelectedRepositoryIds(respositoryIds); onChange({ ...query, endpoint: 'secrets/summary', - queryParameters: { ...query.queryParameters, githubRepositoryId: event.target.value }, + queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, }); + onRunQuery(); }; const onBranchChange = (event: ChangeEvent) => { @@ -44,14 +48,13 @@ export function SecretsSummarySubquery(props: Props) { return query.endpoint === 'secrets/summary' ? ( <> - diff --git a/src/datasource.ts b/src/datasource.ts index 1d61b4d..ebff860 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -7,6 +7,7 @@ import { dateTime, } from '@grafana/data'; +import { z } from 'zod'; import _ from 'lodash'; import { getBackendSrv, isFetchError } from '@grafana/runtime'; import { lastValueFrom } from 'rxjs'; @@ -31,6 +32,24 @@ export class NullifyDataSource extends DataSourceApi): Promise { const promises = options.targets.map(async (target) => { if (target.endpoint === 'sast/summary') { diff --git a/src/types.ts b/src/types.ts index aa1379e..adf90d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -35,7 +35,7 @@ export interface SastSummaryQueryOptions extends BaseQueryOptions { } export interface SastSummaryQueryParameters { - githubRepositoryId?: string; + githubRepositoryIds?: number[]; severity?: string; } @@ -47,7 +47,7 @@ export interface SastEventsQueryOptions extends BaseQueryOptions { } export interface SastEventsQueryParameters { - githubRepositoryId?: string; + githubRepositoryIds?: number[]; branch?: string; eventTypes?: string[]; } @@ -60,7 +60,7 @@ export interface ScaSummaryQueryOptions extends BaseQueryOptions { } export interface ScaSummaryQueryParameters { - githubRepositoryId?: string; + githubRepositoryIds?: number[]; package?: string; } @@ -72,7 +72,7 @@ export interface ScaEventsQueryOptions extends BaseQueryOptions { } export interface ScaEventsQueryParameters { - githubRepositoryId?: string; + githubRepositoryIds?: number[]; branch?: string; eventTypes?: string[]; } @@ -85,7 +85,7 @@ export interface SecretsSummaryQueryOptions extends BaseQueryOptions { } export interface SecretsSummaryQueryParameters { - githubRepositoryId?: string; + githubRepositoryIds?: number[]; branch?: string; type?: string; allowlisted?: boolean; @@ -99,7 +99,7 @@ export interface SecretsEventsQueryOptions extends BaseQueryOptions { } export interface SecretsEventsQueryParameters { - githubRepositoryId?: string; + githubRepositoryIds?: number[]; branch?: string; eventTypes?: string[]; } From 23bda240c68b8ff5d783aba458d1f1e4996dc851 Mon Sep 17 00:00:00 2001 From: Kevin Zhu Date: Wed, 28 Feb 2024 22:20:19 +1100 Subject: [PATCH 2/6] formatting --- src/datasource.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/datasource.ts b/src/datasource.ts index ebff860..b4d7a78 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -35,10 +35,12 @@ export class NullifyDataSource extends DataSourceApi ({ status: 'error', message: `Error in Secrets Summary: ${JSON.stringify(err.message ?? err)}` })), + ).catch((err) => ({ + status: 'error', + message: `Error in Secrets Summary: ${JSON.stringify(err.message ?? err)}`, + })), processSecretsEvents( { refId: 'test', endpoint: 'secrets/events', queryParameters: {} }, testTimeRange, this._request.bind(this) - ).catch((err) => ({ status: 'error', message: `Error in Secrets Events: ${JSON.stringify(err.message ?? err)}` })), + ).catch((err) => ({ + status: 'error', + message: `Error in Secrets Events: ${JSON.stringify(err.message ?? err)}`, + })), ]; - + const results = await Promise.allSettled(promises); const err: string[] = []; @@ -137,7 +145,7 @@ export class NullifyDataSource extends DataSourceApi 0) { console.error('Test failed', err.join('\n')); return { From 1db4774a87186f8a7993c261b49225908baababc Mon Sep 17 00:00:00 2001 From: Kevin Zhu Date: Wed, 28 Feb 2024 22:46:51 +1100 Subject: [PATCH 3/6] feat: renames for updated API: - eventTypes => eventType[] - githubRepositoryIds => githubRepositoryId[] --- src/api/sastEvents.ts | 7 ++++--- src/api/sastSummary.ts | 4 ++-- src/api/scaEvents.ts | 14 +++++++------- src/api/scaSummary.ts | 4 ++-- src/api/secretsEvents.ts | 11 ++++++----- src/api/secretsSummary.ts | 12 ++++++------ src/components/Query/SecretsEventsSubquery.tsx | 6 +++--- src/components/Query/SecretsSummarySubquery.tsx | 16 ++++++++-------- src/types.ts | 4 ++-- 9 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/api/sastEvents.ts b/src/api/sastEvents.ts index 94af29f..d1cd94d 100644 --- a/src/api/sastEvents.ts +++ b/src/api/sastEvents.ts @@ -255,8 +255,9 @@ const SastEventsApiResponseSchema = z.object({ type SastEventsEvent = z.infer; interface SastEventsApiRequest { + githubRepositoryId?: number[]; branch?: string; - githubRepositoryIds?: string; + eventType?: string[]; fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 @@ -273,11 +274,11 @@ export const processSastEvents = async ( for (let i = 0; i < MAX_API_REQUESTS; ++i) { const params: SastEventsApiRequest = { ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } + ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 - ? { eventTypes: queryOptions.queryParameters.eventTypes.join(',') } + ? { eventType: queryOptions.queryParameters.eventTypes } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), sort: 'asc', diff --git a/src/api/sastSummary.ts b/src/api/sastSummary.ts index 7805963..95bd739 100644 --- a/src/api/sastSummary.ts +++ b/src/api/sastSummary.ts @@ -11,7 +11,7 @@ const SastSummaryApiResponseSchema = z.object({ }); interface SastSummaryApiRequest { - githubRepositoryIds?: string; + githubRepositoryId?: number[]; severity?: string; } @@ -21,7 +21,7 @@ export const processSastSummary = async ( ): Promise => { const params: SastSummaryApiRequest = { ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } + ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } : {}), ...(queryOptions.queryParameters.severity ? { severity: queryOptions.queryParameters.severity } : {}), }; diff --git a/src/api/scaEvents.ts b/src/api/scaEvents.ts index 5301f8d..787b258 100644 --- a/src/api/scaEvents.ts +++ b/src/api/scaEvents.ts @@ -9,7 +9,7 @@ const MAX_API_REQUESTS = 10; const _BaseEventSchema = z.object({ id: z.string(), time: z.string(), - timeUnix: z.number(), + timestampUnix: z.number(), }); const ScaEventsGitProvider = z.object({ @@ -204,9 +204,9 @@ const ScaEventsApiResponseSchema = z.object({ type ScaEventsEvent = z.infer; interface ScaEventsApiRequest { + githubRepositoryId?: number[]; branch?: string; - githubRepositoryIds?: string; - eventTypes?: string; + eventType?: string[]; fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 @@ -223,11 +223,11 @@ export const processScaEvents = async ( for (let i = 0; i < MAX_API_REQUESTS; ++i) { const params: ScaEventsApiRequest = { ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } + ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 - ? { eventTypes: queryOptions.queryParameters.eventTypes.join(',') } + ? { eventType: queryOptions.queryParameters.eventTypes } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), sort: 'asc', @@ -257,7 +257,7 @@ export const processScaEvents = async ( if (!parseResult.data.events || parseResult.data.events.length === 0 || !parseResult.data.nextEventId) { // No more events break; - } else if (parseResult.data.events[0].timeUnix > range.to.unix()) { + } else if (parseResult.data.events[0].timestampUnix > range.to.unix()) { // No more events required break; } else { @@ -272,7 +272,7 @@ export const processScaEvents = async ( { name: 'time', type: FieldType.time, - values: events.map((event) => new Date(event.timeUnix * 1000)), + values: events.map((event) => new Date(event.timestampUnix * 1000)), }, { name: 'type', type: FieldType.string, values: events.map((event) => event.type) }, { diff --git a/src/api/scaSummary.ts b/src/api/scaSummary.ts index 2c7278c..dfb9dd6 100644 --- a/src/api/scaSummary.ts +++ b/src/api/scaSummary.ts @@ -10,7 +10,7 @@ const ScaSummaryApiResponseSchema = z.object({ }); interface ScaSummaryApiRequest { - githubRepositoryIds?: string; + githubRepositoryId?: number[]; severity?: string; } @@ -20,7 +20,7 @@ export const processScaSummary = async ( ): Promise => { const params: ScaSummaryApiRequest = { ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } + ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } : {}), ...(queryOptions.queryParameters.package ? { package: queryOptions.queryParameters.package } : {}), }; diff --git a/src/api/secretsEvents.ts b/src/api/secretsEvents.ts index 006105c..e829077 100644 --- a/src/api/secretsEvents.ts +++ b/src/api/secretsEvents.ts @@ -90,12 +90,13 @@ const SecretsEventsApiResponseSchema = z.object({ type SecretsEventsEvent = z.infer; interface SecretsEventsApiRequest { + githubRepositoryId?: number[]; branch?: string; - githubRepositoryIds?: string; + eventType?: string[]; fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 - sort?: "asc" | "desc"; + sort?: 'asc' | 'desc'; } export const processSecretsEvents = async ( @@ -108,11 +109,11 @@ export const processSecretsEvents = async ( for (let i = 0; i < MAX_API_REQUESTS; ++i) { const params: SecretsEventsApiRequest = { ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } + ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } : {}), - ...(queryOptions.queryParameters.branch ? { severity: queryOptions.queryParameters.branch } : {}), + ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 - ? { eventTypes: queryOptions.queryParameters.eventTypes.join(',') } + ? { eventType: queryOptions.queryParameters.eventTypes } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), sort: 'asc', diff --git a/src/api/secretsSummary.ts b/src/api/secretsSummary.ts index f87e083..c4b265d 100644 --- a/src/api/secretsSummary.ts +++ b/src/api/secretsSummary.ts @@ -10,10 +10,10 @@ const SecretsSummaryApiResponseSchema = z.object({ }); interface SecretsSummaryApiRequest { - githubRepositoryIds?: string; + githubRepositoryId?: number[]; branch?: string; - type?: string; - allowlisted?: boolean; + secretType?: string; + isAllowlisted?: boolean; } export const processSecretsSummary = async ( @@ -22,11 +22,11 @@ export const processSecretsSummary = async ( ): Promise => { const params: SecretsSummaryApiRequest = { ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryIds: queryOptions.queryParameters.githubRepositoryIds.join(',') } + ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), - ...(queryOptions.queryParameters.type ? { type: queryOptions.queryParameters.type } : {}), - ...(queryOptions.queryParameters.allowlisted ? { allowlisted: queryOptions.queryParameters.allowlisted } : {}), + ...(queryOptions.queryParameters.secretType ? { secretType: queryOptions.queryParameters.secretType } : {}), + ...(queryOptions.queryParameters.isAllowlisted ? { isAllowlisted: queryOptions.queryParameters.isAllowlisted } : {}), }; const endpointPath = 'secrets/summary'; console.log(`[${endpointPath}] starting request with params:`, params); diff --git a/src/components/Query/SecretsEventsSubquery.tsx b/src/components/Query/SecretsEventsSubquery.tsx index 1f535c4..d56a219 100644 --- a/src/components/Query/SecretsEventsSubquery.tsx +++ b/src/components/Query/SecretsEventsSubquery.tsx @@ -18,12 +18,12 @@ export function SecretsEventsSubquery(props: Props) { const { query, onChange, onRunQuery } = props; const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); - const onRepoIdsChange = (respositoryIds: number[]) => { - setSelectedRepositoryIds(respositoryIds); + const onRepoIdsChange = (repositoryIds: number[]) => { + setSelectedRepositoryIds(repositoryIds); onChange({ ...query, endpoint: 'secrets/events', - queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIds: repositoryIds }, }); onRunQuery(); }; diff --git a/src/components/Query/SecretsSummarySubquery.tsx b/src/components/Query/SecretsSummarySubquery.tsx index e575e5d..3194d92 100644 --- a/src/components/Query/SecretsSummarySubquery.tsx +++ b/src/components/Query/SecretsSummarySubquery.tsx @@ -29,11 +29,11 @@ export function SecretsSummarySubquery(props: Props) { }); }; - const onTypeChange = (event: ChangeEvent) => { + const onSecretTypeChange = (event: ChangeEvent) => { onChange({ ...query, endpoint: 'secrets/summary', - queryParameters: { ...query.queryParameters, type: event.target.value }, + queryParameters: { ...query.queryParameters, secretType: event.target.value }, }); }; @@ -41,7 +41,7 @@ export function SecretsSummarySubquery(props: Props) { onChange({ ...query, endpoint: 'secrets/summary', - queryParameters: { ...query.queryParameters, allowlisted: event.target.checked }, + queryParameters: { ...query.queryParameters, isAllowlisted: event.target.checked }, }); }; @@ -65,17 +65,17 @@ export function SecretsSummarySubquery(props: Props) { value={query.queryParameters?.branch || ''} /> - + - + ) : ( diff --git a/src/types.ts b/src/types.ts index adf90d7..1c3f28a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -87,8 +87,8 @@ export interface SecretsSummaryQueryOptions extends BaseQueryOptions { export interface SecretsSummaryQueryParameters { githubRepositoryIds?: number[]; branch?: string; - type?: string; - allowlisted?: boolean; + secretType?: string; + isAllowlisted?: boolean; } // SECRETS EVENTS ENDPOINT From e4371d1c47070f1e553da234ac2c6df4f35faf5a Mon Sep 17 00:00:00 2001 From: Kevin Zhu Date: Sat, 2 Mar 2024 22:17:23 +1100 Subject: [PATCH 4/6] chore: update default datasource name --- provisioning/datasources/datasources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioning/datasources/datasources.yml b/provisioning/datasources/datasources.yml index f0435b8..b749011 100644 --- a/provisioning/datasources/datasources.yml +++ b/provisioning/datasources/datasources.yml @@ -1,7 +1,7 @@ apiVersion: 1 datasources: - - name: 'grafana' + - name: 'Nullify Datasource' type: 'nullify-grafana-datasource' access: proxy isDefault: false From 0dd999fe8dc2f380c415193d4b2c7fb5ca6f6708 Mon Sep 17 00:00:00 2001 From: Kevin Zhu Date: Sun, 3 Mar 2024 01:23:51 +1100 Subject: [PATCH 5/6] chore: formatting --- src/api/sastEvents.ts | 16 +++- src/api/sastSummary.ts | 14 +++- src/api/scaEvents.ts | 16 +++- src/api/scaSummary.ts | 8 +- src/api/secretsEvents.ts | 154 +++++++++++++++++++++++--------------- src/api/secretsSummary.ts | 30 ++++++-- 6 files changed, 158 insertions(+), 80 deletions(-) diff --git a/src/api/sastEvents.ts b/src/api/sastEvents.ts index d1cd94d..fcece22 100644 --- a/src/api/sastEvents.ts +++ b/src/api/sastEvents.ts @@ -261,7 +261,7 @@ interface SastEventsApiRequest { fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 - sort?: "asc" | "desc"; + sort?: 'asc' | 'desc'; } export const processSastEvents = async ( @@ -296,7 +296,7 @@ export const processSastEvents = async ( request_params: params, response: response, data_validation_error: parseResult.error, - } + }, }; } @@ -320,13 +320,21 @@ export const processSastEvents = async ( return createDataFrame({ refId: queryOptions.refId, fields: [ - { name: 'id', type: FieldType.string, values: events.map((event) => event.id) }, + { + name: 'id', + type: FieldType.string, + values: events.map((event) => event.id), + }, { name: 'time', type: FieldType.time, values: events.map((event) => new Date(event.timestampUnix * 1000)), }, - { name: 'type', type: FieldType.string, values: events.map((event) => event.type) }, + { + name: 'type', + type: FieldType.string, + values: events.map((event) => event.type), + }, { name: 'numFindings', type: FieldType.number, diff --git a/src/api/sastSummary.ts b/src/api/sastSummary.ts index 95bd739..e32d35e 100644 --- a/src/api/sastSummary.ts +++ b/src/api/sastSummary.ts @@ -38,14 +38,18 @@ export const processSastSummary = async ( request_params: params, response: response, data_validation_error: parseResult.error, - } + }, }; } return createDataFrame({ refId: queryOptions.refId, fields: [ - { name: 'id', type: FieldType.string, values: parseResult.data.vulnerabilities.map((vuln) => vuln.id) }, + { + name: 'id', + type: FieldType.string, + values: parseResult.data.vulnerabilities.map((vuln) => vuln.id), + }, { name: 'formatted_severity', type: FieldType.string, @@ -56,7 +60,11 @@ export const processSastSummary = async ( type: FieldType.string, values: parseResult.data.vulnerabilities.map((vuln) => vuln.severity), }, - { name: 'cwe', type: FieldType.number, values: parseResult.data.vulnerabilities.map((vuln) => vuln.cwe) }, + { + name: 'cwe', + type: FieldType.number, + values: parseResult.data.vulnerabilities.map((vuln) => vuln.cwe), + }, { name: 'language', type: FieldType.string, diff --git a/src/api/scaEvents.ts b/src/api/scaEvents.ts index 787b258..ea43539 100644 --- a/src/api/scaEvents.ts +++ b/src/api/scaEvents.ts @@ -210,7 +210,7 @@ interface ScaEventsApiRequest { fromTime?: string; // ISO string fromEvent?: string; numItems?: number; //max 100 - sort?: "asc" | "desc"; + sort?: 'asc' | 'desc'; } export const processScaEvents = async ( @@ -245,7 +245,7 @@ export const processScaEvents = async ( request_params: params, response: response, data_validation_error: parseResult.error, - } + }, }; } @@ -268,13 +268,21 @@ export const processScaEvents = async ( return createDataFrame({ refId: queryOptions.refId, fields: [ - { name: 'id', type: FieldType.string, values: events.map((event) => event.id) }, + { + name: 'id', + type: FieldType.string, + values: events.map((event) => event.id), + }, { name: 'time', type: FieldType.time, values: events.map((event) => new Date(event.timestampUnix * 1000)), }, - { name: 'type', type: FieldType.string, values: events.map((event) => event.type) }, + { + name: 'type', + type: FieldType.string, + values: events.map((event) => event.type), + }, { name: 'numFindings', type: FieldType.number, diff --git a/src/api/scaSummary.ts b/src/api/scaSummary.ts index dfb9dd6..c9bec4c 100644 --- a/src/api/scaSummary.ts +++ b/src/api/scaSummary.ts @@ -37,14 +37,18 @@ export const processScaSummary = async ( request_params: params, response: response, data_validation_error: parseResult.error, - } + }, }; } return createDataFrame({ refId: queryOptions.refId, fields: [ - { name: 'id', type: FieldType.string, values: parseResult.data.vulnerabilities?.map((vuln) => vuln.id) }, + { + name: 'id', + type: FieldType.string, + values: parseResult.data.vulnerabilities?.map((vuln) => vuln.id), + }, { name: 'isDirect', type: FieldType.string, diff --git a/src/api/secretsEvents.ts b/src/api/secretsEvents.ts index e829077..08e3108 100644 --- a/src/api/secretsEvents.ts +++ b/src/api/secretsEvents.ts @@ -131,7 +131,7 @@ export const processSecretsEvents = async ( request_params: params, response: response, data_validation_error: parseResult.error, - } + }, }; } @@ -150,71 +150,101 @@ export const processSecretsEvents = async ( } } - let ids: string[] = []; - let types: string[] = []; - let branchs: string[] = []; - let commits: string[] = []; - let repository_names: string[] = []; - let repository_ids: number[] = []; - let finding_ids: string[] = []; - let finding_secretTypes: string[] = []; - let finding_filePaths: string[] = []; - let finding_authors: string[] = []; - let finding_commits: string[] = []; - let finding_timeStamps: Date[] = []; - let finding_ruleIds: string[] = []; - let finding_entropys: number[] = []; - let finding_isBranchHeads: Array = []; - let finding_firstCommitTimestamps: Date[] = []; - let finding_isAllowlisteds: Array = []; - - for (const event of events) { - let findings = - event.type === 'new-finding' || event.type === 'new-allowlisted-finding' - ? [event.data.finding] - : event.data.findings ?? []; - - for (const finding of findings) { - ids.push(event.id); - types.push(event.type); - branchs.push(event.data.branch); - commits.push(event.data.commit); - repository_names.push(event.data.provider.github?.repositoryName ?? ''); - repository_ids.push(event.data.provider.github?.repositoryId ?? -1); - finding_ids.push(finding.id); - finding_secretTypes.push(finding.secretType); - finding_filePaths.push(finding.filePath); - finding_authors.push(finding.author); - finding_commits.push(finding.commit); - finding_timeStamps.push(new Date(finding.timeStamp)); - finding_ruleIds.push(finding.ruleId); - finding_entropys.push(finding.entropy); - finding_isBranchHeads.push(finding.isBranchHead); - finding_firstCommitTimestamps.push(new Date(finding.firstCommitTimestamp)); - finding_isAllowlisteds.push(finding.isAllowlisted); - } - } + const getFindings = (event: SecretsEventsEvent) => + event.type === 'new-finding' || event.type === 'new-allowlisted-finding' + ? [event.data.finding] + : event.data.findings ?? []; return createDataFrame({ refId: queryOptions.refId, fields: [ - { name: 'id', type: FieldType.string, values: ids }, - { name: 'type', type: FieldType.string, values: types }, - { name: 'branch', type: FieldType.string, values: branchs }, - { name: 'commit', type: FieldType.string, values: commits }, - { name: 'repository_name', type: FieldType.string, values: repository_names }, - { name: 'repository_id', type: FieldType.number, values: repository_ids }, - { name: 'finding_id', type: FieldType.string, values: finding_ids }, - { name: 'finding_secretType', type: FieldType.string, values: finding_secretTypes }, - { name: 'finding_filePath', type: FieldType.string, values: finding_filePaths }, - { name: 'finding_author', type: FieldType.string, values: finding_authors }, - { name: 'finding_commit', type: FieldType.string, values: finding_commits }, - { name: 'finding_timeStamp', type: FieldType.time, values: finding_timeStamps }, - { name: 'finding_ruleId', type: FieldType.string, values: finding_ruleIds }, - { name: 'finding_entropy', type: FieldType.number, values: finding_entropys }, - { name: 'finding_isBranchHead', type: FieldType.boolean, values: finding_isBranchHeads }, - { name: 'finding_firstCommitTimestamp', type: FieldType.time, values: finding_firstCommitTimestamps }, - { name: 'finding_isAllowlisted', type: FieldType.boolean, values: finding_isAllowlisteds }, + { + name: 'id', + type: FieldType.string, + values: events.map((event) => event.id), + }, + { + name: 'type', + type: FieldType.string, + values: events.map((event) => event.type), + }, + { + name: 'branch', + type: FieldType.string, + values: events.map((event) => event.data.branch), + }, + { + name: 'commit', + type: FieldType.string, + values: events.map((event) => event.data.commit), + }, + { + name: 'repository_name', + type: FieldType.string, + values: events.map((event) => event.data.provider.github?.repositoryName ?? ''), + }, + { + name: 'repository_id', + type: FieldType.number, + values: events.map((event) => event.data.provider.github?.repositoryId ?? -1), + }, + { + name: 'finding_id', + type: FieldType.string, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.id) ?? []), + }, + { + name: 'finding_secretType', + type: FieldType.string, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.secretType) ?? []), + }, + { + name: 'finding_filePath', + type: FieldType.string, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.filePath) ?? []), + }, + { + name: 'finding_author', + type: FieldType.string, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.author) ?? []), + }, + { + name: 'finding_commit', + type: FieldType.string, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.commit) ?? []), + }, + { + name: 'finding_timeStamp', + type: FieldType.time, + values: events.flatMap((event) => getFindings(event).map((finding) => new Date(finding.timeStamp)) ?? []), + }, + { + name: 'finding_ruleId', + type: FieldType.string, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.ruleId) ?? []), + }, + { + name: 'finding_entropy', + type: FieldType.number, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.entropy) ?? []), + }, + { + name: 'finding_isBranchHead', + type: FieldType.boolean, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.isBranchHead) ?? []), + }, + { + name: 'finding_firstCommitTimestamp', + type: FieldType.time, + values: events.flatMap( + (event) => getFindings(event).map((finding) => new Date(finding.firstCommitTimestamp)) ?? [] + ), + }, + { + name: 'finding_isAllowlisted', + type: FieldType.boolean, + values: events.flatMap((event) => getFindings(event).map((finding) => finding.isAllowlisted) ?? []), + }, ], }); }; diff --git a/src/api/secretsSummary.ts b/src/api/secretsSummary.ts index c4b265d..95802d2 100644 --- a/src/api/secretsSummary.ts +++ b/src/api/secretsSummary.ts @@ -50,21 +50,41 @@ export const processSecretsSummary = async ( return createDataFrame({ refId: queryOptions.refId, fields: [ - { name: 'id', type: FieldType.string, values: parseResult.data.secrets?.map((secret) => secret.id) }, + { + name: 'id', + type: FieldType.string, + values: parseResult.data.secrets?.map((secret) => secret.id), + }, { name: 'secretType', type: FieldType.string, values: parseResult.data.secrets?.map((secret) => secret.secretType), }, - { name: 'filePath', type: FieldType.string, values: parseResult.data.secrets?.map((secret) => secret.filePath) }, - { name: 'author', type: FieldType.string, values: parseResult.data.secrets?.map((secret) => secret.author) }, + { + name: 'filePath', + type: FieldType.string, + values: parseResult.data.secrets?.map((secret) => secret.filePath), + }, + { + name: 'author', + type: FieldType.string, + values: parseResult.data.secrets?.map((secret) => secret.author), + }, { name: 'timeStamp', type: FieldType.time, values: parseResult.data.secrets?.map((secret) => new Date(secret.timeStamp)), }, - { name: 'ruleId', type: FieldType.string, values: parseResult.data.secrets?.map((secret) => secret.ruleId) }, - { name: 'entropy', type: FieldType.number, values: parseResult.data.secrets?.map((secret) => secret.entropy) }, + { + name: 'ruleId', + type: FieldType.string, + values: parseResult.data.secrets?.map((secret) => secret.ruleId), + }, + { + name: 'entropy', + type: FieldType.number, + values: parseResult.data.secrets?.map((secret) => secret.entropy), + }, { name: 'isBranchHead', type: FieldType.boolean, From 25c576c0b30c0d38f79eac671780d3d6ae9a3afd Mon Sep 17 00:00:00 2001 From: Kevin Zhu Date: Mon, 4 Mar 2024 16:21:36 +1100 Subject: [PATCH 6/6] feat: repository custom variables for repository filter --- ...09093.json => Nullify Demo Dashboard.json} | 114 +++++++++++++----- src/api/sastEvents.ts | 13 +- src/api/sastSummary.ts | 15 ++- src/api/scaEvents.ts | 13 +- src/api/scaSummary.ts | 9 +- src/api/secretsEvents.ts | 14 ++- src/api/secretsSummary.ts | 19 +-- src/components/Fields/RepositoryField.tsx | 80 ++++++++---- src/components/Query/SastEventsSubquery.tsx | 12 +- src/components/Query/SastSummarySubquery.tsx | 12 +- src/components/Query/ScaEventsSubquery.tsx | 12 +- src/components/Query/ScaSummarySubquery.tsx | 12 +- .../Query/SecretsEventsSubquery.tsx | 12 +- .../Query/SecretsSummarySubquery.tsx | 12 +- src/components/VariableQueryEditor.tsx | 20 +++ src/datasource.ts | 13 +- src/module.ts | 4 +- src/types.ts | 20 ++- src/utils/utils.ts | 25 ++++ 19 files changed, 308 insertions(+), 123 deletions(-) rename example/{Nullify Demo Dashboard-1707705909093.json => Nullify Demo Dashboard.json} (92%) create mode 100644 src/components/VariableQueryEditor.tsx diff --git a/example/Nullify Demo Dashboard-1707705909093.json b/example/Nullify Demo Dashboard.json similarity index 92% rename from example/Nullify Demo Dashboard-1707705909093.json rename to example/Nullify Demo Dashboard.json index f250b59..842fe34 100644 --- a/example/Nullify Demo Dashboard-1707705909093.json +++ b/example/Nullify Demo Dashboard.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_NUL.DEV.NULLIFY.AI", - "label": "nul.dev.nullify.ai", + "name": "DS_NULLIFY", + "label": "Nullify", "description": "", "type": "datasource", "pluginId": "nullify-grafana-datasource", @@ -27,7 +27,7 @@ "type": "datasource", "id": "nullify-grafana-datasource", "name": "Nullify", - "version": "1.0.0" + "version": "0.0.0" }, { "type": "panel", @@ -62,7 +62,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -232,10 +232,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "sast/summary", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -316,7 +320,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -395,10 +399,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "secrets/events", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -448,7 +456,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -526,10 +534,14 @@ "constant": 6.5, "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "sast/summary", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -572,7 +584,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -651,10 +663,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "secrets/events", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -704,7 +720,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -781,10 +797,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "sast/summary", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -839,7 +859,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -915,10 +935,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "secrets/summary", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -947,7 +971,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -1025,10 +1049,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "sast/events", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -1038,7 +1066,7 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "fieldConfig": { "defaults": { @@ -1173,10 +1201,14 @@ { "datasource": { "type": "nullify-grafana-datasource", - "uid": "${DS_NUL.DEV.NULLIFY.AI}" + "uid": "${DS_NULLIFY}" }, "endpoint": "sca/summary", - "queryParameters": {}, + "queryParameters": { + "githubRepositoryIdsOrQueries": [ + "$repository" + ] + }, "refId": "A" } ], @@ -1245,7 +1277,31 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "current": {}, + "datasource": { + "type": "nullify-grafana-datasource", + "uid": "${DS_NULLIFY}" + }, + "definition": "Nullify Repository Query", + "description": "Filter for the repositories for which to query data", + "hide": 0, + "includeAll": true, + "label": "Repository", + "multi": true, + "name": "repository", + "options": [], + "query": { + "queryType": "Repository" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] }, "time": { "from": "now-7d", @@ -1255,6 +1311,6 @@ "timezone": "", "title": "Nullify Demo Dashboard", "uid": "ecf0f5c8-bf81-479a-b72f-94b5c2855d04", - "version": 6, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/src/api/sastEvents.ts b/src/api/sastEvents.ts index fcece22..08ad37b 100644 --- a/src/api/sastEvents.ts +++ b/src/api/sastEvents.ts @@ -3,6 +3,7 @@ import { DataFrame, FieldType, TimeRange, createDataFrame } from '@grafana/data' import { FetchResponse, isFetchError } from '@grafana/runtime'; import { SastEventsQueryOptions } from 'types'; import { SastFinding } from './sastCommon'; +import { unwrapRepositoryTemplateVariables } from 'utils/utils'; const MAX_API_REQUESTS = 10; @@ -273,15 +274,19 @@ export const processSastEvents = async ( let prevEventId = null; for (let i = 0; i < MAX_API_REQUESTS; ++i) { const params: SastEventsApiRequest = { - ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } + ...(queryOptions.queryParameters.githubRepositoryIdsOrQueries + ? { + githubRepositoryId: unwrapRepositoryTemplateVariables( + queryOptions.queryParameters.githubRepositoryIdsOrQueries + ), + } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 ? { eventType: queryOptions.queryParameters.eventTypes } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), - sort: 'asc', + sort: 'desc', }; const endpointPath = 'sast/events'; console.log(`[${endpointPath}] starting request with params:`, params); @@ -303,8 +308,6 @@ export const processSastEvents = async ( if (parseResult.data.events) { events.push(...parseResult.data.events); } - // console.log('parseResult', parseResult); - // console.log('events', events); if (!parseResult.data.events || parseResult.data.events.length === 0 || !parseResult.data.nextEventId) { // No more events diff --git a/src/api/sastSummary.ts b/src/api/sastSummary.ts index e32d35e..9c270af 100644 --- a/src/api/sastSummary.ts +++ b/src/api/sastSummary.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { DataFrame, FieldType, createDataFrame } from '@grafana/data'; import { FetchResponse } from '@grafana/runtime'; import { SastSummaryQueryOptions } from 'types'; -import { prepend_severity_idx } from 'utils/utils'; +import { prepend_severity_idx, unwrapRepositoryTemplateVariables } from 'utils/utils'; import { SastFinding } from './sastCommon'; const SastSummaryApiResponseSchema = z.object({ @@ -20,8 +20,12 @@ export const processSastSummary = async ( request_fn: (endpoint_path: string, params?: Record) => Promise> ): Promise => { const params: SastSummaryApiRequest = { - ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } + ...(queryOptions.queryParameters.githubRepositoryIdsOrQueries + ? { + githubRepositoryId: unwrapRepositoryTemplateVariables( + queryOptions.queryParameters.githubRepositoryIdsOrQueries + ), + } : {}), ...(queryOptions.queryParameters.severity ? { severity: queryOptions.queryParameters.severity } : {}), }; @@ -80,6 +84,11 @@ export const processSastSummary = async ( type: FieldType.boolean, values: parseResult.data.vulnerabilities.map((vuln) => vuln.isAllowlisted), }, + { + name: 'repositoryName', + type: FieldType.string, + values: parseResult.data.vulnerabilities.map((vuln) => vuln.repository), + }, ], }); }; diff --git a/src/api/scaEvents.ts b/src/api/scaEvents.ts index ea43539..4c11eb4 100644 --- a/src/api/scaEvents.ts +++ b/src/api/scaEvents.ts @@ -3,6 +3,7 @@ import { DataFrame, FieldType, TimeRange, createDataFrame } from '@grafana/data' import { FetchResponse } from '@grafana/runtime'; import { ScaEventsQueryOptions } from 'types'; import { ScaEventsDependencyFinding } from './scaCommon'; +import { unwrapRepositoryTemplateVariables } from 'utils/utils'; const MAX_API_REQUESTS = 10; @@ -222,15 +223,19 @@ export const processScaEvents = async ( let prevEventId = null; for (let i = 0; i < MAX_API_REQUESTS; ++i) { const params: ScaEventsApiRequest = { - ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } + ...(queryOptions.queryParameters.githubRepositoryIdsOrQueries + ? { + githubRepositoryId: unwrapRepositoryTemplateVariables( + queryOptions.queryParameters.githubRepositoryIdsOrQueries + ), + } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 ? { eventType: queryOptions.queryParameters.eventTypes } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), - sort: 'asc', + sort: 'desc', }; const endpointPath = 'sca/events'; console.log(`[${endpointPath}] starting request with params:`, params); @@ -252,8 +257,6 @@ export const processScaEvents = async ( if (parseResult.data.events) { events.push(...parseResult.data.events); } - // console.log('parseResult', parseResult); - // console.log('events', events); if (!parseResult.data.events || parseResult.data.events.length === 0 || !parseResult.data.nextEventId) { // No more events break; diff --git a/src/api/scaSummary.ts b/src/api/scaSummary.ts index c9bec4c..c6c8c69 100644 --- a/src/api/scaSummary.ts +++ b/src/api/scaSummary.ts @@ -3,6 +3,7 @@ import { DataFrame, FieldType, createDataFrame } from '@grafana/data'; import { FetchResponse } from '@grafana/runtime'; import { ScaSummaryQueryOptions } from 'types'; import { ScaEventsDependencyFinding } from './scaCommon'; +import { unwrapRepositoryTemplateVariables } from 'utils/utils'; const ScaSummaryApiResponseSchema = z.object({ vulnerabilities: z.array(ScaEventsDependencyFinding).nullable(), @@ -19,8 +20,12 @@ export const processScaSummary = async ( request_fn: (endpoint_path: string, params?: Record) => Promise> ): Promise => { const params: ScaSummaryApiRequest = { - ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } + ...(queryOptions.queryParameters.githubRepositoryIdsOrQueries + ? { + githubRepositoryId: unwrapRepositoryTemplateVariables( + queryOptions.queryParameters.githubRepositoryIdsOrQueries + ), + } : {}), ...(queryOptions.queryParameters.package ? { package: queryOptions.queryParameters.package } : {}), }; diff --git a/src/api/secretsEvents.ts b/src/api/secretsEvents.ts index 08e3108..4d5e51e 100644 --- a/src/api/secretsEvents.ts +++ b/src/api/secretsEvents.ts @@ -3,6 +3,7 @@ import { DataFrame, FieldType, TimeRange, createDataFrame } from '@grafana/data' import { FetchResponse } from '@grafana/runtime'; import { SecretsEventsQueryOptions } from 'types'; import { SecretsScannerFindingEvent } from './secretsCommon'; +import { unwrapRepositoryTemplateVariables } from 'utils/utils'; const MAX_API_REQUESTS = 10; @@ -108,20 +109,26 @@ export const processSecretsEvents = async ( let prevEventId = null; for (let i = 0; i < MAX_API_REQUESTS; ++i) { const params: SecretsEventsApiRequest = { - ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } + ...(queryOptions.queryParameters.githubRepositoryIdsOrQueries + ? { + githubRepositoryId: unwrapRepositoryTemplateVariables( + queryOptions.queryParameters.githubRepositoryIdsOrQueries + ), + } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.eventTypes && queryOptions.queryParameters.eventTypes.length > 0 ? { eventType: queryOptions.queryParameters.eventTypes } : {}), ...(prevEventId ? { fromEvent: prevEventId } : { fromTime: range.from.toISOString() }), - sort: 'asc', + sort: 'desc', }; const endpointPath = 'secrets/events'; console.log(`[${endpointPath}] starting request with params:`, params); const response = await request_fn(endpointPath, params); + console.log(`[${endpointPath}] response:`, response); + const parseResult = SecretsEventsApiResponseSchema.safeParse(response.data); if (!parseResult.success) { throw { @@ -138,7 +145,6 @@ export const processSecretsEvents = async ( if (parseResult.data.events) { events.push(...parseResult.data.events); } - // console.log('Secrets events', events); if (!parseResult.data.events || parseResult.data.events.length === 0 || !parseResult.data.nextEventId) { // No more events break; diff --git a/src/api/secretsSummary.ts b/src/api/secretsSummary.ts index 95802d2..3e353d8 100644 --- a/src/api/secretsSummary.ts +++ b/src/api/secretsSummary.ts @@ -1,8 +1,9 @@ import { z } from 'zod'; import { DataFrame, FieldType, createDataFrame } from '@grafana/data'; -import { FetchResponse } from '@grafana/runtime'; +import { FetchResponse, getTemplateSrv } from '@grafana/runtime'; import { SecretsSummaryQueryOptions } from 'types'; import { SecretsScannerFindingEvent } from './secretsCommon'; +import { unwrapRepositoryTemplateVariables } from 'utils/utils'; const SecretsSummaryApiResponseSchema = z.object({ secrets: z.array(SecretsScannerFindingEvent).nullable(), @@ -21,12 +22,18 @@ export const processSecretsSummary = async ( request_fn: (endpoint_path: string, params?: Record) => Promise> ): Promise => { const params: SecretsSummaryApiRequest = { - ...(queryOptions.queryParameters.githubRepositoryIds - ? { githubRepositoryId: queryOptions.queryParameters.githubRepositoryIds } + ...(queryOptions.queryParameters.githubRepositoryIdsOrQueries + ? { + githubRepositoryId: unwrapRepositoryTemplateVariables( + queryOptions.queryParameters.githubRepositoryIdsOrQueries + ), + } : {}), ...(queryOptions.queryParameters.branch ? { branch: queryOptions.queryParameters.branch } : {}), ...(queryOptions.queryParameters.secretType ? { secretType: queryOptions.queryParameters.secretType } : {}), - ...(queryOptions.queryParameters.isAllowlisted ? { isAllowlisted: queryOptions.queryParameters.isAllowlisted } : {}), + ...(queryOptions.queryParameters.isAllowlisted + ? { isAllowlisted: queryOptions.queryParameters.isAllowlisted } + : {}), }; const endpointPath = 'secrets/summary'; console.log(`[${endpointPath}] starting request with params:`, params); @@ -41,12 +48,10 @@ export const processSecretsSummary = async ( request_params: params, response: response, data_validation_error: parseResult.error, - } + }, }; } - // console.log('parseResult', parseResult); - return createDataFrame({ refId: queryOptions.refId, fields: [ diff --git a/src/components/Fields/RepositoryField.tsx b/src/components/Fields/RepositoryField.tsx index ef7b2d1..849b609 100644 --- a/src/components/Fields/RepositoryField.tsx +++ b/src/components/Fields/RepositoryField.tsx @@ -1,6 +1,7 @@ -import { SelectableValue } from '@grafana/data'; +import { SelectableValue, TypedVariableModel } from '@grafana/data'; +import { getTemplateSrv } from '@grafana/runtime'; import { MultiSelect } from '@grafana/ui'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; export interface Repository { id: string; @@ -9,30 +10,68 @@ export interface Repository { export interface RepositoryFieldProps { getRepositories: () => Promise; - selectedRepositoryIds: number[]; - setSelectedRepositoryIds: (selectedRepositories: number[]) => void; + selectedRepositoryIdsOrQueries: Array; + setSelectedRepositoryIdsOrQueries: (selectedRepositories: Array) => void; } export const RepositoryField = (props: RepositoryFieldProps) => { - const [selectedValues, setSelectedValues] = useState>>([]); + const [selectedValues, setSelectedValues] = useState>>( + props.selectedRepositoryIdsOrQueries.map((selected) => ({ label: selected.toString(), value: selected })) + ); const [customOptions, setCustomOptions] = useState>>([]); - const [allRepositoryOptions, setAllRepositoryOptions] = useState> | undefined>( + const [allRepositoryOptions, setAllRepositoryOptions] = useState> | undefined>( undefined ); + const getRepositoriesRef = useRef(props.getRepositories); + const selectedRepositoryIdsOrQueriesRef = useRef(props.selectedRepositoryIdsOrQueries); + + const onSelectedValuesChange = (newSelectedValues: Array>) => { + setSelectedValues(newSelectedValues); + const selectedRepositories = newSelectedValues + .map((selectable) => selectable.value) + .filter((repoIdOrQuery): repoIdOrQuery is string | number => repoIdOrQuery !== undefined); + props.setSelectedRepositoryIdsOrQueries(selectedRepositories); + }; + useEffect(() => { - props.getRepositories().then((repos) => { - if (repos) { - setAllRepositoryOptions( - repos.map((repo) => { - return { label: `${repo.name} (${repo.id})`, value: parseInt(repo.id, 10) }; - }) - ); - } else { + const formatRepositorySelectableValue = (repo: Repository): SelectableValue => { + return { label: `${repo.name} (${repo.id})`, value: Number(repo.id) }; + }; + const formatVariableSelectableValue = (variableName: string): SelectableValue => { + return { label: `$${variableName}`, value: `$${variableName}` }; + }; + + const variables: TypedVariableModel[] = getTemplateSrv().getVariables(); + + getRepositoriesRef + .current() + .then((repos) => { + if (repos) { + setAllRepositoryOptions([ + ...variables.map((variable) => formatVariableSelectableValue(variable.id)), + ...repos.map(formatRepositorySelectableValue), + ]); + setSelectedValues([ + ...variables + .map((variable) => formatVariableSelectableValue(variable.id)) + .filter((selectableValue) => + selectedRepositoryIdsOrQueriesRef.current.includes(selectableValue.value || '') + ), + ...repos + .map(formatRepositorySelectableValue) + .filter((selectableValue) => + selectedRepositoryIdsOrQueriesRef.current.includes(selectableValue.value || '') + ), + ]); + } else { + setAllRepositoryOptions([]); + } + }) + .catch(() => { setAllRepositoryOptions([]); - } - }); - }); + }); + }, []); return ( <> @@ -44,17 +83,14 @@ export const RepositoryField = (props: RepositoryFieldProps) => { noOptionsMessage="No repositories found. To query a repository not listed, enter the repository ID." allowCustomValue value={selectedValues} - onChange={(values) => { - setSelectedValues(values); - props.setSelectedRepositoryIds(values.map((value) => value.value ?? -1)); - }} + onChange={onSelectedValuesChange} isValidNewOption={(inputValue) => { return !isNaN(Number(inputValue)); }} onCreateOption={(optionValue) => { const optionNumber = Number(optionValue); setCustomOptions([...customOptions, { value: optionNumber, label: `${optionNumber}` }]); - setSelectedValues([...selectedValues, { value: optionNumber, label: `${optionNumber}` }]); + onSelectedValuesChange([...selectedValues, { value: optionNumber, label: `${optionNumber}` }]); }} /> diff --git a/src/components/Query/SastEventsSubquery.tsx b/src/components/Query/SastEventsSubquery.tsx index 6d2d1b8..1573e57 100644 --- a/src/components/Query/SastEventsSubquery.tsx +++ b/src/components/Query/SastEventsSubquery.tsx @@ -29,14 +29,14 @@ const SastEventTypeOptions: Array> = [ export function SastEventsSubquery(props: Props) { const { query, onChange, onRunQuery } = props; - const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); + const [selectedRepositories, setSelectedRepositories] = useState>(query.queryParameters?.githubRepositoryIdsOrQueries || []); - const onRepoIdsChange = (respositoryIds: number[]) => { - setSelectedRepositoryIds(respositoryIds); + const onRepositoriesChange = (repositories: Array<(number | string)>) => { + setSelectedRepositories(repositories); onChange({ ...query, endpoint: 'sast/events', - queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIdsOrQueries: repositories }, }); onRunQuery(); }; @@ -58,8 +58,8 @@ export function SastEventsSubquery(props: Props) { > diff --git a/src/components/Query/SastSummarySubquery.tsx b/src/components/Query/SastSummarySubquery.tsx index 94c10b5..9070a6d 100644 --- a/src/components/Query/SastSummarySubquery.tsx +++ b/src/components/Query/SastSummarySubquery.tsx @@ -17,14 +17,14 @@ const severity_options: Array> = [ export function SastSummarySubquery(props: Props) { const { query, onChange, onRunQuery } = props; - const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); + const [selectedRepositories, setSelectedRepositories] = useState>(query.queryParameters?.githubRepositoryIdsOrQueries || []); - const onRepoIdsChange = (respositoryIds: number[]) => { - setSelectedRepositoryIds(respositoryIds); + const onRepositoriesChange = (repositories: Array<(number | string)>) => { + setSelectedRepositories(repositories); onChange({ ...query, endpoint: 'sast/summary', - queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIdsOrQueries: repositories }, }); onRunQuery(); }; @@ -46,8 +46,8 @@ export function SastSummarySubquery(props: Props) { > > = [ export function ScaEventsSubquery(props: Props) { const { query, onChange, onRunQuery } = props; - const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); + const [selectedRepositories, setSelectedRepositories] = useState>(query.queryParameters?.githubRepositoryIdsOrQueries || []); - const onRepoIdsChange = (respositoryIds: number[]) => { - setSelectedRepositoryIds(respositoryIds); + const onRepositoriesChange = (repositories: Array<(number | string)>) => { + setSelectedRepositories(repositories); onChange({ ...query, endpoint: 'sca/events', - queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIdsOrQueries: repositories }, }); onRunQuery(); }; @@ -52,8 +52,8 @@ export function ScaEventsSubquery(props: Props) { > diff --git a/src/components/Query/ScaSummarySubquery.tsx b/src/components/Query/ScaSummarySubquery.tsx index 931cdb0..9a9e2d1 100644 --- a/src/components/Query/ScaSummarySubquery.tsx +++ b/src/components/Query/ScaSummarySubquery.tsx @@ -9,14 +9,14 @@ type Props = QueryEditorProps([]); + const [selectedRepositories, setSelectedRepositories] = useState>(query.queryParameters?.githubRepositoryIdsOrQueries || []); - const onRepoIdsChange = (respositoryIds: number[]) => { - setSelectedRepositoryIds(respositoryIds); + const onRepositoriesChange = (repositories: Array<(number | string)>) => { + setSelectedRepositories(repositories); onChange({ ...query, endpoint: 'sca/summary', - queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIdsOrQueries: repositories }, }); onRunQuery(); }; @@ -37,8 +37,8 @@ export function ScaSummarySubquery(props: Props) { > diff --git a/src/components/Query/SecretsEventsSubquery.tsx b/src/components/Query/SecretsEventsSubquery.tsx index d56a219..2aebfff 100644 --- a/src/components/Query/SecretsEventsSubquery.tsx +++ b/src/components/Query/SecretsEventsSubquery.tsx @@ -16,14 +16,14 @@ const SecretsEventTypeOptions: Array> = [ export function SecretsEventsSubquery(props: Props) { const { query, onChange, onRunQuery } = props; - const [selectedRepositoryIds, setSelectedRepositoryIds] = useState([]); + const [selectedRepositories, setSelectedRepositories] = useState>(query.queryParameters?.githubRepositoryIdsOrQueries || []); - const onRepoIdsChange = (repositoryIds: number[]) => { - setSelectedRepositoryIds(repositoryIds); + const onRepositoriesChange = (repositories: Array<(number | string)>) => { + setSelectedRepositories(repositories); onChange({ ...query, endpoint: 'secrets/events', - queryParameters: { ...query.queryParameters, githubRepositoryIds: repositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIdsOrQueries: repositories }, }); onRunQuery(); }; @@ -45,8 +45,8 @@ export function SecretsEventsSubquery(props: Props) { > diff --git a/src/components/Query/SecretsSummarySubquery.tsx b/src/components/Query/SecretsSummarySubquery.tsx index 3194d92..c35fa9e 100644 --- a/src/components/Query/SecretsSummarySubquery.tsx +++ b/src/components/Query/SecretsSummarySubquery.tsx @@ -9,14 +9,14 @@ type Props = QueryEditorProps([]); + const [selectedRepositories, setSelectedRepositories] = useState>(query.queryParameters?.githubRepositoryIdsOrQueries || []); - const onRepoIdsChange = (respositoryIds: number[]) => { - setSelectedRepositoryIds(respositoryIds); + const onRepositoriesChange = (repositories: Array<(number | string)>) => { + setSelectedRepositories(repositories); onChange({ ...query, endpoint: 'secrets/summary', - queryParameters: { ...query.queryParameters, githubRepositoryIds: respositoryIds }, + queryParameters: { ...query.queryParameters, githubRepositoryIdsOrQueries: repositories }, }); onRunQuery(); }; @@ -53,8 +53,8 @@ export function SecretsSummarySubquery(props: Props) { > diff --git a/src/components/VariableQueryEditor.tsx b/src/components/VariableQueryEditor.tsx new file mode 100644 index 0000000..9663460 --- /dev/null +++ b/src/components/VariableQueryEditor.tsx @@ -0,0 +1,20 @@ +import React, { useEffect, useState } from 'react'; +import { NullifyVariableQuery, VariableQueryType } from '../types'; + +interface VariableQueryProps { + query: NullifyVariableQuery; + onChange: (query: NullifyVariableQuery, definition: string) => void; +} + +export const VariableQueryEditor = ({ onChange, query }: VariableQueryProps) => { + useEffect(() => { + onChange({ queryType: VariableQueryType.Repository }, `Nullify ${VariableQueryType.Repository} Query`); + }, [onChange]); + + return ( + <> + This query variable enables the creation of a dashboard-wide filter for repositories. + {/* ADD SELECTOR FOR OTHER QUERY TYPES (e.g. Repo/Branch/Team) */} + + ); +}; diff --git a/src/datasource.ts b/src/datasource.ts index b4d7a78..d6b104c 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -9,10 +9,10 @@ import { import { z } from 'zod'; import _ from 'lodash'; -import { getBackendSrv, isFetchError } from '@grafana/runtime'; +import { getBackendSrv, getTemplateSrv, isFetchError } from '@grafana/runtime'; import { lastValueFrom } from 'rxjs'; -import { NullifyDataSourceOptions, NullifyQueryOptions } from './types'; +import { NullifyVariableQuery, NullifyDataSourceOptions, NullifyQueryOptions } from './types'; import { processSastSummary } from 'api/sastSummary'; import { processSastEvents } from 'api/sastEvents'; import { processScaSummary } from 'api/scaSummary'; @@ -32,6 +32,11 @@ export class NullifyDataSource extends DataSourceApi ({text: repo.name, value: repo.id})) || []; + } + async getRepositories() { const response = await this._request('admin/repositories'); const AdminRepositoriesSchema = z.object({ @@ -80,9 +85,11 @@ export class NullifyDataSource extends DataSourceApi(NullifyDataSource) .setConfigEditor(ConfigEditor) - .setQueryEditor(QueryEditor); + .setQueryEditor(QueryEditor) + .setVariableQueryEditor(VariableQueryEditor); diff --git a/src/types.ts b/src/types.ts index 1c3f28a..5146b6e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -16,6 +16,14 @@ export interface NullifySecureJsonData { apiKey?: string; } +export enum VariableQueryType { + Repository = 'Repository', +} + +export interface NullifyVariableQuery { + queryType: VariableQueryType; +} + export type NullifyEndpointPaths = | 'sast/summary' | 'sast/events' @@ -35,7 +43,7 @@ export interface SastSummaryQueryOptions extends BaseQueryOptions { } export interface SastSummaryQueryParameters { - githubRepositoryIds?: number[]; + githubRepositoryIdsOrQueries?: Array; severity?: string; } @@ -47,7 +55,7 @@ export interface SastEventsQueryOptions extends BaseQueryOptions { } export interface SastEventsQueryParameters { - githubRepositoryIds?: number[]; + githubRepositoryIdsOrQueries?: Array; branch?: string; eventTypes?: string[]; } @@ -60,7 +68,7 @@ export interface ScaSummaryQueryOptions extends BaseQueryOptions { } export interface ScaSummaryQueryParameters { - githubRepositoryIds?: number[]; + githubRepositoryIdsOrQueries?: Array; package?: string; } @@ -72,7 +80,7 @@ export interface ScaEventsQueryOptions extends BaseQueryOptions { } export interface ScaEventsQueryParameters { - githubRepositoryIds?: number[]; + githubRepositoryIdsOrQueries?: Array; branch?: string; eventTypes?: string[]; } @@ -85,7 +93,7 @@ export interface SecretsSummaryQueryOptions extends BaseQueryOptions { } export interface SecretsSummaryQueryParameters { - githubRepositoryIds?: number[]; + githubRepositoryIdsOrQueries?: Array; branch?: string; secretType?: string; isAllowlisted?: boolean; @@ -99,7 +107,7 @@ export interface SecretsEventsQueryOptions extends BaseQueryOptions { } export interface SecretsEventsQueryParameters { - githubRepositoryIds?: number[]; + githubRepositoryIdsOrQueries?: Array; branch?: string; eventTypes?: string[]; } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index a49b6ca..d836ed1 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,3 +1,5 @@ +import { getTemplateSrv } from '@grafana/runtime'; + export const prepend_severity_idx = (severity: string) => { severity = severity.toUpperCase(); switch (severity) { @@ -13,3 +15,26 @@ export const prepend_severity_idx = (severity: string) => { return severity; } }; + +export const unwrapRepositoryTemplateVariables = (githubRepositoryIdsOrQueries: Array) => { + const repoIds = githubRepositoryIdsOrQueries + ?.map((idOrQuery) => { + if (typeof idOrQuery === 'number') { + return idOrQuery; + } else { + return getTemplateSrv() + .replace(idOrQuery, undefined, 'csv') + .split(',') + .map((repoId) => parseInt(repoId, 10)) + .filter((id) => { + if (Number.isNaN(id)) { + console.error(`Selection for ${idOrQuery} variable is invalid: ${id}`); + } + return !Number.isNaN(id); + }); + } + }) + .flat(); + + return [...new Set(repoIds)]; +};