From ac4dfc677f9b097cd8df6ba96dd9b891b35bacbf Mon Sep 17 00:00:00 2001 From: Shorpo R Date: Fri, 8 Mar 2024 12:57:33 +0000 Subject: [PATCH] feat: Alert Acks rename ack -> acknowledgement --- docker-compose.dev.yml | 1 + docker-compose.yml | 1 + packages/api/src/controllers/alerts.ts | 84 +++++++++- packages/api/src/models/alert.ts | 26 +++ packages/api/src/routers/api/alerts.ts | 71 +++++++++ packages/api/src/routers/api/root.ts | 38 +++++ packages/api/src/tasks/checkAlerts.ts | 9 ++ packages/app/src/AlertsPage.tsx | 177 +++++++++++++++++++-- packages/app/src/DashboardPage.tsx | 14 +- packages/app/src/api.ts | 20 +++ packages/app/src/types.ts | 6 +- packages/app/styles/AlertsPage.module.scss | 1 - packages/app/styles/app.scss | 4 + 13 files changed, 436 insertions(+), 16 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 734bb3f62..d5f0bb978 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -168,6 +168,7 @@ services: CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} CLICKHOUSE_PASSWORD: worker CLICKHOUSE_USER: worker + EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋' FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS) HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1 HDX_NODE_BETA_MODE: 0 diff --git a/docker-compose.yml b/docker-compose.yml index 40339c0cc..b995914f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,6 +127,7 @@ services: CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} CLICKHOUSE_PASSWORD: worker CLICKHOUSE_USER: worker + EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋' FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS) HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1 HDX_NODE_BETA_MODE: 0 diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index c93a95b1d..534418e94 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -1,4 +1,5 @@ import { getHours, getMinutes } from 'date-fns'; +import { sign, verify } from 'jsonwebtoken'; import ms from 'ms'; import { z } from 'zod'; @@ -14,6 +15,8 @@ import Alert, { } from '@/models/alert'; import Dashboard, { IDashboard } from '@/models/dashboard'; import LogView, { ILogView } from '@/models/logView'; +import { IUser } from '@/models/user'; +import logger from '@/utils/logger'; import { alertSchema } from '@/utils/zod'; export type AlertInput = { @@ -34,6 +37,13 @@ export type AlertInput = { // Chart alerts dashboardId?: string; chartId?: string; + + // Silenced + silenced?: { + by?: ObjectId; + at: Date; + until: Date; + }; }; const getCron = (interval: AlertInterval) => { @@ -131,7 +141,7 @@ export const createAlert = async ( }).save(); }; -const dashboardLogViewIds = async (teamId: ObjectId) => { +const dashboardLogViewIds = async (teamId: ObjectId | string) => { const [dashboards, logViews] = await Promise.all([ Dashboard.find({ team: teamId }, { _id: 1 }), LogView.find({ team: teamId }, { _id: 1 }), @@ -194,7 +204,10 @@ export const getAlerts = async (teamId: ObjectId) => { }); }; -export const getAlertById = async (alertId: string, teamId: ObjectId) => { +export const getAlertById = async ( + alertId: ObjectId | string, + teamId: ObjectId | string, +) => { const [logViewIds, dashboardIds] = await dashboardLogViewIds(teamId); return Alert.findOne({ @@ -233,7 +246,10 @@ export const getAlertsWithLogViewAndDashboard = async (teamId: ObjectId) => { }).populate<{ logView: ILogView; dashboardId: IDashboard; - }>(['logView', 'dashboardId']); + silenced?: IAlert['silenced'] & { + by: IUser; + }; + }>(['logView', 'dashboardId', 'silenced.by']); }; export const deleteAlert = async (id: string, teamId: ObjectId) => { @@ -255,3 +271,65 @@ export const deleteAlert = async (id: string, teamId: ObjectId) => { ], }); }; + +export const generateAlertSilenceToken = async ( + alertId: ObjectId | string, + teamId: ObjectId | string, +) => { + const secret = process.env.EXPRESS_SESSION_SECRET; + + if (!secret) { + logger.error( + 'EXPRESS_SESSION_SECRET is not set for signing token, skipping alert silence JWT generation', + ); + return ''; + } + + const alert = await getAlertById(alertId, teamId); + if (alert == null) { + throw new Error('Alert not found'); + } + + const token = sign( + { alertId: alert._id.toString(), teamId: teamId.toString() }, + secret, + { expiresIn: '1h' }, + ); + + // Slack does not accept ids longer than 255 characters + if (token.length > 255) { + logger.error( + 'Alert silence JWT length is greater than 255 characters, this may cause issues with some clients.', + ); + } + + return token; +}; + +export const silenceAlertByToken = async (token: string) => { + const secret = process.env.EXPRESS_SESSION_SECRET; + + if (!secret) { + throw new Error('EXPRESS_SESSION_SECRET is not set for verifying token'); + } + + const decoded = verify(token, secret, { + algorithms: ['HS256'], + }) as { alertId: string; teamId: string }; + + if (!decoded?.alertId || !decoded?.teamId) { + throw new Error('Invalid token'); + } + + const alert = await getAlertById(decoded.alertId, decoded.teamId); + if (alert == null) { + throw new Error('Alert not found'); + } + + alert.silenced = { + at: new Date(), + until: new Date(Date.now() + ms('30m')), + }; + + return alert.save(); +}; diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 09d0b2b3d..5ec369156 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -52,6 +52,13 @@ export interface IAlert { // Chart alerts dashboardId?: ObjectId; chartId?: string; + + // Silenced + silenced?: { + by?: ObjectId; + at: Date; + until: Date; + }; } export type AlertDocument = mongoose.HydratedDocument; @@ -125,6 +132,25 @@ const AlertSchema = new Schema( type: String, required: false, }, + silenced: { + required: false, + type: { + by: { + type: mongoose.Schema.Types.ObjectId, + ref: 'User', + required: false, + }, + at: { + type: Date, + required: true, + }, + until: { + type: Date, + required: true, + }, + required: false, + }, + }, }, { timestamps: true, diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index e7bd1d122..ffd30f226 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -6,6 +6,7 @@ import { validateRequest } from 'zod-express-middleware'; import { createAlert, deleteAlert, + getAlertById, getAlertsWithLogViewAndDashboard, updateAlert, validateGroupByProperty, @@ -73,6 +74,13 @@ router.get('/', async (req, res, next) => { return { history, + silenced: alert.silenced + ? { + by: alert.silenced.by?.email, + at: alert.silenced.at, + until: alert.silenced.until, + } + : undefined, channel: _.pick(alert.channel, ['type']), ...(alert.dashboardId && { dashboard: { @@ -164,6 +172,69 @@ router.put( }, ); +router.post( + '/:id/silenced', + validateRequest({ + body: z.object({ + mutedUntil: z.string().datetime(), + }), + params: z.object({ + id: objectIdSchema, + }), + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null || req.user == null) { + return res.sendStatus(403); + } + + const alert = await getAlertById(req.params.id, teamId); + if (!alert) { + throw new Error('Alert not found'); + } + alert.silenced = { + by: req.user._id, + at: new Date(), + until: new Date(req.body.mutedUntil), + }; + await alert.save(); + + res.sendStatus(200); + } catch (e) { + next(e); + } + }, +); + +router.delete( + '/:id/silenced', + validateRequest({ + params: z.object({ + id: objectIdSchema, + }), + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const alert = await getAlertById(req.params.id, teamId); + if (!alert) { + throw new Error('Alert not found'); + } + alert.silenced = undefined; + await alert.save(); + + res.sendStatus(200); + } catch (e) { + next(e); + } + }, +); + router.delete( '/:id', validateRequest({ diff --git a/packages/api/src/routers/api/root.ts b/packages/api/src/routers/api/root.ts index 12ef31671..e3d8fae52 100644 --- a/packages/api/src/routers/api/root.ts +++ b/packages/api/src/routers/api/root.ts @@ -4,6 +4,10 @@ import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; import * as config from '@/config'; +import { + generateAlertSilenceToken, + silenceAlertByToken, +} from '@/controllers/alerts'; import { createTeam, isTeamExisting } from '@/controllers/team'; import { handleAuthError, redirectToDashboard } from '@/middleware/auth'; import TeamInvite from '@/models/teamInvite'; @@ -167,4 +171,38 @@ router.post('/team/setup/:token', async (req, res, next) => { } }); +router.get('/ext/silence-alert/:token', async (req, res) => { + let isError = false; + + try { + const token = req.params.token; + await silenceAlertByToken(token); + } catch (e) { + isError = true; + logger.error(e); + } + + // TODO: Create a template for utility pages + return res.send(` + + + HyperDX + + + +
+ +
+
+ ${ + isError + ? '

Link is invalid or expired. Please try again.

' + : '

Alert silenced. You can close this window now.

' + } + Back to HyperDX +
+ + `); +}); + export default router; diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index 076b1e35e..cf7b9db39 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -604,6 +604,15 @@ const fireChannelEvent = async ({ throw new Error('Team not found'); } + if ((alert.silenced?.until?.getTime() ?? 0) > Date.now()) { + logger.info({ + message: 'Skipped firing alert due to silence', + alert, + silenced: alert.silenced, + }); + return; + } + const attributesNested = expandToNestedObject(attributes); const templateView: AlertMessageTemplateDefaultView = { alert: { diff --git a/packages/app/src/AlertsPage.tsx b/packages/app/src/AlertsPage.tsx index 615a130bd..cd18b0e1f 100644 --- a/packages/app/src/AlertsPage.tsx +++ b/packages/app/src/AlertsPage.tsx @@ -2,7 +2,10 @@ import * as React from 'react'; import Head from 'next/head'; import Link from 'next/link'; import cx from 'classnames'; -import { formatRelative } from 'date-fns'; +import { add, Duration, formatRelative } from 'date-fns'; +import { ErrorBoundary } from 'react-error-boundary'; +import { useQueryClient } from 'react-query'; +import { toast } from 'react-toastify'; import { ArrayParam, useQueryParam, withDefault } from 'use-query-params'; import { Alert as MAlert, @@ -10,6 +13,7 @@ import { Button, Container, Group, + Menu, Stack, Tooltip, } from '@mantine/core'; @@ -19,6 +23,7 @@ import { withAppNav } from './layout'; import { Tags } from './Tags'; import type { Alert, AlertHistory, LogView } from './types'; import { AlertState } from './types'; +import { formatHumanReadableDate } from './utils'; import styles from '../styles/AlertsPage.module.scss'; @@ -81,7 +86,7 @@ function AlertHistoryCardList({ history }: { history: AlertHistory[] }) {
))} - {items.reverse().map((history, index) => ( + {[...items].reverse().map((history, index) => ( ))}
@@ -95,6 +100,163 @@ function disableAlert(alertId?: string) { // TODO do some lovely disabling of the alert here } +function AckAlert({ alert }: { alert: Alert }) { + const queryClient = useQueryClient(); + const silenceAlert = api.useSilenceAlert(); + const unsilenceAlert = api.useUnsilenceAlert(); + + const mutateOptions = React.useMemo( + () => ({ + onSuccess: () => { + queryClient.invalidateQueries('alerts'); + }, + onError: () => { + toast.error('Failed to silence alert, please try again later.'); + }, + }), + [queryClient], + ); + + const handleUnsilenceAlert = React.useCallback(() => { + unsilenceAlert.mutate(alert._id || '', mutateOptions); // TODO: update types + }, [alert._id, mutateOptions, unsilenceAlert]); + + const isNoLongerMuted = React.useMemo(() => { + return alert.silenced ? new Date() > new Date(alert.silenced.until) : false; + }, [alert.silenced]); + + const handleSilenceAlert = React.useCallback( + (duration: Duration) => { + const mutedUntil = add(new Date(), duration); + silenceAlert.mutate( + { + alertId: alert._id || '', // TODO: update types + mutedUntil, + }, + mutateOptions, + ); + }, + [alert._id, mutateOptions, silenceAlert], + ); + + if (alert.silenced?.at) { + return ( + Something went wrong}> + + + + + + + Acknowledged{' '} + {alert.silenced?.by ? ( + <> + by {alert.silenced?.by} + + ) : null}{' '} + on
+ {formatHumanReadableDate(new Date(alert.silenced?.at))} + .
+ + + + {isNoLongerMuted ? ( + 'Alert resumed.' + ) : ( + <> + Resumes{' '} + {formatHumanReadableDate(new Date(alert.silenced.until))} + + )} + + + {isNoLongerMuted ? 'Unacknowledge' : 'Resume alert'} + + + + + ); + } + + if (alert.state === 'ALERT') { + return ( + Something went wrong}> + + + + + + + Acknowledge and silence for + + + handleSilenceAlert({ + minutes: 30, + }) + } + > + 30 minutes + + + handleSilenceAlert({ + hours: 1, + }) + } + > + 1 hour + + + handleSilenceAlert({ + hours: 6, + }) + } + > + 6 hours + + + handleSilenceAlert({ + hours: 24, + }) + } + > + 24 hours + + + + + ); + } + + return null; +} + function AlertDetails({ alert }: { alert: AlertData }) { const alertName = React.useMemo(() => { if (alert.source === 'CHART' && alert.dashboard) { @@ -133,17 +295,13 @@ function AlertDetails({ alert }: { alert: AlertData }) {
{alert.state === AlertState.ALERT && ( - + Alert )} - {alert.state === AlertState.OK && ( - - Ok - - )} + {alert.state === AlertState.OK && Ok} {alert.state === AlertState.DISABLED && ( - + Disabled )} @@ -180,6 +338,7 @@ function AlertDetails({ alert }: { alert: AlertData }) { + {/* can we disable an alert that is alarming? hmmmmm */} {/* also, will make the alert jump from under the cursor to the disabled area */} diff --git a/packages/app/src/DashboardPage.tsx b/packages/app/src/DashboardPage.tsx index 2c3458d77..d7dd7b181 100644 --- a/packages/app/src/DashboardPage.tsx +++ b/packages/app/src/DashboardPage.tsx @@ -250,11 +250,21 @@ const Tile = forwardRef( zIndex={1} size={alert?.state === 'OK' ? 6 : 8} processing={alert?.state === 'ALERT'} - color={alert?.state === 'OK' ? 'green' : 'red'} + color={ + alert?.state === 'OK' + ? 'green' + : alert.silenced?.at + ? 'yellow' + : 'red' + } >
diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 271b0fe8c..7cd1d11e1 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -29,6 +29,11 @@ type ApiAlertInput = { chartId?: string; }; +type ApiAlertAckInput = { + alertId: string; + mutedUntil: Date; +}; + type ServicesResponse = { data: Record< string, @@ -572,6 +577,21 @@ const api = { }), ); }, + useSilenceAlert() { + return useMutation('alerts', async alertAck => + server(`alerts/${alertAck.alertId}/silenced`, { + method: 'POST', + json: alertAck, + }), + ); + }, + useUnsilenceAlert() { + return useMutation(`alerts`, async (alertId: string) => + server(`alerts/${alertId}/silenced`, { + method: 'DELETE', + }), + ); + }, useLogHistogram( q: string, startDate: Date, diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index 8ae615627..94e1fe1e4 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -96,7 +96,11 @@ export type Alert = { timezone?: string; type: AlertType; source: AlertSource; - + silenced?: { + by?: string; + at: string; + until: string; + }; // Log alerts logView?: string; message?: string; diff --git a/packages/app/styles/AlertsPage.module.scss b/packages/app/styles/AlertsPage.module.scss index 2dc5f77f0..88e07af89 100644 --- a/packages/app/styles/AlertsPage.module.scss +++ b/packages/app/styles/AlertsPage.module.scss @@ -4,7 +4,6 @@ .header { align-items: center; - background-color: $body-bg; border-bottom: 1px solid $slate-950; color: $slate-200; display: flex; diff --git a/packages/app/styles/app.scss b/packages/app/styles/app.scss index d31dcd61c..640a28519 100644 --- a/packages/app/styles/app.scss +++ b/packages/app/styles/app.scss @@ -797,3 +797,7 @@ div.react-datepicker { color: white; } } + +// html { +// filter: invert(1) hue-rotate(180deg); +// }